diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts index 4a32e94faa..1f528294dc 100644 --- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts +++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts @@ -182,6 +182,7 @@ describe("HyperframesRenderStack — snapshot", () => { present: true, }); } + expect(collected.has("FONT_FETCH_UNAVAILABLE")).toBe(false); }); it("classifies plan v2 integrity failures as terminal in every v2 Lambda task", () => { diff --git a/packages/aws-lambda/src/handler.test.ts b/packages/aws-lambda/src/handler.test.ts index 3c4a4e6565..048c43c7c6 100644 --- a/packages/aws-lambda/src/handler.test.ts +++ b/packages/aws-lambda/src/handler.test.ts @@ -249,6 +249,8 @@ describe("handler dispatch", () => { "PLAN_TOO_LARGE", "PLAN_PROTOCOL_UNSUPPORTED", "PLAN_V2_INTEGRITY_UNRECOVERABLE", + "FONT_FETCH_FAILED", + "FONT_FETCH_UNAVAILABLE", "VIDEO_SOURCE_UNRENDERABLE", "VIDEO_EXTRACTION_FAILED", "INVALID_VIDEO_METADATA", diff --git a/packages/aws-lambda/src/handler.ts b/packages/aws-lambda/src/handler.ts index 8cb4c8c1bc..e224fde6e0 100644 --- a/packages/aws-lambda/src/handler.ts +++ b/packages/aws-lambda/src/handler.ts @@ -150,6 +150,8 @@ function normalizeTerminalErrorName(error: unknown): void { candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" || candidate.code === "PLAN_TOO_LARGE" || candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE" || + candidate.code === "FONT_FETCH_FAILED" || + candidate.code === "FONT_FETCH_UNAVAILABLE" || candidate.code === "VIDEO_SOURCE_UNRENDERABLE" || candidate.code === "VIDEO_EXTRACTION_FAILED" || candidate.code === "INVALID_VIDEO_METADATA" diff --git a/packages/gcp-cloud-run/src/server.test.ts b/packages/gcp-cloud-run/src/server.test.ts index 4a82ed077f..e8e3e05cc3 100644 --- a/packages/gcp-cloud-run/src/server.test.ts +++ b/packages/gcp-cloud-run/src/server.test.ts @@ -578,7 +578,9 @@ describe("createApp HTTP mapping", () => { it.each([ ["VIDEO_SOURCE_UNRENDERABLE", 400], ["VIDEO_EXTRACTION_FAILED", 500], - ] as const)("routes producer video code %s with HTTP %s", async (code, expectedStatus) => { + ["FONT_FETCH_FAILED", 400], + ["FONT_FETCH_UNAVAILABLE", 500], + ] as const)("routes producer code %s with HTTP %s", async (code, expectedStatus) => { const gcs = new FakeGcs(); await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH); const app = createApp( diff --git a/packages/gcp-cloud-run/src/server.ts b/packages/gcp-cloud-run/src/server.ts index 1b14a06855..674a25e0ee 100644 --- a/packages/gcp-cloud-run/src/server.ts +++ b/packages/gcp-cloud-run/src/server.ts @@ -183,6 +183,8 @@ function normalizeTerminalErrorName(error: unknown): void { candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" || candidate.code === "PLAN_TOO_LARGE" || candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE" || + candidate.code === "FONT_FETCH_FAILED" || + candidate.code === "FONT_FETCH_UNAVAILABLE" || candidate.code === "VIDEO_SOURCE_UNRENDERABLE" || candidate.code === "VIDEO_EXTRACTION_FAILED" || candidate.code === "INVALID_VIDEO_METADATA" diff --git a/packages/producer/src/index.ts b/packages/producer/src/index.ts index 6b14eee3eb..93e1329794 100644 --- a/packages/producer/src/index.ts +++ b/packages/producer/src/index.ts @@ -114,7 +114,13 @@ export { normalizeErrorMessage } from "./utils/errorMessage.js"; // timing. The render pipeline runs this in its compile stage; the CLI audit // paths (snapshot/check) reuse it so their captures match the render. export { + FONT_FETCH_FAILED, + FONT_FETCH_UNAVAILABLE, + FontFetchError, + FontFetchUnavailableError, injectDeterministicFontFaces, + type FontFetchErrorCode, + type FontFetchRetryPolicy, type InjectDeterministicFontFacesOptions, } from "./services/deterministicFonts.js"; export { quantizeTimeToFrame } from "./utils/parityContract.js"; diff --git a/packages/producer/src/services/deterministicFonts-failClosed.test.ts b/packages/producer/src/services/deterministicFonts-failClosed.test.ts index f2c7cb4c94..5a1e3c1388 100644 --- a/packages/producer/src/services/deterministicFonts-failClosed.test.ts +++ b/packages/producer/src/services/deterministicFonts-failClosed.test.ts @@ -17,7 +17,9 @@ import { describe, expect, it } from "bun:test"; import { FONT_FETCH_FAILED, + FONT_FETCH_UNAVAILABLE, FontFetchError, + FontFetchUnavailableError, injectDeterministicFontFaces, } from "./deterministicFonts.js"; @@ -75,6 +77,15 @@ function makeGoogleFontFetch(cssRequests: string[]): typeof fetch { }) as unknown as typeof fetch; } +async function rejectedError(promise: Promise): Promise { + try { + await promise; + } catch (error) { + return error; + } + throw new Error("Expected promise to reject"); +} + describe("injectDeterministicFontFaces — failClosedFontFetch: false (default)", () => { it("swallows a network failure and returns the original HTML (no throw)", async () => { const result = await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { @@ -141,19 +152,18 @@ describe("injectDeterministicFontFaces — failClosedFontFetch: true", () => { }); } - it("throws FontFetchError on a network failure", async () => { - let caught: unknown; - try { - await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { + it("throws FontFetchUnavailableError after retrying a network failure", async () => { + const caught = await rejectedError( + injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { failClosedFontFetch: true, fetchImpl: makeFailingFetch(), - }); - } catch (err) { - caught = err; - } + fontFetchRetryPolicy: { baseDelayMs: 0 }, + }), + ); expect(caught).toBeInstanceOf(FontFetchError); - expect((caught as FontFetchError).code).toBe(FONT_FETCH_FAILED); - expect((caught as FontFetchError).code).toBe("FONT_FETCH_FAILED"); + expect(caught).toBeInstanceOf(FontFetchUnavailableError); + expect((caught as FontFetchError).code).toBe(FONT_FETCH_UNAVAILABLE); + expect((caught as FontFetchError).code).toBe("FONT_FETCH_UNAVAILABLE"); expect((caught as FontFetchError).familyName).toBe("NotARealFontFamilyForTest"); expect((caught as Error).message).toContain("simulated network failure"); }); @@ -164,62 +174,53 @@ describe("injectDeterministicFontFaces — failClosedFontFetch: true", () => { // unresolvable (no alias, no Google Fonts, no system capture) which IS // a fail-closed error — the render would use a fallback font, producing // non-deterministic output across machines. - let caught: unknown; - try { - await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { + const caught = await rejectedError( + injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { failClosedFontFetch: true, allowSystemFontCapture: false, fetchImpl: makeHttp400Fetch(), - }); - } catch (err) { - caught = err; - } + }), + ); expect(caught).toBeInstanceOf(FontFetchError); expect((caught as FontFetchError).code).toBe(FONT_FETCH_FAILED); expect((caught as FontFetchError).familyName).toBe("NotARealFontFamilyForTest"); }); it("throws on a 404 response when font is completely unresolvable", async () => { - let caught: unknown; - try { - await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { + const caught = await rejectedError( + injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { failClosedFontFetch: true, allowSystemFontCapture: false, fetchImpl: makeHttp404Fetch(), - }); - } catch (err) { - caught = err; - } + }), + ); expect(caught).toBeInstanceOf(FontFetchError); expect((caught as FontFetchError).code).toBe(FONT_FETCH_FAILED); }); - it("throws FontFetchError on a 5xx response — non-deterministic, could differ on retry", async () => { - let caught: unknown; - try { - await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { + it("throws FontFetchUnavailableError on an exhausted 5xx response", async () => { + const caught = await rejectedError( + injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { failClosedFontFetch: true, fetchImpl: makeHttp503Fetch(), - }); - } catch (err) { - caught = err; - } + fontFetchRetryPolicy: { baseDelayMs: 0 }, + }), + ); expect(caught).toBeInstanceOf(FontFetchError); - expect((caught as FontFetchError).code).toBe(FONT_FETCH_FAILED); + expect(caught).toBeInstanceOf(FontFetchUnavailableError); + expect((caught as FontFetchError).code).toBe(FONT_FETCH_UNAVAILABLE); expect((caught as Error).message).toContain("HTTP 503"); expect((caught as Error).message).toContain("NotARealFontFamilyForTest"); }); it("includes the requested URL in 5xx errors", async () => { - let caught: unknown; - try { - await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { + const caught = await rejectedError( + injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { failClosedFontFetch: true, fetchImpl: makeHttp503Fetch(), - }); - } catch (err) { - caught = err; - } + fontFetchRetryPolicy: { baseDelayMs: 0 }, + }), + ); expect((caught as FontFetchError).url).toContain("fonts.googleapis.com"); expect((caught as FontFetchError).url).toContain("NotARealFontFamilyForTest"); }); diff --git a/packages/producer/src/services/deterministicFonts-retries.test.ts b/packages/producer/src/services/deterministicFonts-retries.test.ts new file mode 100644 index 0000000000..5dd7020bea --- /dev/null +++ b/packages/producer/src/services/deterministicFonts-retries.test.ts @@ -0,0 +1,417 @@ +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + FONT_FETCH_FAILED, + FONT_FETCH_UNAVAILABLE, + FontFetchError, + FontFetchUnavailableError, + injectDeterministicFontFaces, + type FontFetchRetryPolicy, +} from "./deterministicFonts.js"; + +let previousCacheDir: string | undefined; +let testCacheDir: string; + +beforeAll(() => { + previousCacheDir = process.env.HYPERFRAMES_FONT_CACHE_DIR; + testCacheDir = mkdtempSync(join(tmpdir(), "hf-font-retries-")); + process.env.HYPERFRAMES_FONT_CACHE_DIR = testCacheDir; +}); + +afterAll(() => { + if (previousCacheDir === undefined) delete process.env.HYPERFRAMES_FONT_CACHE_DIR; + else process.env.HYPERFRAMES_FONT_CACHE_DIR = previousCacheDir; + rmSync(testCacheDir, { recursive: true, force: true }); +}); + +const FAST_RETRY = { + maxAttempts: 2, + attemptTimeoutMs: 20, + maxElapsedMs: 100, + baseDelayMs: 0, +} as const; + +let familySequence = 0; + +function unresolvedHtml(): { family: string; html: string } { + familySequence += 1; + const family = `RetryTestFont${process.pid}${familySequence}`; + return { + family, + html: `retry`, + }; +} + +function googleCss(woff2Url: string): Response { + return new Response( + `@font-face { + font-style: normal; + font-weight: 400; + src: url(${woff2Url}) format('woff2'); + }`, + { status: 200 }, + ); +} + +function requestUrl(input: string | URL | Request): string { + return input instanceof Request ? input.url : String(input); +} + +function injectWithRetries( + html: string, + fetchImpl: typeof fetch, + fontFetchRetryPolicy: Partial = FAST_RETRY, +): Promise { + return injectDeterministicFontFaces(html, { + failClosedFontFetch: true, + allowSystemFontCapture: false, + fetchImpl, + fontFetchRetryPolicy, + }); +} + +describe("deterministic Google Fonts retries", () => { + for (const status of [408, 425, 429, 500, 502, 503]) { + it(`retries HTTP ${status} exactly once, then reports temporary unavailability`, async () => { + const { html } = unresolvedHtml(); + let calls = 0; + const fetchImpl = (async () => { + calls += 1; + return new Response("", { + status, + headers: status === 429 ? { "Retry-After": "0" } : undefined, + }); + }) as typeof fetch; + + const result = injectWithRetries(html, fetchImpl); + + await expect(result).rejects.toMatchObject({ + code: FONT_FETCH_UNAVAILABLE, + name: "FontFetchUnavailableError", + }); + expect(calls).toBe(2); + }); + } + + it("recovers when the Google Fonts CSS retry succeeds", async () => { + const { html } = unresolvedHtml(); + const woff2Url = `https://fonts.gstatic.com/s/retry/${process.pid}-css.woff2`; + let cssCalls = 0; + let woff2Calls = 0; + const fetchImpl = (async (input: string | URL | Request) => { + if (requestUrl(input).startsWith("https://fonts.googleapis.com/")) { + cssCalls += 1; + return cssCalls === 1 ? new Response("", { status: 502 }) : googleCss(woff2Url); + } + woff2Calls += 1; + return new Response(new Uint8Array([1, 2, 3]), { status: 200 }); + }) as typeof fetch; + + const result = await injectWithRetries(html, fetchImpl); + + expect(result).toContain("data-hyperframes-deterministic-fonts"); + expect(cssCalls).toBe(2); + expect(woff2Calls).toBe(1); + }); + + it("does not wait for a stalled retryable response-body cancellation", async () => { + const { html } = unresolvedHtml(); + const woff2Url = `https://fonts.gstatic.com/s/retry/${process.pid}-cancel.woff2`; + let cssCalls = 0; + let cancelledBodies = 0; + const fetchImpl = (async (input: string | URL | Request) => { + if (!requestUrl(input).startsWith("https://fonts.googleapis.com/")) { + return new Response(new Uint8Array([10, 11, 12]), { status: 200 }); + } + cssCalls += 1; + if (cssCalls === 1) { + return new Response( + new ReadableStream({ + cancel() { + cancelledBodies += 1; + return new Promise(() => {}); + }, + }), + { status: 502 }, + ); + } + return googleCss(woff2Url); + }) as typeof fetch; + + const result = await injectWithRetries(html, fetchImpl); + + expect(result).toContain("data-hyperframes-deterministic-fonts"); + expect(cssCalls).toBe(2); + expect(cancelledBodies).toBe(1); + }); + + it("recovers when a woff2 retry succeeds", async () => { + const { html } = unresolvedHtml(); + const woff2Url = `https://fonts.gstatic.com/s/retry/${process.pid}-woff2.woff2`; + let cssCalls = 0; + let woff2Calls = 0; + const fetchImpl = (async (input: string | URL | Request) => { + if (requestUrl(input).startsWith("https://fonts.googleapis.com/")) { + cssCalls += 1; + return googleCss(woff2Url); + } + woff2Calls += 1; + return woff2Calls === 1 + ? new Response("", { status: 503 }) + : new Response(new Uint8Array([4, 5, 6]), { status: 200 }); + }) as typeof fetch; + + const result = await injectWithRetries(html, fetchImpl); + + expect(result).toContain("data-hyperframes-deterministic-fonts"); + expect(cssCalls).toBe(1); + expect(woff2Calls).toBe(2); + }); + + it("reports temporary unavailability when woff2 retries are exhausted", async () => { + const { html } = unresolvedHtml(); + const woff2Url = `https://fonts.gstatic.com/s/retry/${process.pid}-woff2-exhausted.woff2`; + let cssCalls = 0; + let woff2Calls = 0; + const fetchImpl = (async (input: string | URL | Request) => { + if (requestUrl(input).startsWith("https://fonts.googleapis.com/")) { + cssCalls += 1; + return googleCss(woff2Url); + } + woff2Calls += 1; + return new Response("", { status: 502 }); + }) as typeof fetch; + + const result = injectWithRetries(html, fetchImpl); + + await expect(result).rejects.toMatchObject({ + code: FONT_FETCH_UNAVAILABLE, + url: woff2Url, + }); + expect(cssCalls).toBe(1); + expect(woff2Calls).toBe(2); + }); + + it("retries transport exceptions and preserves the final cause", async () => { + const { html } = unresolvedHtml(); + let calls = 0; + const fetchImpl = (async () => { + calls += 1; + throw new TypeError(`network failure ${calls}`); + }) as typeof fetch; + + let caught: unknown; + try { + await injectWithRetries(html, fetchImpl); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(FontFetchUnavailableError); + expect(caught).toBeInstanceOf(FontFetchError); + expect((caught as FontFetchUnavailableError).code).toBe(FONT_FETCH_UNAVAILABLE); + expect((caught as Error).message).toContain("network failure 2"); + expect(calls).toBe(2); + }); + + it("retries a response-body transport failure", async () => { + const { html } = unresolvedHtml(); + const woff2Url = `https://fonts.gstatic.com/s/retry/${process.pid}-body.woff2`; + let cssCalls = 0; + const fetchImpl = (async (input: string | URL | Request) => { + if (!requestUrl(input).startsWith("https://fonts.googleapis.com/")) { + return new Response(new Uint8Array([7, 8, 9]), { status: 200 }); + } + cssCalls += 1; + if (cssCalls === 1) { + return new Response( + new ReadableStream({ + start(controller) { + controller.error(new TypeError("CSS body connection reset")); + }, + }), + { status: 200 }, + ); + } + return googleCss(woff2Url); + }) as typeof fetch; + + const result = await injectWithRetries(html, fetchImpl); + + expect(result).toContain("data-hyperframes-deterministic-fonts"); + expect(cssCalls).toBe(2); + }); + + it("times out each hung attempt and reports temporary unavailability", async () => { + const { html } = unresolvedHtml(); + let calls = 0; + const fetchImpl = ((_input: string | URL | Request, init?: RequestInit) => { + calls += 1; + return new Promise((_resolve, reject) => { + const signal = init?.signal; + if (!signal) { + reject(new Error("missing timeout signal")); + return; + } + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + }); + }) as typeof fetch; + + const result = injectWithRetries(html, fetchImpl, { + ...FAST_RETRY, + attemptTimeoutMs: 5, + maxElapsedMs: 50, + }); + + await expect(result).rejects.toMatchObject({ code: FONT_FETCH_UNAVAILABLE }); + expect(calls).toBe(2); + }); + + it("does not exceed the shared budget to honor a long Retry-After", async () => { + const { html } = unresolvedHtml(); + let calls = 0; + const fetchImpl = (async () => { + calls += 1; + return new Response("", { + status: 429, + headers: { "Retry-After": "1" }, + }); + }) as typeof fetch; + + const result = injectWithRetries(html, fetchImpl, { + ...FAST_RETRY, + maxElapsedMs: 100, + }); + + await expect(result).rejects.toMatchObject({ code: FONT_FETCH_UNAVAILABLE }); + expect(calls).toBe(1); + }); + + it("shares one elapsed-time budget across multiple font resources", async () => { + const { html } = unresolvedHtml(); + const firstUrl = `https://fonts.gstatic.com/s/retry/${process.pid}-budget-first.woff2`; + const secondUrl = `https://fonts.gstatic.com/s/retry/${process.pid}-budget-second.woff2`; + let firstCalls = 0; + let secondCalls = 0; + const fetchImpl = ((input: string | URL | Request, init?: RequestInit): Promise => { + const url = requestUrl(input); + if (url.startsWith("https://fonts.googleapis.com/")) { + return Promise.resolve( + new Response( + `@font-face { + font-style: normal; + font-weight: 400; + src: url(${firstUrl}) format('woff2'); + } + @font-face { + font-style: normal; + font-weight: 700; + src: url(${secondUrl}) format('woff2'); + }`, + { status: 200 }, + ), + ); + } + if (url === firstUrl) { + firstCalls += 1; + return new Promise((resolve, reject) => { + const timeout = setTimeout( + () => resolve(new Response(new Uint8Array([13, 14, 15]), { status: 200 })), + 30, + ); + init?.signal?.addEventListener( + "abort", + () => { + clearTimeout(timeout); + reject(init.signal?.reason); + }, + { once: true }, + ); + }); + } + secondCalls += 1; + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(init.signal?.reason), { + once: true, + }); + }); + }) as typeof fetch; + + const result = injectWithRetries(html, fetchImpl, { + maxAttempts: 2, + attemptTimeoutMs: 70, + maxElapsedMs: 100, + baseDelayMs: 0, + }); + + await expect(result).rejects.toMatchObject({ code: FONT_FETCH_UNAVAILABLE }); + expect(firstCalls).toBe(1); + expect(secondCalls).toBe(1); + }); + + it("does not retry an ordinary 4xx and retains deterministic failure classification", async () => { + const { html } = unresolvedHtml(); + let calls = 0; + const fetchImpl = (async () => { + calls += 1; + return new Response("", { status: 400 }); + }) as typeof fetch; + + const result = injectWithRetries(html, fetchImpl); + + await expect(result).rejects.toMatchObject({ + code: FONT_FETCH_FAILED, + name: "FontFetchError", + }); + expect(calls).toBe(1); + }); + + it("propagates caller cancellation without wrapping or retrying it", async () => { + const { html } = unresolvedHtml(); + const controller = new AbortController(); + const cancellation = new Error("render cancelled by caller"); + let calls = 0; + const fetchImpl = ((_input: string | URL | Request, init?: RequestInit) => { + calls += 1; + return new Promise((_resolve, reject) => { + const signal = init?.signal; + signal?.addEventListener("abort", () => reject(signal.reason), { once: true }); + }); + }) as typeof fetch; + + const result = injectDeterministicFontFaces(html, { + failClosedFontFetch: true, + allowSystemFontCapture: false, + fetchImpl, + abortSignal: controller.signal, + fontFetchRetryPolicy: FAST_RETRY, + }); + controller.abort(cancellation); + + await expect(result).rejects.toBe(cancellation); + expect(calls).toBe(1); + }); + + it("keeps fail-open mode at one attempt with no typed failure", async () => { + const { html } = unresolvedHtml(); + let calls = 0; + const fetchImpl = (async () => { + calls += 1; + return new Response("", { status: 503 }); + }) as typeof fetch; + + const result = await injectDeterministicFontFaces(html, { + failClosedFontFetch: false, + allowSystemFontCapture: false, + fetchImpl, + fontFetchRetryPolicy: FAST_RETRY, + }); + + expect(result).not.toContain("data-hyperframes-deterministic-fonts"); + expect(calls).toBe(1); + }); +}); diff --git a/packages/producer/src/services/deterministicFonts.ts b/packages/producer/src/services/deterministicFonts.ts index df4664d4a1..85e3e27d8a 100644 --- a/packages/producer/src/services/deterministicFonts.ts +++ b/packages/producer/src/services/deterministicFonts.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import { defaultLogger } from "../logger.js"; @@ -581,19 +581,22 @@ function warnUnresolvedFonts(unresolved: string[]): void { // Google Fonts on-demand fetch + local cache // --------------------------------------------------------------------------- -// On AWS Lambda `$HOME` resolves to a `/home/sbx_*` tree that's -// read-only; only `/tmp` is writable. Route the cache there when -// running inside Lambda, and honor `HYPERFRAMES_FONT_CACHE_DIR` as -// an explicit override for any environment. +let lambdaFontCacheRoot: string | undefined; + +// On AWS Lambda `$HOME` resolves to a `/home/sbx_*` tree that's read-only; +// only `/tmp` is writable. Create one private, unguessable cache directory per +// warm process and reuse it across invocations. Honor HYPERFRAMES_FONT_CACHE_DIR +// as an explicit override for any environment. function resolveFontCacheRoot(): string { - return ( - process.env.HYPERFRAMES_FONT_CACHE_DIR ?? - (process.env.AWS_LAMBDA_FUNCTION_NAME - ? join(tmpdir(), "hyperframes", "fonts") - : join(homedir(), ".cache", "hyperframes", "fonts")) - ); + if (process.env.HYPERFRAMES_FONT_CACHE_DIR) { + return process.env.HYPERFRAMES_FONT_CACHE_DIR; + } + if (process.env.AWS_LAMBDA_FUNCTION_NAME) { + lambdaFontCacheRoot ??= mkdtempSync(join(tmpdir(), "hyperframes-fonts-")); + return lambdaFontCacheRoot; + } + return join(homedir(), ".cache", "hyperframes", "fonts"); } -const GOOGLE_FONTS_CACHE_DIR = resolveFontCacheRoot(); // Chrome UA triggers woff2 responses from Google Fonts CSS API const WOFF2_USER_AGENT = @@ -607,7 +610,7 @@ function fontSlug(familyName: string): string { } function fontCacheDir(slug: string): string { - const dir = join(GOOGLE_FONTS_CACHE_DIR, slug); + const dir = join(resolveFontCacheRoot(), slug); if (!existsSync(dir)) { mkdirSync(dir, { recursive: true }); } @@ -637,37 +640,77 @@ type GoogleFontFace = { }; /** - * Typed code classifying a font-fetch failure as non-retryable for - * distributed workflow adapters — a missing Google Fonts entry will not heal - * on retry. + * Typed codes let distributed workflow adapters distinguish deterministic + * resolution failures from temporary upstream unavailability. */ export const FONT_FETCH_FAILED = "FONT_FETCH_FAILED"; +export const FONT_FETCH_UNAVAILABLE = "FONT_FETCH_UNAVAILABLE"; +export type FontFetchErrorCode = typeof FONT_FETCH_FAILED | typeof FONT_FETCH_UNAVAILABLE; /** * Typed error thrown by {@link injectDeterministicFontFaces} when - * `failClosedFontFetch === true` and an external font fetch fails. The - * default (swallow + warn) preserves the in-process behavior. + * `failClosedFontFetch === true` and deterministic font resolution fails. + * The default (swallow + warn) preserves the in-process behavior. */ export class FontFetchError extends Error { - readonly code: typeof FONT_FETCH_FAILED = FONT_FETCH_FAILED; + readonly code: FontFetchErrorCode; readonly familyName: string; readonly url: string; readonly cause?: unknown; - constructor(familyName: string, url: string, message: string, cause?: unknown) { + constructor( + familyName: string, + url: string, + message: string, + cause?: unknown, + code: FontFetchErrorCode = FONT_FETCH_FAILED, + ) { super(message); this.name = "FontFetchError"; + this.code = code; this.familyName = familyName; this.url = url; this.cause = cause; } } +/** + * Retryable font-fetch failure. Distributed adapters map this code to an + * unavailable response so the workflow can retry the plan activity. + */ +export class FontFetchUnavailableError extends FontFetchError { + constructor(familyName: string, url: string, message: string, cause?: unknown) { + super(familyName, url, message, cause, FONT_FETCH_UNAVAILABLE); + this.name = "FontFetchUnavailableError"; + } +} + +export interface FontFetchRetryPolicy { + /** Total fetch attempts for one CSS or woff2 URL. Default: 2. */ + maxAttempts: number; + /** Timeout for each individual fetch attempt. Default: 8 seconds. */ + attemptTimeoutMs: number; + /** Shared wall-clock budget for all Google Fonts requests in one compile. Default: 20 seconds. */ + maxElapsedMs: number; + /** Initial full-jitter backoff ceiling. Default: 250 ms. */ + baseDelayMs: number; +} + +const DEFAULT_FONT_FETCH_RETRY_POLICY: FontFetchRetryPolicy = { + maxAttempts: 2, + attemptTimeoutMs: 8_000, + maxElapsedMs: 20_000, + baseDelayMs: 250, +}; + /** Internal threading of the failClosed flag + fetch override through callers. */ interface InternalFontFetchOptions { failClosedFontFetch: boolean; fetchImpl: typeof fetch; allowSystemFontCapture: boolean; + abortSignal?: AbortSignal; + retryPolicy: FontFetchRetryPolicy; + retryDeadlineMs: number; } /** @@ -680,16 +723,158 @@ function fontFetchError( url: string, what: "Google Fonts CSS" | `Google Fonts woff2 (${string}/${string})`, cause: { status: number } | { error: unknown }, + unavailable = false, ): FontFetchError { const reason = "status" in cause ? `returned HTTP ${cause.status}` - : `failed: ${(cause.error as Error).message}`; + : `failed: ${cause.error instanceof Error ? cause.error.message : String(cause.error)}`; const message = `[deterministicFonts] ${what} fetch for ${JSON.stringify(familyName)} ${reason}. ` + `Distributed renders require deterministic fonts; system-font fallback would produce ` + `non-byte-identical output.`; - return new FontFetchError(familyName, url, message, "error" in cause ? cause.error : undefined); + const errorCause = "error" in cause ? cause.error : undefined; + return unavailable + ? new FontFetchUnavailableError(familyName, url, message, errorCause) + : new FontFetchError(familyName, url, message, errorCause); +} + +function isRetryableFontFetchStatus(status: number): boolean { + return status === 408 || status === 425 || status === 429 || status >= 500; +} + +function retryAfterMs(response: Response): number | null { + const value = response.headers.get("retry-after"); + if (value === null) return null; + const seconds = Number(value); + if (Number.isFinite(seconds)) return Math.max(0, seconds * 1_000); + const date = Date.parse(value); + return Number.isNaN(date) ? null : Math.max(0, date - Date.now()); +} + +function callerAbortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException("Font fetch cancelled", "AbortError"); +} + +function throwIfCallerAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw callerAbortReason(signal); +} + +async function waitForFontFetchRetry( + delayMs: number, + signal: AbortSignal | undefined, +): Promise { + throwIfCallerAborted(signal); + if (delayMs <= 0) return; + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, delayMs); + const onAbort = (): void => { + clearTimeout(timeout); + reject(signal ? callerAbortReason(signal) : new DOMException("Cancelled", "AbortError")); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); + }); +} + +function retryDelayMs( + response: Response | undefined, + attempt: number, + baseDelayMs: number, +): number { + const requestedDelay = response ? retryAfterMs(response) : null; + if (requestedDelay !== null) return requestedDelay; + const ceiling = baseDelayMs * 2 ** attempt; + return Math.floor(Math.random() * (ceiling + 1)); +} + +function cancelResponseBody(response: Response): void { + try { + void response.body?.cancel().catch(() => { + // Best-effort connection cleanup must not replace the original status. + }); + } catch { + // Best-effort connection cleanup must not replace the original status. + } +} + +type FontFetchResult = + | { ok: true; response: Response; body: T } + | { ok: false; response: Response }; + +type FontFetchAttemptResult = + | { completed: true; result: FontFetchResult } + | { + completed: false; + response?: Response; + cause: { status: number } | { error: unknown }; + }; + +async function runFontFetchAttempt( + url: string, + init: RequestInit | undefined, + readBody: (response: Response) => Promise, + options: InternalFontFetchOptions, + remainingMs: number, +): Promise> { + const timeoutSignal = AbortSignal.timeout( + Math.max(1, Math.min(options.retryPolicy.attemptTimeoutMs, remainingMs)), + ); + const signal = options.abortSignal + ? AbortSignal.any([options.abortSignal, timeoutSignal]) + : timeoutSignal; + let response: Response | undefined; + try { + response = await options.fetchImpl(url, { ...init, signal }); + if (!isRetryableFontFetchStatus(response.status)) { + if (!response.ok) return { completed: true, result: { ok: false, response } }; + const body = await readBody(response); + return { completed: true, result: { ok: true, response, body } }; + } + cancelResponseBody(response); + return { completed: false, response, cause: { status: response.status } }; + } catch (error) { + throwIfCallerAborted(options.abortSignal); + return { completed: false, response, cause: { error } }; + } +} + +async function fetchFontResource( + url: string, + init: RequestInit | undefined, + readBody: (response: Response) => Promise, + familyName: string, + what: "Google Fonts CSS" | `Google Fonts woff2 (${string}/${string})`, + options: InternalFontFetchOptions, +): Promise> { + if (!options.failClosedFontFetch) { + const response = await options.fetchImpl(url, { ...init, signal: options.abortSignal }); + if (!response.ok) return { ok: false, response }; + return { ok: true, response, body: await readBody(response) }; + } + + let lastCause: { status: number } | { error: unknown } = { + error: new DOMException("Font fetch budget exhausted", "TimeoutError"), + }; + for (let attempt = 0; attempt < options.retryPolicy.maxAttempts; attempt += 1) { + throwIfCallerAborted(options.abortSignal); + const remainingMs = options.retryDeadlineMs - Date.now(); + if (remainingMs <= 0) break; + + const attemptResult = await runFontFetchAttempt(url, init, readBody, options, remainingMs); + if (attemptResult.completed) return attemptResult.result; + lastCause = attemptResult.cause; + + if (attempt + 1 >= options.retryPolicy.maxAttempts) break; + const delayMs = retryDelayMs(attemptResult.response, attempt, options.retryPolicy.baseDelayMs); + if (delayMs >= options.retryDeadlineMs - Date.now()) break; + await waitForFontFetchRetry(delayMs, options.abortSignal); + } + + throw fontFetchError(familyName, url, what, lastCause, true); } /** @@ -714,17 +899,20 @@ async function ensureWoff2DataUri( const woff2What = `Google Fonts woff2 (${weight}/${style})` as const; try { - const fontRes = await options.fetchImpl(woff2Url); - if (!fontRes.ok) { - if (fontRes.status >= 500 && options.failClosedFontFetch) { - throw fontFetchError(familyName, woff2Url, woff2What, { status: fontRes.status }); - } - return null; - } + const fontResult = await fetchFontResource( + woff2Url, + undefined, + (response) => response.arrayBuffer(), + familyName, + woff2What, + options, + ); + if (!fontResult.ok) return null; // wx = O_CREAT|O_EXCL: atomic create, rejects symlinks, fails with // EEXIST if a concurrent call cached it between our read and write. - writeFileSync(cachePath, Buffer.from(await fontRes.arrayBuffer()), { flag: "wx", mode: 0o644 }); + writeFileSync(cachePath, Buffer.from(fontResult.body), { flag: "wx", mode: 0o644 }); } catch (err) { + throwIfCallerAborted(options.abortSignal); if (err instanceof FontFetchError) throw err; if ((err as NodeJS.ErrnoException).code === "EEXIST") { // Concurrent call wrote it — read their result below. @@ -755,10 +943,15 @@ async function fetchGoogleFont( let cssText: string; try { - const res = await options.fetchImpl(url, { - headers: { "User-Agent": WOFF2_USER_AGENT }, - }); - if (!res.ok) { + const cssResult = await fetchFontResource( + url, + { headers: { "User-Agent": WOFF2_USER_AGENT } }, + (response) => response.text(), + familyName, + "Google Fonts CSS", + options, + ); + if (!cssResult.ok) { // 4xx is a *deterministic* answer from Google Fonts that this // family is not served (e.g. HTTP 400 for "Segoe UI", "Arial", // "Futura" — names absent from Google's catalog) or is misnamed. @@ -767,16 +960,14 @@ async function fetchGoogleFont( // transient upstream failures) could return faces on retry, which // would break the byte-identical-retry contract distributed // renders rely on — those still fail closed when requested. - if (res.status >= 500 && options.failClosedFontFetch) { - throw fontFetchError(familyName, url, "Google Fonts CSS", { status: res.status }); - } return []; } - cssText = await res.text(); + cssText = cssResult.body; } catch (err) { // Rethrow typed error untouched. Network / DNS / fetch-throws are // non-deterministic infrastructure failures — wrapped when failClosed // is on, swallowed otherwise. + throwIfCallerAborted(options.abortSignal); if (err instanceof FontFetchError) throw err; if (options.failClosedFontFetch) { throw fontFetchError(familyName, url, "Google Fonts CSS", { error: err }); @@ -829,16 +1020,16 @@ async function fetchGoogleFont( */ export interface InjectDeterministicFontFacesOptions { /** - * When `true`, any external font fetch failure (Google Fonts CSS or - * woff2) throws {@link FontFetchError} with code `FONT_FETCH_FAILED`. + * When `true`, exhausted transient fetch failures throw + * {@link FontFetchUnavailableError} with code `FONT_FETCH_UNAVAILABLE`; + * deterministic resolution failures retain `FONT_FETCH_FAILED`. * * Default `false`: failed fetches are silently swallowed; the composition * falls back to system fonts via `warnUnresolvedFonts`. This preserves the * in-process behavior. * * Distributed callers pass `true` so font availability is part of the - * planDir's content-addressed hash and fetch failures surface as typed - * non-retryable errors. + * planDir's content-addressed hash and failures surface as typed errors. */ failClosedFontFetch?: boolean; /** @@ -847,6 +1038,13 @@ export interface InjectDeterministicFontFacesOptions { * network. */ fetchImpl?: typeof fetch; + /** Caller cancellation propagated through fetch attempts and retry waits. */ + abortSignal?: AbortSignal; + /** + * Optional retry tuning. Defaults are deliberately bounded for composition + * planning; tests may lower delays and timeouts without replacing timers. + */ + fontFetchRetryPolicy?: Partial; /** * When `true` (default for local renders), fonts that aren't resolved by * the bundled alias map or Google Fonts are located on the local filesystem, @@ -873,6 +1071,29 @@ function extractGoogleFontsText(html: string): string | undefined { : undefined; } +function resolveFontFetchRetryPolicy( + configured: Partial | undefined, +): FontFetchRetryPolicy { + return { + maxAttempts: Math.max( + 1, + Math.floor(configured?.maxAttempts ?? DEFAULT_FONT_FETCH_RETRY_POLICY.maxAttempts), + ), + attemptTimeoutMs: Math.max( + 1, + configured?.attemptTimeoutMs ?? DEFAULT_FONT_FETCH_RETRY_POLICY.attemptTimeoutMs, + ), + maxElapsedMs: Math.max( + 1, + configured?.maxElapsedMs ?? DEFAULT_FONT_FETCH_RETRY_POLICY.maxElapsedMs, + ), + baseDelayMs: Math.max( + 0, + configured?.baseDelayMs ?? DEFAULT_FONT_FETCH_RETRY_POLICY.baseDelayMs, + ), + }; +} + export async function injectDeterministicFontFaces( html: string, options: InjectDeterministicFontFacesOptions = {}, @@ -880,10 +1101,14 @@ export async function injectDeterministicFontFaces( const failClosedFontFetch = options.failClosedFontFetch === true; const fetchImpl = options.fetchImpl ?? fetch; const allowSystemFontCapture = options.allowSystemFontCapture !== false; + const retryPolicy = resolveFontFetchRetryPolicy(options.fontFetchRetryPolicy); const fetchOptions: InternalFontFetchOptions = { failClosedFontFetch, fetchImpl, allowSystemFontCapture, + abortSignal: options.abortSignal, + retryPolicy, + retryDeadlineMs: Date.now() + retryPolicy.maxElapsedMs, }; const existingFaces = extractExistingFontFaces(html); diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts index 12e021930a..63121e2890 100644 --- a/packages/producer/src/services/distributed/plan.ts +++ b/packages/producer/src/services/distributed/plan.ts @@ -911,6 +911,7 @@ export async function plan( needsAlpha, log, assertNotAborted, + abortSignal, // Distributed renders fail closed on font-fetch errors so the planDir // is content-addressed against deterministic fonts only. failClosedFontFetch: config.failClosedFontFetch !== false, diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts index 436f1ab8e8..0894809e53 100644 --- a/packages/producer/src/services/htmlCompiler.ts +++ b/packages/producer/src/services/htmlCompiler.ts @@ -1714,11 +1714,11 @@ export interface CompileForRenderOptions { log?: ProducerLogger; /** * Threaded through to {@link injectDeterministicFontFaces}. When `true`, - * any external font fetch failure throws `FontFetchError` instead of - * silently falling back to system fonts. Distributed `plan()` sets this - * to `true` so font availability is part of the planDir's content-addressed - * hash and fetch failures surface as typed non-retryable errors. Default - * `false` preserves the in-process behavior. + * deterministic font resolution and exhausted transient fetch failures + * surface as typed errors instead of silently falling back to system fonts. + * Distributed `plan()` sets this to `true` so font availability is part of + * the planDir's content-addressed hash. Default `false` preserves the + * in-process behavior. */ failClosedFontFetch?: boolean; /** @@ -1728,6 +1728,8 @@ export interface CompileForRenderOptions { * prevent host-specific font capture from leaking into the planDir. */ allowSystemFontCapture?: boolean; + /** Caller cancellation propagated through compile-time font fetches. */ + abortSignal?: AbortSignal; /** * Optional persistent cache directory for prep-time animated GIF → WebM * transcodes. When omitted, the render's downloadDir is used. @@ -1839,6 +1841,7 @@ export async function compileForRender( const coalescedHtml = await injectDeterministicFontFaces(normalizedFontHtml, { failClosedFontFetch: options.failClosedFontFetch === true, allowSystemFontCapture: options.allowSystemFontCapture, + abortSignal: options.abortSignal, }); // Download CDN scripts and inline them AFTER coalescing. This order matters: diff --git a/packages/producer/src/services/render/stages/compileStage.test.ts b/packages/producer/src/services/render/stages/compileStage.test.ts index fa85565eb4..08cef195fd 100644 --- a/packages/producer/src/services/render/stages/compileStage.test.ts +++ b/packages/producer/src/services/render/stages/compileStage.test.ts @@ -112,6 +112,16 @@ const IFRAME_HTML = ` `; +const UNRESOLVED_FONT_HTML = ` + + + +
+

cancelled font compile

+
+ +`; + // Portrait/square/landscape fixture template — same shape as PLAIN_HTML but // with caller-supplied composition dimensions. Consumed by the aspect-agnostic // re-map tests below (and reachable from any sibling describe block, unlike a @@ -232,6 +242,40 @@ describe("runCompileStage — forceScreenshot snapshot", () => { }); }); +describe("runCompileStage — font fetch cancellation", () => { + let fixture: CompileFixture | null = null; + + afterEach(() => { + fixture?.cleanup(); + fixture = null; + }); + + it("propagates the caller abort signal through compileForRender to font fetching", async () => { + fixture = setupFixture(UNRESOLVED_FONT_HTML); + const projectDir = join(fixture.workDir, "project"); + const controller = new AbortController(); + const cancellation = new Error("cancel distributed compile"); + controller.abort(cancellation); + + const result = runCompileStage({ + projectDir, + workDir: fixture.workDir, + htmlPath: fixture.htmlPath, + entryFile: "index.html", + job: createJob(), + cfg: createCfg(), + needsAlpha: false, + log: noopLog, + assertNotAborted: () => {}, + abortSignal: controller.signal, + failClosedFontFetch: true, + allowSystemFontCapture: false, + }); + + await expect(result).rejects.toBe(cancellation); + }); +}); + /** * Tests for the aspect-agnostic --resolution re-map in `runCompileStage`. * diff --git a/packages/producer/src/services/render/stages/compileStage.ts b/packages/producer/src/services/render/stages/compileStage.ts index cf3f13995e..dac4834bc8 100644 --- a/packages/producer/src/services/render/stages/compileStage.ts +++ b/packages/producer/src/services/render/stages/compileStage.ts @@ -71,12 +71,14 @@ export interface CompileStageInput { log: ProducerLogger; /** Cooperative-cancellation probe; throws `RenderCancelledError` when aborted. */ assertNotAborted: () => void; + /** Caller cancellation propagated through compile-time network requests. */ + abortSignal?: AbortSignal; /** * When `true`, `compileForRender` threads through to - * `injectDeterministicFontFaces` and any external font fetch failure - * throws `FontFetchError` instead of silently falling back to system - * fonts. Distributed `plan()` passes `true`; the in-process renderer - * leaves it `undefined` to preserve current behavior. + * `injectDeterministicFontFaces` so deterministic resolution and exhausted + * transient fetch failures surface as typed errors instead of silently + * falling back to system fonts. Distributed `plan()` passes `true`; the + * in-process renderer leaves it `undefined` to preserve current behavior. */ failClosedFontFetch?: boolean; /** @@ -158,6 +160,7 @@ export async function runCompileStage(input: CompileStageInput): Promise