From a067dc9494fb6c667aee3722076e9921583fa30b Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Rajak Date: Sun, 14 Jun 2026 05:41:11 +0000 Subject: [PATCH] fix(build): retry transient fetch fails during build --- scripts/data/sponsors.mjs | 3 ++- scripts/markdown/governance.mjs | 3 ++- scripts/markdown/readmes.mjs | 5 ++-- scripts/utils/fetch.mjs | 42 +++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 scripts/utils/fetch.mjs diff --git a/scripts/data/sponsors.mjs b/scripts/data/sponsors.mjs index 0210bee5..2c58bcbf 100644 --- a/scripts/data/sponsors.mjs +++ b/scripts/data/sponsors.mjs @@ -3,6 +3,7 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { fetchWithRetry } from '../utils/fetch.mjs'; const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); const OUTPUT = join(ROOT, 'generated', 'sponsors.json'); @@ -78,7 +79,7 @@ const fetchAllOrders = async () => { const all = []; while (offset < totalCount) { - const res = await fetch(API, { + const res = await fetchWithRetry(API, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ diff --git a/scripts/markdown/governance.mjs b/scripts/markdown/governance.mjs index 76644419..c2184239 100644 --- a/scripts/markdown/governance.mjs +++ b/scripts/markdown/governance.mjs @@ -1,5 +1,6 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; +import { fetchWithRetry } from '../utils/fetch.mjs'; const { GH_TOKEN } = process.env; @@ -57,7 +58,7 @@ await mkdir(outputDir, { recursive: true }); const results = await Promise.all( Object.entries(FILE_MAP).map(async ([source, { output, label }]) => { const url = `https://raw.githubusercontent.com/webpack/governance/HEAD/${source}`; - const res = await fetch(url, { headers: BASE_HEADERS }); + const res = await fetchWithRetry(url, { headers: BASE_HEADERS }); if (!res.ok) { console.error(`Failed: ${source} -> ${res.status} ${res.statusText}`); diff --git a/scripts/markdown/readmes.mjs b/scripts/markdown/readmes.mjs index 7be2b47c..f5144b1a 100644 --- a/scripts/markdown/readmes.mjs +++ b/scripts/markdown/readmes.mjs @@ -1,5 +1,6 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; +import { fetchWithRetry } from '../utils/fetch.mjs'; const { GH_TOKEN } = process.env; @@ -18,7 +19,7 @@ const discoverRepos = async () => { 'https://api.github.com/orgs/webpack/repos?per_page=100&type=public'; while (url) { - const res = await fetch(url, { headers: BASE_HEADERS }); + const res = await fetchWithRetry(url, { headers: BASE_HEADERS }); for (const repo of await res.json()) { if (repo.archived) continue; @@ -50,7 +51,7 @@ const repoName = fullName => fullName.split('/')[1]; const fetchReadme = async fullName => { const url = `https://raw.githubusercontent.com/${fullName}/HEAD/README.md`; - const res = await fetch(url); + const res = await fetchWithRetry(url); return res.text(); }; diff --git a/scripts/utils/fetch.mjs b/scripts/utils/fetch.mjs new file mode 100644 index 00000000..f18bd13c --- /dev/null +++ b/scripts/utils/fetch.mjs @@ -0,0 +1,42 @@ +// fetch() wrapper that retries flakey responses like the GitHub and Open Collective +// occasionally 503 midbuild,so one blip doesn't fail the whole deploy. + +const RETRYABLE = new Set([429, 502, 503, 504]); + +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); + +// Use the server's Retry After when it sends one, otherwise back off. +const delayFor = (attempt, baseDelay, response) => { + const retryAfter = Number(response?.headers.get('retry-after')); + if (retryAfter > 0) return retryAfter * 1000; + return baseDelay * 2 ** attempt + Math.random() * baseDelay; +}; + +/** + * @param {string | URL} url + * @param {RequestInit} [options] + * @param {{ retries?: number, baseDelay?: number, timeout?: number }} [config] + * @returns {Promise} + */ +export const fetchWithRetry = async ( + url, + options, + { retries = 3, baseDelay = 500, timeout = 15000 } = {} +) => { + for (let attempt = 0; ; attempt++) { + let response; + + try { + response = await fetch(url, { + ...options, + signal: AbortSignal.timeout(timeout), + }); + if (response.ok || !RETRYABLE.has(response.status)) return response; + } catch (error) { + if (attempt === retries) throw error; + } + + if (attempt === retries) return response; + await sleep(delayFor(attempt, baseDelay, response)); + } +};