fix(build): retry transient fetch fails during build

scripts/data/sponsors.mjs

@@ -3,6 +3,7 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { fetchWithRetry } from '../utils/fetch.mjs';
 
 const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
 const OUTPUT = join(ROOT, 'generated', 'sponsors.json');
@@ -78,7 +79,7 @@ const fetchAllOrders = async () => {
   const all = [];
 
   while (offset < totalCount) {
-    const res = await fetch(API, {
+    const res = await fetchWithRetry(API, {
       method: 'POST',
       headers: { 'content-type': 'application/json' },
       body: JSON.stringify({

scripts/markdown/governance.mjs

@@ -1,5 +1,6 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
+import { fetchWithRetry } from '../utils/fetch.mjs';
 
 const { GH_TOKEN } = process.env;
 
@@ -57,7 +58,7 @@ await mkdir(outputDir, { recursive: true });
 const results = await Promise.all(
   Object.entries(FILE_MAP).map(async ([source, { output, label }]) => {
     const url = `https://raw.githubusercontent.com/webpack/governance/HEAD/${source}`;
-    const res = await fetch(url, { headers: BASE_HEADERS });
+    const res = await fetchWithRetry(url, { headers: BASE_HEADERS });
 
     if (!res.ok) {
       console.error(`Failed: ${source} -> ${res.status} ${res.statusText}`);

scripts/markdown/readmes.mjs

@@ -1,5 +1,6 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
+import { fetchWithRetry } from '../utils/fetch.mjs';
 
 const { GH_TOKEN } = process.env;
 
@@ -18,7 +19,7 @@ const discoverRepos = async () => {
     'https://api.github.com/orgs/webpack/repos?per_page=100&type=public';
 
   while (url) {
-    const res = await fetch(url, { headers: BASE_HEADERS });
+    const res = await fetchWithRetry(url, { headers: BASE_HEADERS });
 
     for (const repo of await res.json()) {
       if (repo.archived) continue;
@@ -50,7 +51,7 @@ const repoName = fullName => fullName.split('/')[1];
 
 const fetchReadme = async fullName => {
   const url = `https://raw.githubusercontent.com/${fullName}/HEAD/README.md`;
-  const res = await fetch(url);
+  const res = await fetchWithRetry(url);
   return res.text();
 };
 

scripts/utils/fetch.mjs

@@ -0,0 +1,42 @@
+// fetch()  wrapper that retries flakey responses like the  GitHub and Open Collective
+// occasionally 503 midbuild,so one blip doesn't fail the whole deploy.
+
+const RETRYABLE = new Set([429, 502, 503, 504]);
+
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+// Use the server's Retry After when it sends one, otherwise back off.
+const delayFor = (attempt, baseDelay, response) => {
+  const retryAfter = Number(response?.headers.get('retry-after'));
+  if (retryAfter > 0) return retryAfter * 1000;
+  return baseDelay * 2 ** attempt + Math.random() * baseDelay;
+};
+
+/**
+ * @param {string | URL} url
+ * @param {RequestInit} [options]
+ * @param {{ retries?: number, baseDelay?: number, timeout?: number }} [config]
+ * @returns {Promise<Response>}
+ */
+export const fetchWithRetry = async (
+  url,
+  options,
+  { retries = 3, baseDelay = 500, timeout = 15000 } = {}
+) => {
+  for (let attempt = 0; ; attempt++) {
+    let response;
+
+    try {
+      response = await fetch(url, {
+        ...options,
+        signal: AbortSignal.timeout(timeout),
+      });
+      if (response.ok || !RETRYABLE.has(response.status)) return response;
+    } catch (error) {
+      if (attempt === retries) throw error;
+    }
+
+    if (attempt === retries) return response;
+    await sleep(delayFor(attempt, baseDelay, response));
+  }
+};