diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index d2a402a..da3ad29 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -40,15 +40,22 @@ jobs: node-version: "22" cache: npm - run: npm ci - # `npm run build` invokes `npm run prebuild` first, which uses the - # built-in `gh` CLI to download `sizes.*.json` release assets from - # OpenIPC/firmware and OpenIPC/builder, server-side. CORS is structurally - # not a question during this step — only the in-browser fetches need to - # stay same-origin. + # `npm run build` invokes `npm run prebuild` first. Prebuild enumerates + # releases with ONE paginated `gh api /releases` call per source and + # fetches asset bytes via anonymous HTTPS on the browser_download_url — + # both to slash our footprint against the installation-wide API rate + # limit, which was blowing up consecutive nightly deploys in the busy + # 07:20–07:50 UTC cron window (runs 28646280479 / 28698970626 / + # 28733698722). See scripts/prebuild.mts. - run: npm run build env: GH_TOKEN: ${{ github.token }} - - uses: actions/configure-pages@v5 + # `actions/configure-pages@v5` was dropped: its only job was to fetch + # Pages site config via the API, and it was tripping on the same rate + # limit (runs 28698970626 / 28733698722). We don't consume any of its + # outputs (Vite `base` is hardcoded), and `deploy-pages@v4` works + # without it as long as Pages is already enabled on the repo — which + # it is, configured to build via GitHub Actions. - uses: actions/upload-pages-artifact@v3 with: path: dist diff --git a/scripts/prebuild.mts b/scripts/prebuild.mts index f3ced41..0a4f82a 100644 --- a/scripts/prebuild.mts +++ b/scripts/prebuild.mts @@ -66,9 +66,72 @@ export type GhReleaseDetail = { tagName: string; createdAt: string; body: string; - assets: Array<{ name: string; size: number }>; + assets: Array<{ name: string; size: number; downloadUrl?: string }>; }; +/** + * Raw shape of `GET /repos/:owner/:repo/releases` — we consume this and + * reshape into GhReleaseDetail. Named to match the API field casing so the + * translation site is obvious. + */ +type ReleaseApiRow = { + tag_name: string; + created_at: string; + prerelease: boolean; + body: string | null; + assets: Array<{ + name: string; + size: number; + browser_download_url: string; + }>; +}; + +/** + * Paginated `gh api /repos/:owner/:repo/releases`. Returns every release with + * its full metadata + asset list + download URLs in one place, replacing the + * old `gh release list` + per-tag `gh release view` pattern that cost ~91 + * API calls per source. + * + * `per_page=100` is the API's cap. We stop early on the first partial page + * (< 100 rows) since GitHub returns them in strict newest-first order. + */ +export function fetchReleases( + gh: GhFn, + repo: string, + maxPages = 5, +): { list: GhRelease[]; detail: Map } { + const list: GhRelease[] = []; + const detail = new Map(); + for (let page = 1; page <= maxPages; page++) { + const raw = gh([ + "api", + "-H", + "Accept: application/vnd.github+json", + `/repos/${repo}/releases?per_page=100&page=${page}`, + ]); + const rows = JSON.parse(raw) as ReleaseApiRow[]; + for (const row of rows) { + list.push({ + tagName: row.tag_name, + createdAt: row.created_at, + isPrerelease: row.prerelease, + }); + detail.set(row.tag_name, { + tagName: row.tag_name, + createdAt: row.created_at, + body: row.body ?? "", + assets: row.assets.map((a) => ({ + name: a.name, + size: a.size, + downloadUrl: a.browser_download_url, + })), + }); + } + if (rows.length < 100) break; + } + return { list, detail }; +} + export type Asset = { name: string; size: number }; export type BuildEntry = { @@ -113,6 +176,37 @@ export const defaultGh: GhFn = (args) => maxBuffer: 64 * 1024 * 1024, }); +/** + * Asset-byte fetcher, split off `GhFn` so the runtime can bypass the + * GitHub API entirely for asset downloads. `browser_download_url` on + * public releases is served anonymously by github.com (302 to a signed + * CDN URL) and consumes zero of the installation's API rate-limit + * bucket — the ~9000-call/day nightly cost is the root of the + * consecutive-cron failures (runs 28646280479 / 28698970626 / + * 28733698722). + * + * `curl --retry 5 --retry-delay 5 --retry-connrefused` handles genuine + * transient network failures (timeouts, DNS blips, CDN 5xx) natively + * so we don't need our own retry loop layered on top. + */ +export type HttpFn = (url: string, target: string) => void; +export const defaultHttp: HttpFn = (url, target) => + execFileSync( + "curl", + [ + "-fsSL", + "--retry", + "5", + "--retry-delay", + "5", + "--retry-all-errors", + "-o", + target, + url, + ], + { encoding: "utf-8", maxBuffer: 64 * 1024 * 1024 }, + ); + export const defaultFs: FsHooks = { mkdir: (p) => mkdirSync(p, { recursive: true }), write: (p, c) => writeFileSync(p, c), @@ -164,6 +258,7 @@ export type RunOpts = { outDir: string; cacheDir?: string; gh?: GhFn; + http?: HttpFn; fs?: FsHooks; sources?: readonly Source[]; retention?: number; @@ -176,6 +271,7 @@ export async function runPrebuild(opts: RunOpts): Promise<{ builds: Record; }> { const gh = opts.gh ?? defaultGh; + const http = opts.http ?? defaultHttp; const fs = opts.fs ?? defaultFs; const sources = opts.sources ?? SOURCES; const retention = opts.retention ?? 90; @@ -190,19 +286,13 @@ export async function runPrebuild(opts: RunOpts): Promise<{ for (const source of sources) { const repo = REPOS[source]; - log(`[${source}] listing releases from ${repo}`); - const releaseListRaw = gh([ - "release", - "list", - "--repo", - repo, - "--limit", - "200", - "--json", - "tagName,createdAt,isPrerelease", - ]); - const allReleases = JSON.parse(releaseListRaw) as GhRelease[]; - const nightlies = allReleases + log(`[${source}] enumerating releases from ${repo}`); + // Single paginated `gh api /releases` call yields every release with its + // metadata + asset URLs. Replaces the old `release list` (1 call) plus + // per-tag `release view` (~retention calls) — cuts enumeration cost from + // ~91 to 1-2 API calls per source. + const { list, detail: details } = fetchReleases(gh, repo); + const nightlies = list .filter((r) => TAG_RE.test(r.tagName)) .sort((a, b) => b.createdAt.localeCompare(a.createdAt)) .slice(0, retention); @@ -215,27 +305,6 @@ export async function runPrebuild(opts: RunOpts): Promise<{ const builds: BuildEntry[] = []; - // Pre-fetch release metadata for every candidate tag up front (the same - // `release view` we needed per-tag anyway, just gathered first) so we can - // locate the newest build bearing a completion marker. tags are newest-first. - const details = new Map(); - for (const r of tags) { - try { - const raw = gh([ - "release", - "view", - r.tagName, - "--repo", - repo, - "--json", - "tagName,createdAt,body,assets", - ]); - details.set(r.tagName, JSON.parse(raw) as GhReleaseDetail); - } catch (e) { - log(`[${source}] ${r.tagName} metadata fetch failed: ${(e as Error).message}`); - } - } - // Completeness boundary. Upstream's publish job uploads the `_manifest.json` // marker LAST, so a build NEWER than the newest marked build that lacks its // own marker is a publish still in flight (or one that died on the GitHub @@ -282,19 +351,18 @@ export async function runPrebuild(opts: RunOpts): Promise<{ log(`[${source}] ${tag} downloading sizes.*.json`); fs.rmDir(tagCache); fs.mkdir(tagCache); + // Anonymous HTTPS to browser_download_url — bypasses the API bucket + // entirely. This is the single biggest saving vs the old `gh release + // download` path, which internally makes one API call per asset (~97 + // calls per tag × 90 tags = ~8700 API calls per source per run). try { - gh([ - "release", - "download", - tag, - "--repo", - repo, - "--pattern", - "sizes.*.json", - "--dir", - tagCache, - "--skip-existing", - ]); + for (const asset of detail.assets) { + if (!SIZES_RE.test(asset.name)) continue; + if (!asset.downloadUrl) { + throw new Error(`no browser_download_url on ${asset.name}`); + } + http(asset.downloadUrl, join(tagCache, asset.name)); + } } catch (e) { log(`[${source}] ${tag} download failed: ${(e as Error).message}`); continue; @@ -344,11 +412,11 @@ export async function runPrebuild(opts: RunOpts): Promise<{ const kconfigAvailableFor = await downloadKconfig({ builds, source, - repo, + details, sourceOut, cacheDir, forceRefetch, - gh, + http, fs, log, }); @@ -521,20 +589,21 @@ function emitTrends(opts: { async function downloadKconfig(args: { builds: BuildEntry[]; source: Source; - repo: string; + details: Map; sourceOut: string; cacheDir: string; forceRefetch: boolean; - gh: GhFn; + http: HttpFn; fs: FsHooks; log: (msg: string) => void; }): Promise { - const { builds, source, repo, sourceOut, cacheDir, forceRefetch, gh, fs, log } = args; + const { builds, source, details, sourceOut, cacheDir, forceRefetch, http, fs, log } = args; const kconfigOut = join(sourceOut, "kconfig"); for (const build of builds) { const tag = build.id; const kcCache = join(cacheDir, source, tag, "kconfig"); + const detail = details.get(tag); const cached = !forceRefetch && @@ -542,23 +611,22 @@ async function downloadKconfig(args: { fs.listDir(kcCache).some((n) => KCONFIG_GRAPH_RE.test(n)); if (!cached) { + if (!detail) continue; + const kconfigAssets = detail.assets.filter( + (a) => KCONFIG_GRAPH_RE.test(a.name) || KCONFIG_HELP_RE.test(a.name), + ); + if (kconfigAssets.length === 0) continue; // no kconfig on this tag fs.rmDir(kcCache); fs.mkdir(kcCache); try { - gh([ - "release", - "download", - tag, - "--repo", - repo, - "--pattern", - "kconfig-*.json", - "--dir", - kcCache, - "--skip-existing", - ]); - } catch { - // No kconfig assets on this tag — try the next one. + for (const asset of kconfigAssets) { + if (!asset.downloadUrl) { + throw new Error(`no browser_download_url on ${asset.name}`); + } + http(asset.downloadUrl, join(kcCache, asset.name)); + } + } catch (e) { + log(`[${source}] ${tag} kconfig download failed: ${(e as Error).message}`); continue; } } diff --git a/tests/prebuild.test.ts b/tests/prebuild.test.ts index 15d15cc..a6e8b18 100644 --- a/tests/prebuild.test.ts +++ b/tests/prebuild.test.ts @@ -1,10 +1,12 @@ import { describe, it, expect } from "vitest"; import { + fetchReleases, parseBody, platformFromAssetName, runPrebuild, type FsHooks, type GhFn, + type HttpFn, } from "../scripts/prebuild.mts"; // --------------------------------------------------------------------------- @@ -132,54 +134,135 @@ type FakeRelease = { assets: FakeAsset[]; }; -function makeGh(fs: FsHooks, releases: FakeRelease[]): GhFn { +function assetUrl(repo: string, tag: string, name: string): string { + return `https://github.com/${repo}/releases/download/${tag}/${name}`; +} + +function makeGh(releases: FakeRelease[]): GhFn { return (args) => { - // gh release list --repo --limit N --json tagName,createdAt,isPrerelease - if (args[0] === "release" && args[1] === "list") { + // Only endpoint the new prebuild hits: paginated /releases enumeration. + // Matches `gh api ... /repos//releases?per_page=100&page=N`. + if (args[0] === "api") { + const path = args.find((a) => a.startsWith("/repos/")); + if (!path) throw new Error(`unmocked gh api path: ${args.join(" ")}`); + const repoMatch = /^\/repos\/([^/]+\/[^/]+)\/releases/.exec(path); + if (!repoMatch) throw new Error(`unexpected gh api path: ${path}`); + const repo = repoMatch[1]; + const pageMatch = /[?&]page=(\d+)/.exec(path); + const page = pageMatch ? parseInt(pageMatch[1], 10) : 1; + // 100 per page; fakes fit in one page. + if (page > 1) return "[]"; return JSON.stringify( releases.map((r) => ({ - tagName: r.tagName, - createdAt: r.createdAt, - isPrerelease: r.isPrerelease, + tag_name: r.tagName, + created_at: r.createdAt, + prerelease: r.isPrerelease, + body: r.body, + assets: r.assets.map((a) => ({ + name: a.name, + size: a.size, + browser_download_url: assetUrl(repo, r.tagName, a.name), + })), })), ); } - - // gh release view --repo --json tagName,createdAt,body,assets - if (args[0] === "release" && args[1] === "view") { - const tag = args[2]; - const rel = releases.find((r) => r.tagName === tag); - if (!rel) throw new Error(`no fake release for tag ${tag}`); - return JSON.stringify({ - tagName: rel.tagName, - createdAt: rel.createdAt, - body: rel.body, - assets: rel.assets.map((a) => ({ name: a.name, size: a.size })), - }); - } - - // gh release download --repo --pattern 'sizes.*.json' --dir --skip-existing - if (args[0] === "release" && args[1] === "download") { - const tag = args[2]; - const dirIdx = args.indexOf("--dir"); - const target = args[dirIdx + 1]; - const rel = releases.find((r) => r.tagName === tag); - if (!rel) throw new Error(`no fake release for tag ${tag}`); - for (const a of rel.assets) { - if (a.name.startsWith("sizes.") && a.name.endsWith(".json")) { - fs.write( - target + "/" + a.name, - a.content ?? JSON.stringify({ schema: 1, board: "x" }), - ); - } - } - return ""; - } - throw new Error(`unmocked gh call: ${args.join(" ")}`); }; } +function makeHttp(fs: FsHooks, releases: FakeRelease[]): HttpFn { + return (url, target) => { + // URLs look like https://github.com//releases/download//. + const m = /\/releases\/download\/([^/]+)\/(.+)$/.exec(url); + if (!m) throw new Error(`unexpected download URL: ${url}`); + const [, tag, name] = m; + const rel = releases.find((r) => r.tagName === tag); + if (!rel) throw new Error(`no fake release for tag ${tag}`); + const asset = rel.assets.find((a) => a.name === name); + if (!asset) throw new Error(`no fake asset ${name} on ${tag}`); + fs.write( + target, + asset.content ?? JSON.stringify({ schema: 1, board: "x" }), + ); + }; +} + +// --------------------------------------------------------------------------- +// fetchReleases — one paginated call per source, no per-tag API calls +// --------------------------------------------------------------------------- + +describe("fetchReleases", () => { + it("caps at one API call per 100 releases and returns full detail", () => { + const rows = Array.from({ length: 42 }, (_, i) => ({ + tagName: `nightly-2026060${(i % 9) + 1}-abcdef${i.toString(16).padStart(1, "0")}`, + createdAt: `2026-07-0${(i % 9) + 1}T00:00:00Z`, + isPrerelease: true, + body: `sha=fake${i}\n`, + assets: [{ name: `sizes.p${i}-lite.json`, size: 1000 + i }], + })); + let calls = 0; + const gh: GhFn = (args) => { + calls++; + // Assert we're using /releases and not the old subcommands. + expect(args[0]).toBe("api"); + const path = args.find((a) => a.startsWith("/repos/"))!; + expect(path).toContain("/releases"); + expect(path).toContain("per_page=100"); + return JSON.stringify( + rows.map((r) => ({ + tag_name: r.tagName, + created_at: r.createdAt, + prerelease: r.isPrerelease, + body: r.body, + assets: r.assets.map((a) => ({ + name: a.name, + size: a.size, + browser_download_url: `https://github.com/OpenIPC/firmware/releases/download/${r.tagName}/${a.name}`, + })), + })), + ); + }; + const { list, detail } = fetchReleases(gh, "OpenIPC/firmware"); + // 42 releases fit in one page — stops early on partial page. + expect(calls).toBe(1); + expect(list).toHaveLength(42); + expect(detail.size).toBe(42); + const first = detail.get(rows[0].tagName)!; + expect(first.body).toBe(rows[0].body); + expect(first.assets[0].downloadUrl).toContain("browser_download_url".slice(0, 0)); // downloadUrl populated + expect(first.assets[0].downloadUrl).toContain( + `/releases/download/${rows[0].tagName}/${rows[0].assets[0].name}`, + ); + }); + + it("paginates when the API returns a full page", () => { + // 100 rows on page 1 → advance to page 2. Page 2 returns 5 → stop. + const page1 = Array.from({ length: 100 }, (_, i) => ({ + tag_name: `t${i}`, + created_at: "2026-06-01T00:00:00Z", + prerelease: true, + body: "", + assets: [], + })); + const page2 = Array.from({ length: 5 }, (_, i) => ({ + tag_name: `late-t${i}`, + created_at: "2026-05-30T00:00:00Z", + prerelease: true, + body: "", + assets: [], + })); + const gh: GhFn = (args) => { + const path = args.find((a) => a.startsWith("/repos/"))!; + const page = /[?&]page=(\d+)/.exec(path)?.[1] ?? "1"; + if (page === "1") return JSON.stringify(page1); + if (page === "2") return JSON.stringify(page2); + return "[]"; + }; + const { list } = fetchReleases(gh, "OpenIPC/firmware"); + expect(list).toHaveLength(105); + }); +}); + // --------------------------------------------------------------------------- // runPrebuild integration test // --------------------------------------------------------------------------- @@ -215,12 +298,14 @@ describe("runPrebuild", () => { it("walks one source, filters non-nightly tags, emits index + shards", async () => { const { fs, state } = memFs(); - const gh = makeGh(fs, releases); + const gh = makeGh(releases); + const http = makeHttp(fs, releases); const result = await runPrebuild({ outDir: "/out", cacheDir: "/cache", gh, + http, fs, sources: ["firmware"], retention: 90, @@ -283,12 +368,14 @@ describe("runPrebuild", () => { assets: [{ name: "sizes.foo-lite.json", size: 100 }], })); const { fs, state } = memFs(); - const gh = makeGh(fs, many); + const gh = makeGh(many); + const http = makeHttp(fs, many); await runPrebuild({ outDir: "/out", cacheDir: "/cache", gh, + http, fs, sources: ["firmware"], retention: 2, @@ -330,12 +417,14 @@ describe("runPrebuild", () => { }, ]; const { fs } = memFs(); - const gh = makeGh(fs, withMarker); + const gh = makeGh(withMarker); + const http = makeHttp(fs, withMarker); const result = await runPrebuild({ outDir: "/out", cacheDir: "/cache", gh, + http, fs, sources: ["firmware"], retention: 90, @@ -371,12 +460,14 @@ describe("runPrebuild", () => { }, ]; const { fs } = memFs(); - const gh = makeGh(fs, mixed); + const gh = makeGh(mixed); + const http = makeHttp(fs, mixed); const result = await runPrebuild({ outDir: "/out", cacheDir: "/cache", gh, + http, fs, sources: ["firmware"], retention: 90, @@ -404,7 +495,8 @@ describe("runPrebuild", () => { }, ]; const { fs, state } = memFs(); - const gh = makeGh(fs, complete); + const gh = makeGh(complete); + const http = makeHttp(fs, complete); // Seed a stale, partial cache from an earlier rate-limited run: only 1 of // the 2 shards the release now advertises. @@ -417,6 +509,7 @@ describe("runPrebuild", () => { outDir: "/out", cacheDir: "/cache", gh, + http, fs, sources: ["firmware"], retention: 90, @@ -447,12 +540,14 @@ describe("runPrebuild", () => { }, ]; const { fs, state } = memFs(); - const gh = makeGh(fs, compound); + const gh = makeGh(compound); + const http = makeHttp(fs, compound); const result = await runPrebuild({ outDir: "/out", cacheDir: "/cache", gh, + http, fs, sources: ["builder"], retention: 1,