Merge pull request #221 from modelstudioai/fix/fc-oidc-token-refresh

fix(release): refresh OIDC token for FC uploads

Author: gujieye

tools/release/lib/oss-direct-upload.mjs

@@ -38,6 +38,14 @@
 import { readFileSync, statSync } from "node:fs";
 import { basename } from "node:path";
 
+const MAX_RETRIES = 20;
+const ATTEMPT_TIMEOUT_MS = 600_000;
+const RETRY_BACKOFF_CAP_MS = 10_000;
+
+function retryDelayMs(failedAttempt) {
+  return Math.min(1000 * 2 ** (failedAttempt - 1), RETRY_BACKOFF_CAP_MS);
+}
+
 /**
  * Resolve the FC release channel context; null when the channel is disabled.
  * Reuses the shared FC_TRIGGER_URL (the same function already serves
@@ -78,7 +86,10 @@ async function fetchOidcToken(audience) {
   const url = audience
     ? `${requestUrl}${separator}audience=${encodeURIComponent(audience)}`
     : requestUrl;
-  const res = await fetch(url, { headers: { Authorization: `bearer ${requestToken}` } });
+  const res = await fetch(url, {
+    headers: { Authorization: `bearer ${requestToken}` },
+    signal: AbortSignal.timeout(ATTEMPT_TIMEOUT_MS),
+  });
   if (!res.ok) {
     throw new Error(`GitHub OIDC token request failed: HTTP ${res.status}`);
   }
@@ -90,21 +101,22 @@ async function fetchOidcToken(audience) {
 }
 
 /**
- * Call an FC release action with the OIDC token. Retries transient network
- * failures; throws on HTTP errors and on `success: false` responses
- * (FC returns structured errors, e.g. OIDC verification failures).
+ * Call an FC release action with a fresh OIDC token for every attempt. Retries
+ * transient network failures; throws on HTTP errors and on `success: false`
+ * responses (FC returns structured errors, e.g. OIDC verification failures).
  */
-async function fcCall(ctx, action, payload, token, attempts = 3) {
+async function fcCall(ctx, action, payload, maxRetries = MAX_RETRIES) {
   for (let attempt = 1; ; attempt++) {
     try {
+      const token = await fetchOidcToken(ctx.audience);
       const res = await fetch(`${ctx.triggerUrl}/${action}`, {
         method: "POST",
         headers: {
           "Content-Type": "application/json",
           Authorization: `Bearer ${token}`,
         },
         body: JSON.stringify(payload),
-        signal: AbortSignal.timeout(120_000),
+        signal: AbortSignal.timeout(ATTEMPT_TIMEOUT_MS),
       });
       const body = await res.json().catch(() => null);
       if (!res.ok || !body?.success) {
@@ -119,10 +131,10 @@ async function fcCall(ctx, action, payload, token, attempts = 3) {
       // raw network failures, which surface as TypeError "fetch failed".
       const isNetworkError =
         error instanceof TypeError || /fetch failed|timeout/i.test(error.message);
-      if (attempt >= attempts || !isNetworkError) throw error;
-      const delay = 1000 * 2 ** (attempt - 1);
+      if (attempt > maxRetries || !isNetworkError) throw error;
+      const delay = retryDelayMs(attempt);
       process.stdout.write(
-        `  [fc] retry ${attempt}/${attempts - 1} for ${action} in ${delay}ms (${error.message})\n`,
+        `  [fc] retry ${attempt}/${maxRetries} for ${action} in ${delay}ms (${error.message})\n`,
       );
       await new Promise((resolve) => setTimeout(resolve, delay));
     }
@@ -155,7 +167,7 @@ async function runWithConcurrency(tasks, limit) {
 }
 
 /** PUT a local file to a presigned URL with exponential-backoff retries. */
-async function putWithRetry({ putUrl, contentType, body }, attempts = 3) {
+async function putWithRetry({ putUrl, contentType, body }, maxRetries = MAX_RETRIES) {
   for (let attempt = 1; ; attempt++) {
     try {
       // Only Content-Type was signed by FC; sending extra canonical headers
@@ -164,18 +176,18 @@ async function putWithRetry({ putUrl, contentType, body }, attempts = 3) {
         method: "PUT",
         headers: { "Content-Type": contentType },
         body,
-        signal: AbortSignal.timeout(600_000),
+        signal: AbortSignal.timeout(ATTEMPT_TIMEOUT_MS),
       });
       if (!res.ok) {
         const text = await res.text().catch(() => "");
         throw new Error(`HTTP ${res.status} ${text.slice(0, 200)}`);
       }
       return;
     } catch (error) {
-      if (attempt >= attempts) throw error;
-      const delay = 1000 * 2 ** (attempt - 1);
+      if (attempt > maxRetries) throw error;
+      const delay = retryDelayMs(attempt);
       process.stdout.write(
-        `  [oss] retry ${attempt}/${attempts - 1} in ${delay}ms (${error.message})\n`,
+        `  [oss] retry ${attempt}/${maxRetries} in ${delay}ms (${error.message})\n`,
       );
       await new Promise((resolve) => setTimeout(resolve, delay));
     }
@@ -194,20 +206,14 @@ async function putWithRetry({ putUrl, contentType, body }, attempts = 3) {
  * }} params
  */
 async function uploadViaFc({ ctx, prefix, jobs, label }) {
-  const token = await fetchOidcToken(ctx.audience);
-  const prepare = await fcCall(
-    ctx,
-    "release-prepare",
-    {
-      files: jobs.map((job) => ({
-        prefix,
-        tag: job.tag,
-        name: job.name,
-        contentType: contentTypeFor(job.name),
-      })),
-    },
-    token,
-  );
+  const prepare = await fcCall(ctx, "release-prepare", {
+    files: jobs.map((job) => ({
+      prefix,
+      tag: job.tag,
+      name: job.name,
+      contentType: contentTypeFor(job.name),
+    })),
+  });
   const uploads = prepare.uploads ?? [];
   if (uploads.length !== jobs.length) {
     throw new Error(
@@ -248,14 +254,9 @@ async function uploadViaFc({ ctx, prefix, jobs, label }) {
 
   // HEAD byte-size reconciliation now happens FC-side (it holds the only OSS
   // credentials); the runner reports local sizes as ground truth.
-  await fcCall(
-    ctx,
-    "release-finalize",
-    {
-      files: jobs.map((job, index) => ({ key: uploads[index].key, size: statSync(job.path).size })),
-    },
-    token,
-  );
+  await fcCall(ctx, "release-finalize", {
+    files: jobs.map((job, index) => ({ key: uploads[index].key, size: statSync(job.path).size })),
+  });
   process.stdout.write(
     `${label} reconcile ok: ${jobs.length}/${jobs.length} object(s) verified by FC\n`,
   );
@@ -330,13 +331,10 @@ export async function maintainReleaseManifest({ tag, channelJsonPath = null, dry
   }
 
   const body = JSON.parse(readFileSync(channelJsonPath, "utf-8"));
-  const token = await fetchOidcToken(ctx.audience);
-  const result = await fcCall(
-    ctx,
-    "release-finalize",
-    { files: [], manifest: { tag, body } },
-    token,
-  );
+  const result = await fcCall(ctx, "release-finalize", {
+    files: [],
+    manifest: { tag, body },
+  });
   if (result.manifestUpdated) {
     process.stdout.write(`manifest.json → latest=${result.latest ?? tag}\n`);
     process.stdout.write(`latest.json → ${result.latest ?? tag}\n`);

tools/release/lib/oss-direct-upload.test.mjs

@@ -0,0 +1,75 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it, vi } from "vite-plus/test";
+import { mirrorReleaseAssetsToOss } from "./oss-direct-upload.mjs";
+
+describe("OSS upload via FC", () => {
+  afterEach(() => {
+    vi.unstubAllEnvs();
+    vi.unstubAllGlobals();
+    vi.restoreAllMocks();
+  });
+
+  it("refreshes the GitHub OIDC token before release-finalize", async () => {
+    const tempDirectory = await mkdtemp(join(tmpdir(), "bailian-oss-upload-"));
+    const assetPath = join(tempDirectory, "asset.bin");
+    await writeFile(assetPath, "asset");
+
+    vi.stubEnv("FC_TRIGGER_URL", "https://fc.example");
+    vi.stubEnv("FC_RELEASE_AUDIENCE", "release-test");
+    vi.stubEnv("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "request-token");
+    vi.stubEnv("ACTIONS_ID_TOKEN_REQUEST_URL", "https://oidc.example/token");
+
+    let oidcRequestCount = 0;
+    const fcAuthorizations = [];
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async (input, init = {}) => {
+        const url = String(input);
+        if (url.startsWith("https://oidc.example/token")) {
+          oidcRequestCount += 1;
+          return Response.json({ value: `oidc-token-${oidcRequestCount}` });
+        }
+        if (url === "https://fc.example/release-prepare") {
+          fcAuthorizations.push(init.headers.Authorization);
+          return Response.json({
+            success: true,
+            uploads: [
+              {
+                key: "release/test/asset.bin",
+                putUrl: "https://oss.example/asset.bin",
+                contentType: "application/octet-stream",
+              },
+            ],
+          });
+        }
+        if (url === "https://oss.example/asset.bin") {
+          return new Response(null, { status: 200 });
+        }
+        if (url === "https://fc.example/release-finalize") {
+          fcAuthorizations.push(init.headers.Authorization);
+          if (init.headers.Authorization !== "Bearer oidc-token-2") {
+            return Response.json(
+              { success: false, error: "OIDC token 已过期", status: 403 },
+              { status: 403 },
+            );
+          }
+          return Response.json({ success: true });
+        }
+        throw new Error(`Unexpected request: ${url}`);
+      }),
+    );
+    vi.spyOn(process.stdout, "write").mockImplementation(() => true);
+
+    try {
+      await expect(
+        mirrorReleaseAssetsToOss({ plans: [{ tag: "test", paths: [assetPath] }] }),
+      ).resolves.toEqual({ uploaded: 1, skipped: false });
+      expect(oidcRequestCount).toBe(2);
+      expect(fcAuthorizations).toEqual(["Bearer oidc-token-1", "Bearer oidc-token-2"]);
+    } finally {
+      await rm(tempDirectory, { recursive: true, force: true });
+    }
+  });
+});