Retry the in-build macOS notarization

docs/MAC_SIGNING.md

@@ -168,11 +168,16 @@ git push origin v4.0.0
 1. **`Detect mac release mode`** prints `::notice::Using signed + notarized mac release mode.` (If
    you see `::warning::macOS signing/notarization secrets are not configured`, one of the five
    secrets is missing or misnamed — go back to Phase 6.)
-2. **`Build signed Mac app and DMG`** runs (the `if: signed == 'true'` branch).
+2. **`Build signed Mac app and DMG`** runs (the `if: signed == 'true'` branch). After codesigning,
+   electron-builder's `afterSign` hook (`scripts/notarize-mac-app.js`) notarizes and staples the
+   `.app` before the DMG and ZIP are built, trying up to three times 30 s and 60 s apart.
+   electron-builder's own notarization is off (`mac.notarize: false`) because it makes one attempt,
+   and a dropped connection to Apple failed the macOS x64 job for both 5.1.0 and 5.1.1.
 3. **`Verify app signature`** runs `codesign --verify --deep --strict` on each `.app` bundle.
 4. **`Notarize and staple artifacts`** uploads to Apple via `xcrun notarytool submit --wait`, then
-   `xcrun stapler staple` attaches the ticket to each DMG. First submission from a new Team ID
-   typically takes 5–15 minutes; subsequent ones complete in 2–5 minutes.
+   `xcrun stapler staple` attaches the ticket to each DMG. Each call is retried up to three times.
+   First submission from a new Team ID typically takes 5–15 minutes; subsequent ones complete in 2–5
+   minutes.
 5. **`publish-release`** merges the per-arch `latest-mac-<arch>.yml` staging files into a single
    `latest-mac.yml` (the only name electron-updater requests on macOS) and uploads it with the DMG +
    ZIP files to the GitHub Release page. The per-arch staging files are never published.

electron-builder.json

@@ -108,7 +108,9 @@
     "license": "build/license_en.txt",
     "artifactName": "StratoSortStack-Setup-${version}.exe"
   },
+  "afterSign": "./scripts/notarize-mac-app.js",
   "mac": {
+    "notarize": false,
     "icon": "assets/icons/mac/icon.icns",
     "category": "public.app-category.productivity",
     "hardenedRuntime": true,

scripts/notarize-mac-app.js

@@ -0,0 +1,117 @@
+/**
+ * electron-builder `afterSign` hook: notarizes and staples the signed macOS
+ * app, retrying failed attempts.
+ *
+ * With APPLE_* credentials set, electron-builder notarizes the .app itself
+ * right after codesigning, in one attempt. A dropped connection to Apple's
+ * notary service then fails the whole build: the macOS x64 release jobs for
+ * 5.1.0 (NSURLErrorDomain -1005 while stapling) and 5.1.1 (-1009 while waiting
+ * on the submission) both died that way. electron-builder.json sets
+ * `mac.notarize` to false and names this hook, which makes the same
+ * @electron/notarize call with the same credentials at the same point (after
+ * codesigning, before the dmg and zip are built) and retries it.
+ */
+
+const path = require('path');
+
+const MAX_ATTEMPTS = 3;
+const RETRY_DELAY_MS = 30000;
+
+/**
+ * Same credential sources, precedence and errors as electron-builder's
+ * MacTargetHelper.getNotarizeOptions. Returns undefined without credentials.
+ */
+function getNotarizeOptions(appPath, env = process.env) {
+  const tool = 'notarytool';
+  const appleId = env.APPLE_ID;
+  const appleIdPassword = env.APPLE_APP_SPECIFIC_PASSWORD;
+  const teamId = env.APPLE_TEAM_ID;
+  if (appleId || appleIdPassword) {
+    if (!appleId) throw new Error('APPLE_ID env var needs to be set');
+    if (!appleIdPassword) throw new Error('APPLE_APP_SPECIFIC_PASSWORD env var needs to be set');
+    if (!teamId) throw new Error('APPLE_TEAM_ID env var needs to be set');
+    return { tool, appPath, appleId, appleIdPassword, teamId };
+  }
+  const appleApiKey = env.APPLE_API_KEY;
+  const appleApiKeyId = env.APPLE_API_KEY_ID;
+  const appleApiIssuer = env.APPLE_API_ISSUER;
+  if (appleApiKey || appleApiKeyId || appleApiIssuer) {
+    if (!appleApiKey || !appleApiKeyId || !appleApiIssuer) {
+      throw new Error(
+        'Env vars APPLE_API_KEY, APPLE_API_KEY_ID and APPLE_API_ISSUER need to be set'
+      );
+    }
+    return { tool, appPath, appleApiKey, appleApiKeyId, appleApiIssuer };
+  }
+  const keychainProfile = env.APPLE_KEYCHAIN_PROFILE;
+  if (keychainProfile) {
+    const keychain = env.APPLE_KEYCHAIN;
+    return { tool, appPath, keychainProfile, ...(keychain ? { keychain } : {}) };
+  }
+  return undefined;
+}
+
+/**
+ * The @electron/notarize electron-builder itself loads (app-builder-lib's own
+ * copy). The project's top-level @electron/notarize is ESM-only and needs
+ * Node 22.12, newer than the Node 20 the release runners use.
+ */
+function loadNotarize() {
+  const appBuilderLib = path.dirname(require.resolve('app-builder-lib/package.json'));
+  const notarizeEntry = require.resolve('@electron/notarize', { paths: [appBuilderLib] });
+  return require(notarizeEntry).notarize;
+}
+
+function firstLine(error) {
+  return String((error && error.message) || error).split('\n')[0];
+}
+
+/**
+ * Calls notarize(options) up to `attempts` times, waiting delayMs, then
+ * 2 x delayMs, between attempts. Resubmitting an app notarizes the same
+ * signature again; nothing else changes. Returns the attempt that succeeded
+ * and rethrows the last error.
+ */
+async function notarizeWithRetry(
+  notarize,
+  options,
+  {
+    attempts = MAX_ATTEMPTS,
+    delayMs = RETRY_DELAY_MS,
+    sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+    log = console
+  } = {}
+) {
+  for (let attempt = 1; ; attempt += 1) {
+    try {
+      await notarize(options);
+      return attempt;
+    } catch (error) {
+      if (attempt >= attempts) throw error;
+      const wait = delayMs * attempt;
+      log.warn(
+        `[notarize] Attempt ${attempt} of ${attempts} failed: ${firstLine(error)}; ` +
+          `retrying in ${wait / 1000} s`
+      );
+      await sleep(wait);
+    }
+  }
+}
+
+async function afterSign(context, { notarize, env = process.env, log = console, ...retry } = {}) {
+  // Mac App Store builds ("mas") are not notarized, as in electron-builder.
+  if (context.electronPlatformName !== 'darwin') return;
+  const appPath = path.join(context.appOutDir, `${context.packager.appInfo.productFilename}.app`);
+  const options = getNotarizeOptions(appPath, env);
+  if (!options) {
+    log.log('[notarize] Skipped: no APPLE_ID, APPLE_API_KEY or APPLE_KEYCHAIN_PROFILE credentials');
+    return;
+  }
+  const attempt = await notarizeWithRetry(notarize || loadNotarize(), options, { log, ...retry });
+  log.log(
+    `[notarize] ${path.basename(appPath)} notarized and stapled` +
+      (attempt > 1 ? ` on attempt ${attempt}` : '')
+  );
+}
+
+module.exports = { afterSign, getNotarizeOptions, loadNotarize, notarizeWithRetry };

test/notarizeMacApp.test.js

@@ -0,0 +1,228 @@
+/**
+ * @jest-environment node
+ *
+ * The afterSign hook replaces electron-builder's one-shot notarization with
+ * the same call, retried.
+ */
+
+// app-builder-lib loads graceful-fs, which cannot patch the memfs stand-in
+// that test-setup.js installs for 'fs'.
+jest.mock('fs', () => jest.requireActual('fs'));
+jest.mock('fs/promises', () => jest.requireActual('fs/promises'));
+
+const path = require('path');
+const { MacTargetHelper } = require('app-builder-lib/out/mac/MacTargetHelper');
+const builderConfig = require('../electron-builder.json');
+const {
+  afterSign,
+  getNotarizeOptions,
+  loadNotarize,
+  notarizeWithRetry
+} = require('../scripts/notarize-mac-app');
+
+const APP_PATH = '/out/mac/StratoSort Stack.app';
+const NOTARIZE_ENV_KEYS = [
+  'APPLE_ID',
+  'APPLE_APP_SPECIFIC_PASSWORD',
+  'APPLE_TEAM_ID',
+  'APPLE_API_KEY',
+  'APPLE_API_KEY_ID',
+  'APPLE_API_ISSUER',
+  'APPLE_KEYCHAIN',
+  'APPLE_KEYCHAIN_PROFILE'
+];
+
+function silentLog() {
+  return { log: jest.fn(), warn: jest.fn() };
+}
+
+function macContext(electronPlatformName = 'darwin') {
+  return {
+    electronPlatformName,
+    appOutDir: '/out/mac',
+    packager: { appInfo: { productFilename: 'StratoSort Stack' } }
+  };
+}
+
+describe('electron-builder notarization config', () => {
+  test('electron-builder skips its own notarization and runs the retrying hook', () => {
+    expect(builderConfig.mac.notarize).toBe(false);
+    expect(builderConfig.afterSign).toBe('./scripts/notarize-mac-app.js');
+  });
+
+  test('electron-builder resolves the configured hook to afterSign', async () => {
+    const { resolveFunction } = require('app-builder-lib/out/util/resolve');
+    const projectRoot = path.resolve(__dirname, '..');
+    const cwd = process.cwd();
+    // A "./" hook path resolves against the working directory, as in `npm run dist:*`.
+    process.chdir(projectRoot);
+    try {
+      const hook = await resolveFunction(
+        'commonjs',
+        builderConfig.afterSign,
+        'afterSign',
+        projectRoot
+      );
+      expect(typeof hook).toBe('function');
+      expect(hook.name).toBe('afterSign');
+    } finally {
+      process.chdir(cwd);
+    }
+  });
+
+  test('the hook loads the @electron/notarize that electron-builder uses', () => {
+    expect(typeof loadNotarize()).toBe('function');
+  });
+});
+
+describe('getNotarizeOptions matches electron-builder', () => {
+  const saved = {};
+
+  beforeEach(() => {
+    NOTARIZE_ENV_KEYS.forEach((key) => {
+      saved[key] = process.env[key];
+      delete process.env[key];
+    });
+  });
+
+  afterEach(() => {
+    NOTARIZE_ENV_KEYS.forEach((key) => {
+      if (saved[key] === undefined) delete process.env[key];
+      else process.env[key] = saved[key];
+    });
+  });
+
+  function electronBuilderResult() {
+    try {
+      return { options: MacTargetHelper.getNotarizeOptions(APP_PATH) };
+    } catch (error) {
+      return { error: error.message };
+    }
+  }
+
+  function hookResult() {
+    try {
+      return { options: getNotarizeOptions(APP_PATH, process.env) };
+    } catch (error) {
+      return { error: error.message };
+    }
+  }
+
+  test.each([
+    ['no credentials', {}],
+    ['Apple ID', { APPLE_ID: 'a@b.c', APPLE_APP_SPECIFIC_PASSWORD: 'pw', APPLE_TEAM_ID: 'TEAM' }],
+    ['Apple ID without team', { APPLE_ID: 'a@b.c', APPLE_APP_SPECIFIC_PASSWORD: 'pw' }],
+    ['password without Apple ID', { APPLE_APP_SPECIFIC_PASSWORD: 'pw', APPLE_TEAM_ID: 'TEAM' }],
+    ['API key', { APPLE_API_KEY: 'k.p8', APPLE_API_KEY_ID: 'KID', APPLE_API_ISSUER: 'ISS' }],
+    ['incomplete API key', { APPLE_API_KEY: 'k.p8' }],
+    ['keychain profile', { APPLE_KEYCHAIN_PROFILE: 'notary' }],
+    ['keychain profile in a keychain', { APPLE_KEYCHAIN_PROFILE: 'notary', APPLE_KEYCHAIN: 'k' }]
+  ])('%s', (_label, env) => {
+    Object.assign(process.env, env);
+    expect(hookResult()).toEqual(electronBuilderResult());
+  });
+});
+
+describe('notarizeWithRetry', () => {
+  const options = { tool: 'notarytool', appPath: APP_PATH };
+
+  test('succeeds on the first attempt without waiting', async () => {
+    const notarize = jest.fn().mockResolvedValue(undefined);
+    const sleep = jest.fn();
+
+    await expect(notarizeWithRetry(notarize, options, { sleep, log: silentLog() })).resolves.toBe(
+      1
+    );
+    expect(notarize).toHaveBeenCalledWith(options);
+    expect(sleep).not.toHaveBeenCalled();
+  });
+
+  test('retries a dropped connection and backs off', async () => {
+    const offline = new Error(
+      'Failed to notarize via notarytool. Failed with unexpected result: \n' +
+        'Error: HTTPError(statusCode: nil, error: Error Domain=NSURLErrorDomain Code=-1009'
+    );
+    const notarize = jest
+      .fn()
+      .mockRejectedValueOnce(offline)
+      .mockRejectedValueOnce(offline)
+      .mockResolvedValueOnce(undefined);
+    const sleep = jest.fn().mockResolvedValue(undefined);
+    const log = silentLog();
+
+    await expect(notarizeWithRetry(notarize, options, { sleep, log })).resolves.toBe(3);
+    expect(notarize).toHaveBeenCalledTimes(3);
+    expect(sleep.mock.calls).toEqual([[30000], [60000]]);
+    expect(log.warn).toHaveBeenCalledWith(
+      '[notarize] Attempt 1 of 3 failed: Failed to notarize via notarytool. Failed with unexpected result: ; retrying in 30 s'
+    );
+  });
+
+  test('gives up after the last attempt with its error', async () => {
+    const notarize = jest
+      .fn()
+      .mockRejectedValueOnce(new Error('first'))
+      .mockRejectedValueOnce(new Error('second'))
+      .mockRejectedValueOnce(new Error('third'));
+    const sleep = jest.fn().mockResolvedValue(undefined);
+
+    await expect(notarizeWithRetry(notarize, options, { sleep, log: silentLog() })).rejects.toThrow(
+      'third'
+    );
+    expect(notarize).toHaveBeenCalledTimes(3);
+  });
+});
+
+describe('afterSign hook', () => {
+  const appleIdEnv = {
+    APPLE_ID: 'a@b.c',
+    APPLE_APP_SPECIFIC_PASSWORD: 'pw',
+    APPLE_TEAM_ID: 'TEAM'
+  };
+
+  test('notarizes the signed app with the Apple ID credentials', async () => {
+    const notarize = jest.fn().mockResolvedValue(undefined);
+    const log = silentLog();
+
+    await afterSign(macContext(), { notarize, env: appleIdEnv, log });
+    expect(notarize).toHaveBeenCalledWith({
+      tool: 'notarytool',
+      appPath: path.join('/out/mac', 'StratoSort Stack.app'),
+      appleId: 'a@b.c',
+      appleIdPassword: 'pw',
+      teamId: 'TEAM'
+    });
+    expect(log.log).toHaveBeenCalledWith('[notarize] StratoSort Stack.app notarized and stapled');
+  });
+
+  test.each(['win32', 'linux', 'mas'])('does nothing for %s builds', async (platform) => {
+    const notarize = jest.fn();
+    await afterSign(macContext(platform), { notarize, env: appleIdEnv, log: silentLog() });
+    expect(notarize).not.toHaveBeenCalled();
+  });
+
+  test('skips without credentials, as electron-builder did', async () => {
+    const notarize = jest.fn();
+    const log = silentLog();
+    await afterSign(macContext(), { notarize, env: {}, log });
+    expect(notarize).not.toHaveBeenCalled();
+    expect(log.log).toHaveBeenCalledWith(
+      '[notarize] Skipped: no APPLE_ID, APPLE_API_KEY or APPLE_KEYCHAIN_PROFILE credentials'
+    );
+  });
+
+  test('fails the build when every attempt fails', async () => {
+    const notarize = jest
+      .fn()
+      .mockRejectedValue(new Error('The Internet connection appears to be offline.'));
+    await expect(
+      afterSign(macContext(), {
+        notarize,
+        env: appleIdEnv,
+        log: silentLog(),
+        sleep: jest.fn().mockResolvedValue(undefined)
+      })
+    ).rejects.toThrow('offline');
+    expect(notarize).toHaveBeenCalledTimes(3);
+  });
+});