Ship only each build's own better-sqlite3 prebuild; fix the e2e first-run wizard race in every spec

CHANGELOG.md

@@ -34,6 +34,9 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
   14 days make it run the full backend search again. The optional embedding worker
   (`STRATOSORT_ENABLE_EMBEDDING_WORKER`) uses the same record and supported-backend filter.
 - The Windows x64 build no longer ships node-llama-cpp's arm64 prebuilt (about 11 MB).
+- Each build ships only its own SQLite (`better-sqlite3`) native binary instead of the ones for
+  every platform: about 15 MB smaller on macOS, about 12 MB on Linux, and one 2 MB file less on
+  Windows.
 - On Windows and macOS the app no longer forces Chromium's `enable-gpu-rasterization` and
   `enable-zero-copy` switches, so Chromium's per-driver GPU blocklist applies. Linux keeps forcing
   them, as before.

docs/RELEASING.md

@@ -409,14 +409,14 @@ supported-GPU floor moves with it. `verify-packaged-artifacts.js` requires all f
 package code: nothing reads it unless it builds from source, and nothing builds from source inside
 Electron.
 
-| Pattern                                                                                        | Why it is unused at runtime                                                                                                                                                                                                                                           | Estimated saving        |
-| :--------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------------------- |
-| `node-llama-cpp/llama/gitRelease.bundle`                                                       | Read only by `utils/gitReleaseBundles.js`, which only the source download/clone path uses. `getLlama` defaults `build` to `"never"` when `process.versions.electron` is set and the app never overrides it, so `canBuild` is false and the clone path is unreachable. | 36 MB, 1 file           |
-| `node-llama-cpp/llama/{addon,cmake,patches,profiles,toolchains,xpack}`, `llama/CMakeLists.txt` | Inputs to `compileLlamaCpp` (cmake-js), behind the same `canBuild` gate.                                                                                                                                                                                              | 0.3 MB, 57 files        |
-| `node-llama-cpp/templates`                                                                     | Used only by the `init` CLI command.                                                                                                                                                                                                                                  | 0.2 MB, 3 files         |
-| `@fugood/whisper.node/{src,whisper.cpp}`, `CMakeLists.txt`                                     | C/C++ sources for building the addon. `lib/` only `require`s the prebuilt variant packages (or `build/Release`).                                                                                                                                                      | 5.4 MB, about 117 files |
-| `better-sqlite3/{deps,src}`                                                                    | SQLite amalgamation and addon sources for node-gyp. `lib/binding.js` loads `prebuilds/<platform>-<arch>.node` or `build/Release`.                                                                                                                                     | 10.4 MB, 33 files       |
-| `better-sqlite3/prebuilds/{darwin,linux,linuxmusl}-*.node` (in `win.files`)                    | `getPrebuildPath()` only loads the `win32-<arch>` prebuild on Windows.                                                                                                                                                                                                | 13 MB, 6 files          |
+| Pattern                                                                                           | Why it is unused at runtime                                                                                                                                                                                                                                           | Estimated saving                                        |
+| :------------------------------------------------------------------------------------------------ | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------ |
+| `node-llama-cpp/llama/gitRelease.bundle`                                                          | Read only by `utils/gitReleaseBundles.js`, which only the source download/clone path uses. `getLlama` defaults `build` to `"never"` when `process.versions.electron` is set and the app never overrides it, so `canBuild` is false and the clone path is unreachable. | 36 MB, 1 file                                           |
+| `node-llama-cpp/llama/{addon,cmake,patches,profiles,toolchains,xpack}`, `llama/CMakeLists.txt`    | Inputs to `compileLlamaCpp` (cmake-js), behind the same `canBuild` gate.                                                                                                                                                                                              | 0.3 MB, 57 files                                        |
+| `node-llama-cpp/templates`                                                                        | Used only by the `init` CLI command.                                                                                                                                                                                                                                  | 0.2 MB, 3 files                                         |
+| `@fugood/whisper.node/{src,whisper.cpp}`, `CMakeLists.txt`                                        | C/C++ sources for building the addon. `lib/` only `require`s the prebuilt variant packages (or `build/Release`).                                                                                                                                                      | 5.4 MB, about 117 files                                 |
+| `better-sqlite3/{deps,src}`                                                                       | SQLite amalgamation and addon sources for node-gyp. `lib/binding.js` loads `prebuilds/<platform>-<arch>.node` or `build/Release`.                                                                                                                                     | 10.4 MB, 33 files                                       |
+| `better-sqlite3/prebuilds/` other than this build's own (`win.files`, `mac.files`, `linux.files`) | `getPrebuildPath()` loads only `<platform>-<arch>.node`, or `linuxmusl-<arch>.node` on musl Linux. Each platform's rule keeps `win32-${arch}`, `darwin-${arch}` or `linux-${arch}` plus `linuxmusl-${arch}`; electron-builder expands `${arch}` per packed arch.      | 15 MB, 7 files (Windows, macOS); 12 MB, 6 files (Linux) |
 
 Together that is roughly 65 MB and 220 files less to write and scan on first launch (estimates from
 the on-disk sizes in `node_modules`; the download shrinks less, by an estimated 40 to 50 MB, because
@@ -428,7 +428,10 @@ that `llama/gitRelease.bundle` does not. `@node-llama-cpp/win-arm64` (11 MB) dec
 arch's llama prebuilts with `!**/node_modules/@node-llama-cpp/win-!(${arch}|${arch}-*)/**`:
 electron-builder expands `${arch}` per packed arch, so the x64 build drops `win-arm64` and the arm64
 build (which inherits `win.files` through `extends`) drops any `win-x64*` package.
-`verify-packaged-artifacts.js` fails if a package for the other arch still holds files.
+`verify-packaged-artifacts.js` fails if a package for the other arch still holds files. It also
+requires each build's own `better-sqlite3` prebuild (without it `lib/binding.js` falls back to
+`build/Release`, which is not packaged, and SQLite cannot load) and fails if a prebuild for another
+platform or arch was packaged.
 
 ## Signing And Notarization Requirements
 

electron-builder.json

@@ -89,7 +89,7 @@
     "icon": "assets/icons/win/icon.ico",
     "target": ["nsis", "zip"],
     "files": [
-      "!**/node_modules/better-sqlite3/prebuilds/{darwin,linux,linuxmusl}-*.node",
+      "!**/node_modules/better-sqlite3/prebuilds/!(win32-${arch}).node",
       "!**/node_modules/@node-llama-cpp/win-!(${arch}|${arch}-*)/**"
     ]
   },
@@ -110,6 +110,9 @@
     "artifactName": "StratoSortStack-Setup-${version}.exe"
   },
   "mac": {
+    "files": [
+      "!**/node_modules/better-sqlite3/prebuilds/!(darwin-${arch}).node"
+    ],
     "icon": "assets/icons/mac/icon.icns",
     "category": "public.app-category.productivity",
     "hardenedRuntime": true,
@@ -139,6 +142,9 @@
     ]
   },
   "linux": {
+    "files": [
+      "!**/node_modules/better-sqlite3/prebuilds/!(linux-${arch}|linuxmusl-${arch}).node"
+    ],
     "icon": "assets/icons/png/512x512.png",
     "maintainer": "Kennedy Applied Sciences, Inc. <contact@kennedyappliedsciences.com>",
     "target": ["AppImage", "deb"],

scripts/verify-packaged-artifacts.js

@@ -118,6 +118,35 @@ function getForeignArchLlamaPrebuiltPackages(platform, arch) {
     : getRequiredLlamaPrebuiltPackages(platform, 'arm64');
 }
 
+const BETTER_SQLITE3_NODE_PLATFORM = {
+  win: 'win32',
+  win32: 'win32',
+  mac: 'darwin',
+  darwin: 'darwin',
+  linux: 'linux'
+};
+
+/**
+ * better-sqlite3 prebuilds one build may carry. lib/binding.js
+ * getPrebuildPath() loads `<platform>-<arch>.node`, or on Linux
+ * `linuxmusl-<arch>.node` when glibc is absent, so that one may ship too.
+ * electron-builder.json excludes every other prebuild per platform. Without
+ * the matching file binding.js falls back to build/Release, which is not
+ * packaged, and SQLite fails to load. An empty arch is x64, electron-builder's
+ * default.
+ *
+ * @returns {{ required: string[], allowed: string[] }}
+ */
+function getBetterSqlitePrebuilds(platform, arch) {
+  const nodePlatform = BETTER_SQLITE3_NODE_PLATFORM[String(platform || '').toLowerCase()];
+  if (!nodePlatform) return { required: [], allowed: [] };
+  const normalizedArch = String(arch || '').toLowerCase() || 'x64';
+  const required = [`${nodePlatform}-${normalizedArch}.node`];
+  const allowed =
+    nodePlatform === 'linux' ? [...required, `linuxmusl-${normalizedArch}.node`] : required;
+  return { required, allowed };
+}
+
 /** True when `dir` holds at least one file at any depth. */
 function containsAnyFile(dir) {
   if (!fs.existsSync(dir)) return false;
@@ -288,6 +317,27 @@ function verifyLlamaPrebuiltPackages(unpackedNodeModulesRoot, platform, arch) {
   });
 }
 
+function verifyBetterSqlitePrebuilds(unpackedNodeModulesRoot, platform, arch) {
+  const prebuildsDir = path.join(unpackedNodeModulesRoot, 'better-sqlite3', 'prebuilds');
+  const { required, allowed } = getBetterSqlitePrebuilds(platform, arch);
+  required.forEach((fileName) => {
+    ensureExists(
+      path.join(prebuildsDir, fileName),
+      `better-sqlite3 prebuild ${fileName} (lib/binding.js loads it; build/Release is not packaged)`
+    );
+  });
+  if (!fs.existsSync(prebuildsDir)) return;
+  const foreign = fs
+    .readdirSync(prebuildsDir)
+    .filter((name) => name.endsWith('.node') && !allowed.includes(name));
+  if (foreign.length > 0) {
+    throw new Error(
+      `better-sqlite3 prebuilds for another platform or arch were packaged: ${foreign.join(', ')} ` +
+        `in ${prebuildsDir}. Check the better-sqlite3 exclusions in electron-builder.json "files".`
+    );
+  }
+}
+
 function verifyWhisperVariantFolders(unpackedNodeModulesRoot, platform, arch) {
   const requiredPackages = getRequiredWhisperVariantPackages(platform, arch);
   requiredPackages.forEach((packageName) => {
@@ -331,6 +381,7 @@ function verifyWindowsArtifacts(buildRoot, arch, version = readPackageVersion())
   verifyNativeModuleFolders(unpackedModules);
   verifyNodeLlamaCppPayload(unpackedModules);
   verifyLlamaPrebuiltPackages(unpackedModules, 'win32', isArm64 ? 'arm64' : 'x64');
+  verifyBetterSqlitePrebuilds(unpackedModules, 'win32', isArm64 ? 'arm64' : 'x64');
   verifyWhisperVariantFolders(unpackedModules, 'win32', isArm64 ? 'arm64' : 'x64');
 }
 
@@ -363,6 +414,7 @@ function verifyMacArtifacts(buildRoot, arch) {
     );
     verifyNativeModuleFolders(unpackedModules);
     verifyNodeLlamaCppPayload(unpackedModules);
+    verifyBetterSqlitePrebuilds(unpackedModules, 'darwin', arch);
     verifyWhisperVariantFolders(unpackedModules, 'darwin', arch);
   });
 }
@@ -386,6 +438,7 @@ function verifyLinuxArtifacts(buildRoot, arch) {
   );
   verifyNativeModuleFolders(unpackedModules);
   verifyNodeLlamaCppPayload(unpackedModules);
+  verifyBetterSqlitePrebuilds(unpackedModules, 'linux', isArm64 ? 'arm64' : 'x64');
   verifyWhisperVariantFolders(unpackedModules, 'linux', isArm64 ? 'arm64' : 'x64');
 }
 
@@ -428,6 +481,8 @@ if (require.main === module) {
 module.exports = {
   getRequiredLlamaPrebuiltPackages,
   getForeignArchLlamaPrebuiltPackages,
+  getBetterSqlitePrebuilds,
+  verifyBetterSqlitePrebuilds,
   getRequiredMacElectronRuntimeFiles,
   getRequiredWhisperVariantPackages,
   getWindowsZipArtifactName,

test/e2e/helpers/electronApp.js

@@ -425,14 +425,22 @@ async function getMainWindow(app) {
  * Wait for the app to be ready for interaction
  *
  * @param {Page} window - The Playwright page object
- * @param {{ timeout?: number, dismissModals?: boolean }} [options]
+ * @param {{ timeout?: number, dismissModals?: boolean, dismissModelWizard?: boolean }} [options]
  *   timeout - maximum wait time in milliseconds (default 30000).
  *   dismissModals - close any open modal before returning (default true).
  *   Pass false when the modal is what the test is about: the first-run model
  *   wizard opens with no delay when the background setup has already fetched
  *   the embedding model (under 2 s on a fast runner), so it can be open by now.
+ *   dismissModelWizard - also wait for the first-run model wizard, which opens
+ *   ~2 s AFTER ready in a model-less profile, and skip it (default: the value
+ *   of dismissModals). Without this the single dismissModals() pass above can
+ *   run before the wizard mounts, and its aria-modal overlay then swallows the
+ *   test's later clicks. Costs up to 7 s when no wizard appears.
  */
-async function waitForAppReady(window, { timeout = 30000, dismissModals: dismiss = true } = {}) {
+async function waitForAppReady(
+  window,
+  { timeout = 30000, dismissModals: dismiss = true, dismissModelWizard = dismiss } = {}
+) {
   console.log('[E2E] Waiting for app to be ready...');
 
   // Wait for the navigation bar to be visible (indicates app is loaded)
@@ -465,6 +473,7 @@ async function waitForAppReady(window, { timeout = 30000, dismissModals: dismiss
 
   // Dismiss any modal dialogs that might be blocking (e.g., first-launch setup dialogs)
   if (dismiss) await dismissModals(window);
+  if (dismissModelWizard) await dismissModelSetupWizard(window);
 
   console.log('[E2E] App is ready');
 }
@@ -530,6 +539,9 @@ async function dismissModals(window) {
   }
 }
 
+/** Windows whose first-run model wizard was skipped (see dismissModelSetupWizard). */
+const modelWizardSkippedWindows = new WeakSet();
+
 /**
  * Dismiss the first-run "Set up on-device AI" model wizard if/when it appears.
  *
@@ -557,6 +569,10 @@ async function dismissModals(window) {
  */
 async function dismissModelSetupWizard(window, { timeout = 7000 } = {}) {
   if (!window || (typeof window.isClosed === 'function' && window.isClosed())) return false;
+  // Skipping latches the wizard closed for this renderer session, so a later
+  // call on the same window (waitForAppReady already ran this) returns at once
+  // instead of waiting `timeout` for a wizard that cannot reopen.
+  if (modelWizardSkippedWindows.has(window)) return false;
 
   // Identify the wizard by the hosting Modal's title (a stable, human-readable
   // string) rather than the Modal's generated aria id.
@@ -590,12 +606,15 @@ async function dismissModelSetupWizard(window, { timeout = 7000 } = {}) {
     await window.keyboard.press('Escape').catch(() => {});
   }
 
-  await wizardDialog
+  const closed = await wizardDialog
     .first()
     .waitFor({ state: 'hidden', timeout: 5000 })
+    .then(() => true)
     .catch(() => {
       console.log('[E2E] Model setup wizard may still be visible after dismiss attempt');
+      return false;
     });
+  if (closed) modelWizardSkippedWindows.add(window);
 
   return true;
 }

test/electron-builder-config.test.js

@@ -102,8 +102,7 @@ describe('electron-builder.json node_modules exclusions', () => {
     'node_modules/@node-llama-cpp/win-x64-cuda/bins/win-x64-cuda/ggml-cuda.dll',
     'node_modules/@fugood/whisper.node/lib/binding.js',
     'node_modules/better-sqlite3/lib/binding.js',
-    'node_modules/better-sqlite3/prebuilds/win32-x64.node',
-    'node_modules/better-sqlite3/prebuilds/win32-arm64.node'
+    'node_modules/better-sqlite3/prebuilds/win32-x64.node'
   ])('Windows build keeps runtime file %s', (moduleRelativePath) => {
     expect(isPackaged('win', moduleRelativePath)).toBe(true);
   });
@@ -124,8 +123,31 @@ describe('electron-builder.json node_modules exclusions', () => {
     expect(isPackaged('win', moduleRelativePath, arch)).toBe(expected);
   });
 
-  test('the Windows-only prebuild exclusion does not leak into macOS or Linux builds', () => {
-    expect(isPackaged('mac', 'node_modules/better-sqlite3/prebuilds/darwin-arm64.node')).toBe(true);
-    expect(isPackaged('linux', 'node_modules/better-sqlite3/prebuilds/linux-x64.node')).toBe(true);
+  // lib/binding.js loads prebuilds/<platform>-<arch>.node (linuxmusl-<arch> on
+  // musl Linux) and otherwise falls back to build/Release, which is not
+  // packaged, so each build must keep exactly its own prebuild.
+  const SQLITE_PREBUILDS = [
+    'darwin-arm64',
+    'darwin-x64',
+    'linux-arm64',
+    'linux-x64',
+    'linuxmusl-arm64',
+    'linuxmusl-x64',
+    'win32-arm64',
+    'win32-x64'
+  ];
+  test.each([
+    ['win', 'x64', ['win32-x64']],
+    ['win', 'arm64', ['win32-arm64']],
+    ['mac', 'x64', ['darwin-x64']],
+    ['mac', 'arm64', ['darwin-arm64']],
+    ['linux', 'x64', ['linux-x64', 'linuxmusl-x64']],
+    ['linux', 'arm64', ['linux-arm64', 'linuxmusl-arm64']]
+  ])('%s %s build keeps only its own better-sqlite3 prebuilds', (platformKey, arch, expected) => {
+    const kept = SQLITE_PREBUILDS.filter((name) =>
+      isPackaged(platformKey, `node_modules/better-sqlite3/prebuilds/${name}.node`, arch)
+    );
+    expect(kept).toEqual(expected);
+    expect(isPackaged(platformKey, 'node_modules/better-sqlite3/lib/binding.js', arch)).toBe(true);
   });
 });