diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c9f3b7..ea26f34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,9 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 14 days make it run the full backend search again. The optional embedding worker (`STRATOSORT_ENABLE_EMBEDDING_WORKER`) uses the same record and supported-backend filter. - The Windows x64 build no longer ships node-llama-cpp's arm64 prebuilt (about 11 MB). +- Each build ships only its own SQLite (`better-sqlite3`) native binary instead of the ones for + every platform: about 15 MB smaller on macOS, about 12 MB on Linux, and one 2 MB file less on + Windows. - On Windows and macOS the app no longer forces Chromium's `enable-gpu-rasterization` and `enable-zero-copy` switches, so Chromium's per-driver GPU blocklist applies. Linux keeps forcing them, as before. diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 4885ef0..077f5f4 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -409,14 +409,14 @@ supported-GPU floor moves with it. `verify-packaged-artifacts.js` requires all f package code: nothing reads it unless it builds from source, and nothing builds from source inside Electron. -| Pattern | Why it is unused at runtime | Estimated saving | -| :--------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------------------- | -| `node-llama-cpp/llama/gitRelease.bundle` | Read only by `utils/gitReleaseBundles.js`, which only the source download/clone path uses. `getLlama` defaults `build` to `"never"` when `process.versions.electron` is set and the app never overrides it, so `canBuild` is false and the clone path is unreachable. | 36 MB, 1 file | -| `node-llama-cpp/llama/{addon,cmake,patches,profiles,toolchains,xpack}`, `llama/CMakeLists.txt` | Inputs to `compileLlamaCpp` (cmake-js), behind the same `canBuild` gate. | 0.3 MB, 57 files | -| `node-llama-cpp/templates` | Used only by the `init` CLI command. | 0.2 MB, 3 files | -| `@fugood/whisper.node/{src,whisper.cpp}`, `CMakeLists.txt` | C/C++ sources for building the addon. `lib/` only `require`s the prebuilt variant packages (or `build/Release`). | 5.4 MB, about 117 files | -| `better-sqlite3/{deps,src}` | SQLite amalgamation and addon sources for node-gyp. `lib/binding.js` loads `prebuilds/-.node` or `build/Release`. | 10.4 MB, 33 files | -| `better-sqlite3/prebuilds/{darwin,linux,linuxmusl}-*.node` (in `win.files`) | `getPrebuildPath()` only loads the `win32-` prebuild on Windows. | 13 MB, 6 files | +| Pattern | Why it is unused at runtime | Estimated saving | +| :------------------------------------------------------------------------------------------------ | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------ | +| `node-llama-cpp/llama/gitRelease.bundle` | Read only by `utils/gitReleaseBundles.js`, which only the source download/clone path uses. `getLlama` defaults `build` to `"never"` when `process.versions.electron` is set and the app never overrides it, so `canBuild` is false and the clone path is unreachable. | 36 MB, 1 file | +| `node-llama-cpp/llama/{addon,cmake,patches,profiles,toolchains,xpack}`, `llama/CMakeLists.txt` | Inputs to `compileLlamaCpp` (cmake-js), behind the same `canBuild` gate. | 0.3 MB, 57 files | +| `node-llama-cpp/templates` | Used only by the `init` CLI command. | 0.2 MB, 3 files | +| `@fugood/whisper.node/{src,whisper.cpp}`, `CMakeLists.txt` | C/C++ sources for building the addon. `lib/` only `require`s the prebuilt variant packages (or `build/Release`). | 5.4 MB, about 117 files | +| `better-sqlite3/{deps,src}` | SQLite amalgamation and addon sources for node-gyp. `lib/binding.js` loads `prebuilds/-.node` or `build/Release`. | 10.4 MB, 33 files | +| `better-sqlite3/prebuilds/` other than this build's own (`win.files`, `mac.files`, `linux.files`) | `getPrebuildPath()` loads only `-.node`, or `linuxmusl-.node` on musl Linux. Each platform's rule keeps `win32-${arch}`, `darwin-${arch}` or `linux-${arch}` plus `linuxmusl-${arch}`; electron-builder expands `${arch}` per packed arch. | 15 MB, 7 files (Windows, macOS); 12 MB, 6 files (Linux) | Together that is roughly 65 MB and 220 files less to write and scan on first launch (estimates from the on-disk sizes in `node_modules`; the download shrinks less, by an estimated 40 to 50 MB, because @@ -428,7 +428,10 @@ that `llama/gitRelease.bundle` does not. `@node-llama-cpp/win-arm64` (11 MB) dec arch's llama prebuilts with `!**/node_modules/@node-llama-cpp/win-!(${arch}|${arch}-*)/**`: electron-builder expands `${arch}` per packed arch, so the x64 build drops `win-arm64` and the arm64 build (which inherits `win.files` through `extends`) drops any `win-x64*` package. -`verify-packaged-artifacts.js` fails if a package for the other arch still holds files. +`verify-packaged-artifacts.js` fails if a package for the other arch still holds files. It also +requires each build's own `better-sqlite3` prebuild (without it `lib/binding.js` falls back to +`build/Release`, which is not packaged, and SQLite cannot load) and fails if a prebuild for another +platform or arch was packaged. ## Signing And Notarization Requirements diff --git a/electron-builder.json b/electron-builder.json index cb0bcdf..ced8862 100644 --- a/electron-builder.json +++ b/electron-builder.json @@ -89,7 +89,7 @@ "icon": "assets/icons/win/icon.ico", "target": ["nsis", "zip"], "files": [ - "!**/node_modules/better-sqlite3/prebuilds/{darwin,linux,linuxmusl}-*.node", + "!**/node_modules/better-sqlite3/prebuilds/!(win32-${arch}).node", "!**/node_modules/@node-llama-cpp/win-!(${arch}|${arch}-*)/**" ] }, @@ -110,6 +110,9 @@ "artifactName": "StratoSortStack-Setup-${version}.exe" }, "mac": { + "files": [ + "!**/node_modules/better-sqlite3/prebuilds/!(darwin-${arch}).node" + ], "icon": "assets/icons/mac/icon.icns", "category": "public.app-category.productivity", "hardenedRuntime": true, @@ -139,6 +142,9 @@ ] }, "linux": { + "files": [ + "!**/node_modules/better-sqlite3/prebuilds/!(linux-${arch}|linuxmusl-${arch}).node" + ], "icon": "assets/icons/png/512x512.png", "maintainer": "Kennedy Applied Sciences, Inc. ", "target": ["AppImage", "deb"], diff --git a/scripts/verify-packaged-artifacts.js b/scripts/verify-packaged-artifacts.js index 678cd33..eeb6714 100644 --- a/scripts/verify-packaged-artifacts.js +++ b/scripts/verify-packaged-artifacts.js @@ -118,6 +118,35 @@ function getForeignArchLlamaPrebuiltPackages(platform, arch) { : getRequiredLlamaPrebuiltPackages(platform, 'arm64'); } +const BETTER_SQLITE3_NODE_PLATFORM = { + win: 'win32', + win32: 'win32', + mac: 'darwin', + darwin: 'darwin', + linux: 'linux' +}; + +/** + * better-sqlite3 prebuilds one build may carry. lib/binding.js + * getPrebuildPath() loads `-.node`, or on Linux + * `linuxmusl-.node` when glibc is absent, so that one may ship too. + * electron-builder.json excludes every other prebuild per platform. Without + * the matching file binding.js falls back to build/Release, which is not + * packaged, and SQLite fails to load. An empty arch is x64, electron-builder's + * default. + * + * @returns {{ required: string[], allowed: string[] }} + */ +function getBetterSqlitePrebuilds(platform, arch) { + const nodePlatform = BETTER_SQLITE3_NODE_PLATFORM[String(platform || '').toLowerCase()]; + if (!nodePlatform) return { required: [], allowed: [] }; + const normalizedArch = String(arch || '').toLowerCase() || 'x64'; + const required = [`${nodePlatform}-${normalizedArch}.node`]; + const allowed = + nodePlatform === 'linux' ? [...required, `linuxmusl-${normalizedArch}.node`] : required; + return { required, allowed }; +} + /** True when `dir` holds at least one file at any depth. */ function containsAnyFile(dir) { if (!fs.existsSync(dir)) return false; @@ -288,6 +317,27 @@ function verifyLlamaPrebuiltPackages(unpackedNodeModulesRoot, platform, arch) { }); } +function verifyBetterSqlitePrebuilds(unpackedNodeModulesRoot, platform, arch) { + const prebuildsDir = path.join(unpackedNodeModulesRoot, 'better-sqlite3', 'prebuilds'); + const { required, allowed } = getBetterSqlitePrebuilds(platform, arch); + required.forEach((fileName) => { + ensureExists( + path.join(prebuildsDir, fileName), + `better-sqlite3 prebuild ${fileName} (lib/binding.js loads it; build/Release is not packaged)` + ); + }); + if (!fs.existsSync(prebuildsDir)) return; + const foreign = fs + .readdirSync(prebuildsDir) + .filter((name) => name.endsWith('.node') && !allowed.includes(name)); + if (foreign.length > 0) { + throw new Error( + `better-sqlite3 prebuilds for another platform or arch were packaged: ${foreign.join(', ')} ` + + `in ${prebuildsDir}. Check the better-sqlite3 exclusions in electron-builder.json "files".` + ); + } +} + function verifyWhisperVariantFolders(unpackedNodeModulesRoot, platform, arch) { const requiredPackages = getRequiredWhisperVariantPackages(platform, arch); requiredPackages.forEach((packageName) => { @@ -331,6 +381,7 @@ function verifyWindowsArtifacts(buildRoot, arch, version = readPackageVersion()) verifyNativeModuleFolders(unpackedModules); verifyNodeLlamaCppPayload(unpackedModules); verifyLlamaPrebuiltPackages(unpackedModules, 'win32', isArm64 ? 'arm64' : 'x64'); + verifyBetterSqlitePrebuilds(unpackedModules, 'win32', isArm64 ? 'arm64' : 'x64'); verifyWhisperVariantFolders(unpackedModules, 'win32', isArm64 ? 'arm64' : 'x64'); } @@ -363,6 +414,7 @@ function verifyMacArtifacts(buildRoot, arch) { ); verifyNativeModuleFolders(unpackedModules); verifyNodeLlamaCppPayload(unpackedModules); + verifyBetterSqlitePrebuilds(unpackedModules, 'darwin', arch); verifyWhisperVariantFolders(unpackedModules, 'darwin', arch); }); } @@ -386,6 +438,7 @@ function verifyLinuxArtifacts(buildRoot, arch) { ); verifyNativeModuleFolders(unpackedModules); verifyNodeLlamaCppPayload(unpackedModules); + verifyBetterSqlitePrebuilds(unpackedModules, 'linux', isArm64 ? 'arm64' : 'x64'); verifyWhisperVariantFolders(unpackedModules, 'linux', isArm64 ? 'arm64' : 'x64'); } @@ -428,6 +481,8 @@ if (require.main === module) { module.exports = { getRequiredLlamaPrebuiltPackages, getForeignArchLlamaPrebuiltPackages, + getBetterSqlitePrebuilds, + verifyBetterSqlitePrebuilds, getRequiredMacElectronRuntimeFiles, getRequiredWhisperVariantPackages, getWindowsZipArtifactName, diff --git a/test/e2e/helpers/electronApp.js b/test/e2e/helpers/electronApp.js index bd6fe5c..c29b584 100644 --- a/test/e2e/helpers/electronApp.js +++ b/test/e2e/helpers/electronApp.js @@ -425,14 +425,22 @@ async function getMainWindow(app) { * Wait for the app to be ready for interaction * * @param {Page} window - The Playwright page object - * @param {{ timeout?: number, dismissModals?: boolean }} [options] + * @param {{ timeout?: number, dismissModals?: boolean, dismissModelWizard?: boolean }} [options] * timeout - maximum wait time in milliseconds (default 30000). * dismissModals - close any open modal before returning (default true). * Pass false when the modal is what the test is about: the first-run model * wizard opens with no delay when the background setup has already fetched * the embedding model (under 2 s on a fast runner), so it can be open by now. + * dismissModelWizard - also wait for the first-run model wizard, which opens + * ~2 s AFTER ready in a model-less profile, and skip it (default: the value + * of dismissModals). Without this the single dismissModals() pass above can + * run before the wizard mounts, and its aria-modal overlay then swallows the + * test's later clicks. Costs up to 7 s when no wizard appears. */ -async function waitForAppReady(window, { timeout = 30000, dismissModals: dismiss = true } = {}) { +async function waitForAppReady( + window, + { timeout = 30000, dismissModals: dismiss = true, dismissModelWizard = dismiss } = {} +) { console.log('[E2E] Waiting for app to be ready...'); // Wait for the navigation bar to be visible (indicates app is loaded) @@ -465,6 +473,7 @@ async function waitForAppReady(window, { timeout = 30000, dismissModals: dismiss // Dismiss any modal dialogs that might be blocking (e.g., first-launch setup dialogs) if (dismiss) await dismissModals(window); + if (dismissModelWizard) await dismissModelSetupWizard(window); console.log('[E2E] App is ready'); } @@ -530,6 +539,9 @@ async function dismissModals(window) { } } +/** Windows whose first-run model wizard was skipped (see dismissModelSetupWizard). */ +const modelWizardSkippedWindows = new WeakSet(); + /** * Dismiss the first-run "Set up on-device AI" model wizard if/when it appears. * @@ -557,6 +569,10 @@ async function dismissModals(window) { */ async function dismissModelSetupWizard(window, { timeout = 7000 } = {}) { if (!window || (typeof window.isClosed === 'function' && window.isClosed())) return false; + // Skipping latches the wizard closed for this renderer session, so a later + // call on the same window (waitForAppReady already ran this) returns at once + // instead of waiting `timeout` for a wizard that cannot reopen. + if (modelWizardSkippedWindows.has(window)) return false; // Identify the wizard by the hosting Modal's title (a stable, human-readable // string) rather than the Modal's generated aria id. @@ -590,12 +606,15 @@ async function dismissModelSetupWizard(window, { timeout = 7000 } = {}) { await window.keyboard.press('Escape').catch(() => {}); } - await wizardDialog + const closed = await wizardDialog .first() .waitFor({ state: 'hidden', timeout: 5000 }) + .then(() => true) .catch(() => { console.log('[E2E] Model setup wizard may still be visible after dismiss attempt'); + return false; }); + if (closed) modelWizardSkippedWindows.add(window); return true; } diff --git a/test/electron-builder-config.test.js b/test/electron-builder-config.test.js index 3c03100..a42137e 100644 --- a/test/electron-builder-config.test.js +++ b/test/electron-builder-config.test.js @@ -102,8 +102,7 @@ describe('electron-builder.json node_modules exclusions', () => { 'node_modules/@node-llama-cpp/win-x64-cuda/bins/win-x64-cuda/ggml-cuda.dll', 'node_modules/@fugood/whisper.node/lib/binding.js', 'node_modules/better-sqlite3/lib/binding.js', - 'node_modules/better-sqlite3/prebuilds/win32-x64.node', - 'node_modules/better-sqlite3/prebuilds/win32-arm64.node' + 'node_modules/better-sqlite3/prebuilds/win32-x64.node' ])('Windows build keeps runtime file %s', (moduleRelativePath) => { expect(isPackaged('win', moduleRelativePath)).toBe(true); }); @@ -124,8 +123,31 @@ describe('electron-builder.json node_modules exclusions', () => { expect(isPackaged('win', moduleRelativePath, arch)).toBe(expected); }); - test('the Windows-only prebuild exclusion does not leak into macOS or Linux builds', () => { - expect(isPackaged('mac', 'node_modules/better-sqlite3/prebuilds/darwin-arm64.node')).toBe(true); - expect(isPackaged('linux', 'node_modules/better-sqlite3/prebuilds/linux-x64.node')).toBe(true); + // lib/binding.js loads prebuilds/-.node (linuxmusl- on + // musl Linux) and otherwise falls back to build/Release, which is not + // packaged, so each build must keep exactly its own prebuild. + const SQLITE_PREBUILDS = [ + 'darwin-arm64', + 'darwin-x64', + 'linux-arm64', + 'linux-x64', + 'linuxmusl-arm64', + 'linuxmusl-x64', + 'win32-arm64', + 'win32-x64' + ]; + test.each([ + ['win', 'x64', ['win32-x64']], + ['win', 'arm64', ['win32-arm64']], + ['mac', 'x64', ['darwin-x64']], + ['mac', 'arm64', ['darwin-arm64']], + ['linux', 'x64', ['linux-x64', 'linuxmusl-x64']], + ['linux', 'arm64', ['linux-arm64', 'linuxmusl-arm64']] + ])('%s %s build keeps only its own better-sqlite3 prebuilds', (platformKey, arch, expected) => { + const kept = SQLITE_PREBUILDS.filter((name) => + isPackaged(platformKey, `node_modules/better-sqlite3/prebuilds/${name}.node`, arch) + ); + expect(kept).toEqual(expected); + expect(isPackaged(platformKey, 'node_modules/better-sqlite3/lib/binding.js', arch)).toBe(true); }); }); diff --git a/test/verifyPackagedArtifacts.test.js b/test/verifyPackagedArtifacts.test.js index 52e1cc1..f140a7e 100644 --- a/test/verifyPackagedArtifacts.test.js +++ b/test/verifyPackagedArtifacts.test.js @@ -8,10 +8,12 @@ const { NODE_LLAMA_CPP_RUNTIME_FILES, REQUIRED_ELECTRON_RUNTIME_FILES, REQUIRED_NATIVE_MODULE_PATHS, + getBetterSqlitePrebuilds, getForeignArchLlamaPrebuiltPackages, getRequiredLlamaPrebuiltPackages, getRequiredWhisperVariantPackages, getWindowsZipArtifactName, + verifyBetterSqlitePrebuilds, verifyMacArtifacts, verifyWindowsArtifacts } = require('../scripts/verify-packaged-artifacts'); @@ -45,6 +47,12 @@ function seedLlamaPrebuilts(unpackedModules, platform, arch) { }); } +function seedSqlitePrebuilds(unpackedModules, platform, arch) { + getBetterSqlitePrebuilds(platform, arch).required.forEach((fileName) => { + writeFile(path.join(unpackedModules, 'better-sqlite3', 'prebuilds', fileName)); + }); +} + function seedWhisperVariants(unpackedModules, platform, arch) { getRequiredWhisperVariantPackages(platform, arch).forEach((packageName) => { const segments = packageName.split('/'); @@ -76,6 +84,7 @@ function createWindowsBuildRoot({ arch = 'x64', skipWhisperVariants = false } = seedNativeModules(unpackedModules); seedLlamaPrebuilts(unpackedModules, 'win32', arch); + seedSqlitePrebuilds(unpackedModules, 'win32', arch); if (!skipWhisperVariants) { seedWhisperVariants(unpackedModules, 'win32', arch); } @@ -113,6 +122,7 @@ function createMacBuildRoot({ writeExecutable(path.join(resources, 'assets', 'runtime', 'ffmpeg', 'ffmpeg')); seedNativeModules(unpackedModules); + seedSqlitePrebuilds(unpackedModules, 'darwin', arch); if (!skipWhisperVariants) { seedWhisperVariants(unpackedModules, 'darwin', arch); } @@ -317,6 +327,69 @@ describe('verify-packaged-artifacts', () => { expect(() => verifyWindowsArtifacts(buildRoot, 'x64', VERSION)).not.toThrow(); }); + test('rejects Windows x64 builds missing their better-sqlite3 prebuild', () => { + const buildRoot = createWindowsBuildRoot(); + tempDirs.push(buildRoot); + fs.rmSync( + path.join(windowsUnpackedModules(buildRoot), 'better-sqlite3', 'prebuilds', 'win32-x64.node') + ); + + expect(() => verifyWindowsArtifacts(buildRoot, 'x64', VERSION)).toThrow( + /better-sqlite3 prebuild win32-x64\.node .* missing/ + ); + }); + + test('rejects macOS arm64 builds that also packaged the x64 better-sqlite3 prebuild', () => { + const buildRoot = createMacBuildRoot({ arch: 'arm64' }); + tempDirs.push(buildRoot); + writeFile( + path.join( + buildRoot, + 'mac-arm64', + 'StratoSort Stack.app', + 'Contents', + 'Resources', + 'app.asar.unpacked', + 'node_modules', + 'better-sqlite3', + 'prebuilds', + 'darwin-x64.node' + ) + ); + + expect(() => verifyMacArtifacts(buildRoot, 'arm64')).toThrow( + /better-sqlite3 prebuilds for another platform or arch were packaged: darwin-x64\.node/ + ); + }); + + test('Linux builds may carry the musl prebuild for the same arch only', () => { + const modules = fs.mkdtempSync(path.join(os.tmpdir(), 'stratosort-sqlite-prebuilds-')); + tempDirs.push(modules); + const prebuilds = path.join(modules, 'better-sqlite3', 'prebuilds'); + writeFile(path.join(prebuilds, 'linux-x64.node')); + writeFile(path.join(prebuilds, 'linuxmusl-x64.node')); + + expect(() => verifyBetterSqlitePrebuilds(modules, 'linux', 'x64')).not.toThrow(); + + writeFile(path.join(prebuilds, 'linux-arm64.node')); + expect(() => verifyBetterSqlitePrebuilds(modules, 'linux', 'x64')).toThrow( + /another platform or arch were packaged: linux-arm64\.node/ + ); + }); + + describe('getBetterSqlitePrebuilds policy', () => { + test.each([ + ['win', 'x64', ['win32-x64.node'], ['win32-x64.node']], + ['win32', 'arm64', ['win32-arm64.node'], ['win32-arm64.node']], + ['mac', 'arm64', ['darwin-arm64.node'], ['darwin-arm64.node']], + ['linux', '', ['linux-x64.node'], ['linux-x64.node', 'linuxmusl-x64.node']], + ['linux', 'arm64', ['linux-arm64.node'], ['linux-arm64.node', 'linuxmusl-arm64.node']], + ['freebsd', 'x64', [], []] + ])('%s %s', (platform, arch, required, allowed) => { + expect(getBetterSqlitePrebuilds(platform, arch)).toEqual({ required, allowed }); + }); + }); + test('rejects builds that packaged the node-llama-cpp git bundle', () => { const buildRoot = createWindowsBuildRoot(); tempDirs.push(buildRoot);