From 302fd12447c96c42b818920f15dd5477edef01d6 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sun, 19 Jul 2026 18:57:42 +0200 Subject: [PATCH 1/7] fix: context disposal race condition crash --- llama/addon/AddonModel.cpp | 2 +- llama/addon/addonGlobals.cpp | 2 + src/evaluator/LlamaContext/LlamaContext.ts | 47 ++++++++++++------- ...{functions.test.ts => checkpoints.test.ts} | 22 +++++++++ 4 files changed, 54 insertions(+), 19 deletions(-) rename test/modelDependent/qwen3.5-0.8b/{functions.test.ts => checkpoints.test.ts} (78%) diff --git a/llama/addon/AddonModel.cpp b/llama/addon/AddonModel.cpp index 72cfc79e..bd447e11 100644 --- a/llama/addon/AddonModel.cpp +++ b/llama/addon/AddonModel.cpp @@ -530,7 +530,7 @@ Napi::Value AddonModel::Dispose(const Napi::CallbackInfo& info) { return worker->GetPromise(); } else { disposeMT(); - + Napi::Promise::Deferred deferred = Napi::Promise::Deferred::New(info.Env()); deferred.Resolve(info.Env().Undefined()); return deferred.Promise(); diff --git a/llama/addon/addonGlobals.cpp b/llama/addon/addonGlobals.cpp index 69584fbc..63f804db 100644 --- a/llama/addon/addonGlobals.cpp +++ b/llama/addon/addonGlobals.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include #include "addonGlobals.h" #include "napi.h" diff --git a/src/evaluator/LlamaContext/LlamaContext.ts b/src/evaluator/LlamaContext/LlamaContext.ts index cb3be164..cde97350 100644 --- a/src/evaluator/LlamaContext/LlamaContext.ts +++ b/src/evaluator/LlamaContext/LlamaContext.ts @@ -1989,26 +1989,37 @@ export class LlamaContextSequence { if (!this.needsCheckpoints || this._nextTokenIndex === 0 || this._checkpoints.hasCheckpoint(name, this._nextTokenIndex - 1)) return; - if (this._checkpointOptions.maxMemory != null) - this._checkpoints.prepareMemoryForIncomingCheckpoint(this._checkpointOptions.maxMemory); - - const checkpoint = new this.model._llama._bindings.AddonContextSequenceCheckpoint(); - await checkpoint.init(this._context._ctx, this._sequenceId); - if (this._nextTokenIndex - 1 !== checkpoint.maxPos) - this.model._llama._log( - LlamaLogLevel.warn, - `Checkpoint max position mismatch: expected ${this._nextTokenIndex - 1}, got ${checkpoint.maxPos}` - ); + let preventDisposalHandle: DisposalPreventionHandle; + try { + preventDisposalHandle = this._context._backendContextDisposeGuard.createPreventDisposalHandle(); + } catch (err) { + return; + } - this._checkpoints.storeCheckpoint({ - name, - maxNamedCheckpoints, - checkpoint, - currentMaxPos: checkpoint.maxPos - }); + try { + if (this._checkpointOptions.maxMemory != null) + this._checkpoints.prepareMemoryForIncomingCheckpoint(this._checkpointOptions.maxMemory); + + const checkpoint = new this.model._llama._bindings.AddonContextSequenceCheckpoint(); + await checkpoint.init(this._context._ctx, this._sequenceId); + if (this._nextTokenIndex - 1 !== checkpoint.maxPos) + this.model._llama._log( + LlamaLogLevel.warn, + `Checkpoint max position mismatch: expected ${this._nextTokenIndex - 1}, got ${checkpoint.maxPos}` + ); + + this._checkpoints.storeCheckpoint({ + name, + maxNamedCheckpoints, + checkpoint, + currentMaxPos: checkpoint.maxPos + }); - if (this._checkpointOptions.maxMemory != null) - this._checkpoints.pruneToKeepUnderMemoryUsage(this._checkpointOptions.maxMemory); + if (this._checkpointOptions.maxMemory != null) + this._checkpoints.pruneToKeepUnderMemoryUsage(this._checkpointOptions.maxMemory); + } finally { + preventDisposalHandle.dispose(); + } } /** @internal */ diff --git a/test/modelDependent/qwen3.5-0.8b/functions.test.ts b/test/modelDependent/qwen3.5-0.8b/checkpoints.test.ts similarity index 78% rename from test/modelDependent/qwen3.5-0.8b/functions.test.ts rename to test/modelDependent/qwen3.5-0.8b/checkpoints.test.ts index fae80858..62b58c95 100644 --- a/test/modelDependent/qwen3.5-0.8b/functions.test.ts +++ b/test/modelDependent/qwen3.5-0.8b/checkpoints.test.ts @@ -72,5 +72,27 @@ describe("qwen3.5 0.8b", () => { expect(chatSession.sequence.lastCheckpointIndex).toMatchInlineSnapshot("448"); expect(chatSession.sequence.nextTokenIndex).toMatchInlineSnapshot("463"); }); + + test("disposing the context asynchronously works", {timeout: 1000 * 60 * 60 * 2}, async () => { + const modelPath = await getModelFile("Qwen3.5-0.8B-Q8_0.gguf"); + const llama = await getTestLlama(); + + const model = await llama.loadModel({ + modelPath + }); + + for (let i = 0; i < 4; i++) { + const context = await model.createContext({ + contextSize: 4096 + }); + const contextSequence = context.getSequence(); + const chatSession = new LlamaChatSession({contextSequence}); + + await chatSession.prompt("Tell me a short story", {maxTokens: 40}); + + void contextSequence.takeCheckpoint(); + await context.dispose(); // should not crash the process + } + }); }); }); From 797aa015cbd47f63a55bc51a0108693b46dc01c1 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sun, 19 Jul 2026 19:00:42 +0200 Subject: [PATCH 2/7] style: enforce using types instead of interfaces --- eslint.config.js | 1 + templates/electron-typescript-react/eslint.config.js | 1 + templates/node-typescript/eslint.config.js | 1 + 3 files changed, 3 insertions(+) diff --git a/eslint.config.js b/eslint.config.js index 179e0bef..29b92d98 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -188,6 +188,7 @@ export default defineConfig({ allow: [] }], "@typescript-eslint/explicit-member-accessibility": ["warn"], + "@typescript-eslint/consistent-type-definitions": ["warn", "type"], "@stylistic/member-delimiter-style": ["warn", { multiline: { delimiter: "comma", diff --git a/templates/electron-typescript-react/eslint.config.js b/templates/electron-typescript-react/eslint.config.js index 55aac1bb..0d33df06 100644 --- a/templates/electron-typescript-react/eslint.config.js +++ b/templates/electron-typescript-react/eslint.config.js @@ -169,6 +169,7 @@ export default defineConfig({ allow: [] }], "@typescript-eslint/explicit-member-accessibility": ["warn"], + "@typescript-eslint/consistent-type-definitions": ["warn", "type"], "@stylistic/member-delimiter-style": ["warn", { multiline: { delimiter: "comma", diff --git a/templates/node-typescript/eslint.config.js b/templates/node-typescript/eslint.config.js index 7f94b897..49ea5657 100644 --- a/templates/node-typescript/eslint.config.js +++ b/templates/node-typescript/eslint.config.js @@ -167,6 +167,7 @@ export default defineConfig({ allow: [] }], "@typescript-eslint/explicit-member-accessibility": ["warn"], + "@typescript-eslint/consistent-type-definitions": ["warn", "type"], "@stylistic/member-delimiter-style": ["warn", { multiline: { delimiter: "comma", From 441edc76cadb0dc64c660b3a5636cb5d3f06fbdc Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sun, 19 Jul 2026 19:39:16 +0200 Subject: [PATCH 3/7] docs: improve the fuzzy search --- .vitepress/config.ts | 62 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/.vitepress/config.ts b/.vitepress/config.ts index 139a8f38..380ad8e9 100644 --- a/.vitepress/config.ts +++ b/.vitepress/config.ts @@ -414,7 +414,8 @@ export default defineConfig({ ], module: ts.ModuleKind.ES2022, target: ts.ScriptTarget.ES2022, - moduleDetection: ts.ModuleDetectionKind.Force + moduleDetection: ts.ModuleDetectionKind.Force, + rootDir: undefined }, tsModule: ts } @@ -467,7 +468,66 @@ export default defineConfig({ options: { detailedView: true, miniSearch: { + options: { + processTerm(term) { + function splitIdentifier(value: string): string[] { + return value + // dryRun → dry Run + .replace(/([a-z\d])([A-Z])/g, "$1 $2") + // XMLHttpRequest → XML Http Request + .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") + .toLowerCase() + .match(/[\p{L}\p{N}]+/gu) ?? []; + } + + function addBoundedCompounds(parts: readonly string[], maxParts = 3): string[] { + const terms = new Set([parts.join("")]); + + for (let start = 0; start < parts.length; start++) { + let compound = ""; + + for (let end = start; end < parts.length && end < start + maxParts; end++) { + compound += parts[end]; + terms.add(compound); + } + } + + return [...terms]; + } + + return addBoundedCompounds(splitIdentifier(term)); + } + }, searchOptions: { + fuzzy: 0.05, + tokenize(query) { + function splitIdentifier(value: string): string[] { + return value + // dryRun → dry Run + .replace(/([a-z\d])([A-Z])/g, "$1 $2") + // XMLHttpRequest → XML Http Request + .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") + .toLowerCase() + .match(/[\p{L}\p{N}]+/gu) ?? []; + } + + function addBoundedCompounds(parts: readonly string[], maxParts = 3): string[] { + const terms = new Set([parts.join("")]); + + for (let start = 0; start < parts.length; start++) { + let compound = ""; + + for (let end = start; end < parts.length && end < start + maxParts; end++) { + compound += parts[end]; + terms.add(compound); + } + } + + return [...terms]; + } + + return addBoundedCompounds((query.match(/[\p{L}\p{N}]+/gu) ?? []).flatMap(splitIdentifier)); + }, boostDocument(term, documentId, storedFields) { const firstTitle = (storedFields?.titles as string[])?.[0]; if (firstTitle?.startsWith("Type Alias: ")) From 8e4b67d57c24736c606f30ef13da6629332595c3 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sun, 19 Jul 2026 20:14:12 +0200 Subject: [PATCH 4/7] build: modernize `tsconfig.json` --- .gitignore | 1 + .vitepress/tsconfig.json | 3 ++- packages/@node-llama-cpp/linux-arm64/.gitignore | 1 + packages/@node-llama-cpp/linux-arm64/tsconfig.json | 1 + packages/@node-llama-cpp/linux-armv7l/.gitignore | 1 + packages/@node-llama-cpp/linux-armv7l/tsconfig.json | 1 + packages/@node-llama-cpp/linux-riscv64/.gitignore | 1 + packages/@node-llama-cpp/linux-riscv64/tsconfig.json | 1 + packages/@node-llama-cpp/linux-x64-cuda-ext/.gitignore | 1 + packages/@node-llama-cpp/linux-x64-cuda-ext/tsconfig.json | 1 + packages/@node-llama-cpp/linux-x64-cuda/.gitignore | 1 + packages/@node-llama-cpp/linux-x64-cuda/tsconfig.json | 1 + packages/@node-llama-cpp/linux-x64-vulkan/.gitignore | 1 + packages/@node-llama-cpp/linux-x64-vulkan/tsconfig.json | 1 + packages/@node-llama-cpp/linux-x64/.gitignore | 1 + packages/@node-llama-cpp/linux-x64/tsconfig.json | 1 + packages/@node-llama-cpp/mac-arm64-metal/.gitignore | 1 + packages/@node-llama-cpp/mac-arm64-metal/tsconfig.json | 1 + packages/@node-llama-cpp/mac-x64/.gitignore | 1 + packages/@node-llama-cpp/mac-x64/tsconfig.json | 1 + packages/@node-llama-cpp/win-arm64/.gitignore | 1 + packages/@node-llama-cpp/win-arm64/tsconfig.json | 1 + packages/@node-llama-cpp/win-x64-cuda-ext/.gitignore | 1 + packages/@node-llama-cpp/win-x64-cuda-ext/tsconfig.json | 1 + packages/@node-llama-cpp/win-x64-cuda/.gitignore | 1 + packages/@node-llama-cpp/win-x64-cuda/tsconfig.json | 1 + packages/@node-llama-cpp/win-x64-vulkan/.gitignore | 1 + packages/@node-llama-cpp/win-x64-vulkan/tsconfig.json | 1 + packages/@node-llama-cpp/win-x64/.gitignore | 1 + packages/@node-llama-cpp/win-x64/tsconfig.json | 1 + packages/create-node-llama-cpp/.gitignore | 1 + packages/create-node-llama-cpp/tsconfig.json | 1 + templates/electron-typescript-react/.gitignore | 1 + templates/electron-typescript-react/package.json | 2 +- templates/node-typescript/.gitignore | 1 + templates/node-typescript/tsconfig.json | 1 + tsconfig.json | 1 + 37 files changed, 38 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 3dfe5583..781d098b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ node_modules *.cpuprofile /dist +/tsconfig.tsbuildinfo /docs-site /docs/api /templates/packed diff --git a/.vitepress/tsconfig.json b/.vitepress/tsconfig.json index 02138974..4b7b80b2 100644 --- a/.vitepress/tsconfig.json +++ b/.vitepress/tsconfig.json @@ -15,11 +15,12 @@ "noUncheckedIndexedAccess": true, "moduleDetection": "force", "skipLibCheck": true, - "moduleResolution": "bundler", + "moduleResolution": "node16", "resolveJsonModule": true, "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "..", "outDir": "./dist", "strict": true, "sourceMap": true, diff --git a/packages/@node-llama-cpp/linux-arm64/.gitignore b/packages/@node-llama-cpp/linux-arm64/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/linux-arm64/.gitignore +++ b/packages/@node-llama-cpp/linux-arm64/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/linux-arm64/tsconfig.json b/packages/@node-llama-cpp/linux-arm64/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/linux-arm64/tsconfig.json +++ b/packages/@node-llama-cpp/linux-arm64/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/@node-llama-cpp/linux-armv7l/.gitignore b/packages/@node-llama-cpp/linux-armv7l/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/linux-armv7l/.gitignore +++ b/packages/@node-llama-cpp/linux-armv7l/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/linux-armv7l/tsconfig.json b/packages/@node-llama-cpp/linux-armv7l/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/linux-armv7l/tsconfig.json +++ b/packages/@node-llama-cpp/linux-armv7l/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/@node-llama-cpp/linux-riscv64/.gitignore b/packages/@node-llama-cpp/linux-riscv64/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/linux-riscv64/.gitignore +++ b/packages/@node-llama-cpp/linux-riscv64/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/linux-riscv64/tsconfig.json b/packages/@node-llama-cpp/linux-riscv64/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/linux-riscv64/tsconfig.json +++ b/packages/@node-llama-cpp/linux-riscv64/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/@node-llama-cpp/linux-x64-cuda-ext/.gitignore b/packages/@node-llama-cpp/linux-x64-cuda-ext/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/linux-x64-cuda-ext/.gitignore +++ b/packages/@node-llama-cpp/linux-x64-cuda-ext/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/linux-x64-cuda-ext/tsconfig.json b/packages/@node-llama-cpp/linux-x64-cuda-ext/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/linux-x64-cuda-ext/tsconfig.json +++ b/packages/@node-llama-cpp/linux-x64-cuda-ext/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/@node-llama-cpp/linux-x64-cuda/.gitignore b/packages/@node-llama-cpp/linux-x64-cuda/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/linux-x64-cuda/.gitignore +++ b/packages/@node-llama-cpp/linux-x64-cuda/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/linux-x64-cuda/tsconfig.json b/packages/@node-llama-cpp/linux-x64-cuda/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/linux-x64-cuda/tsconfig.json +++ b/packages/@node-llama-cpp/linux-x64-cuda/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/@node-llama-cpp/linux-x64-vulkan/.gitignore b/packages/@node-llama-cpp/linux-x64-vulkan/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/linux-x64-vulkan/.gitignore +++ b/packages/@node-llama-cpp/linux-x64-vulkan/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/linux-x64-vulkan/tsconfig.json b/packages/@node-llama-cpp/linux-x64-vulkan/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/linux-x64-vulkan/tsconfig.json +++ b/packages/@node-llama-cpp/linux-x64-vulkan/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/@node-llama-cpp/linux-x64/.gitignore b/packages/@node-llama-cpp/linux-x64/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/linux-x64/.gitignore +++ b/packages/@node-llama-cpp/linux-x64/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/linux-x64/tsconfig.json b/packages/@node-llama-cpp/linux-x64/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/linux-x64/tsconfig.json +++ b/packages/@node-llama-cpp/linux-x64/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/@node-llama-cpp/mac-arm64-metal/.gitignore b/packages/@node-llama-cpp/mac-arm64-metal/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/mac-arm64-metal/.gitignore +++ b/packages/@node-llama-cpp/mac-arm64-metal/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/mac-arm64-metal/tsconfig.json b/packages/@node-llama-cpp/mac-arm64-metal/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/mac-arm64-metal/tsconfig.json +++ b/packages/@node-llama-cpp/mac-arm64-metal/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/@node-llama-cpp/mac-x64/.gitignore b/packages/@node-llama-cpp/mac-x64/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/mac-x64/.gitignore +++ b/packages/@node-llama-cpp/mac-x64/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/mac-x64/tsconfig.json b/packages/@node-llama-cpp/mac-x64/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/mac-x64/tsconfig.json +++ b/packages/@node-llama-cpp/mac-x64/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/@node-llama-cpp/win-arm64/.gitignore b/packages/@node-llama-cpp/win-arm64/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/win-arm64/.gitignore +++ b/packages/@node-llama-cpp/win-arm64/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/win-arm64/tsconfig.json b/packages/@node-llama-cpp/win-arm64/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/win-arm64/tsconfig.json +++ b/packages/@node-llama-cpp/win-arm64/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/@node-llama-cpp/win-x64-cuda-ext/.gitignore b/packages/@node-llama-cpp/win-x64-cuda-ext/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/win-x64-cuda-ext/.gitignore +++ b/packages/@node-llama-cpp/win-x64-cuda-ext/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/win-x64-cuda-ext/tsconfig.json b/packages/@node-llama-cpp/win-x64-cuda-ext/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/win-x64-cuda-ext/tsconfig.json +++ b/packages/@node-llama-cpp/win-x64-cuda-ext/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/@node-llama-cpp/win-x64-cuda/.gitignore b/packages/@node-llama-cpp/win-x64-cuda/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/win-x64-cuda/.gitignore +++ b/packages/@node-llama-cpp/win-x64-cuda/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/win-x64-cuda/tsconfig.json b/packages/@node-llama-cpp/win-x64-cuda/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/win-x64-cuda/tsconfig.json +++ b/packages/@node-llama-cpp/win-x64-cuda/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/@node-llama-cpp/win-x64-vulkan/.gitignore b/packages/@node-llama-cpp/win-x64-vulkan/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/win-x64-vulkan/.gitignore +++ b/packages/@node-llama-cpp/win-x64-vulkan/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/win-x64-vulkan/tsconfig.json b/packages/@node-llama-cpp/win-x64-vulkan/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/win-x64-vulkan/tsconfig.json +++ b/packages/@node-llama-cpp/win-x64-vulkan/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/@node-llama-cpp/win-x64/.gitignore b/packages/@node-llama-cpp/win-x64/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/@node-llama-cpp/win-x64/.gitignore +++ b/packages/@node-llama-cpp/win-x64/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/win-x64/tsconfig.json b/packages/@node-llama-cpp/win-x64/tsconfig.json index 527d791c..c3182e71 100644 --- a/packages/@node-llama-cpp/win-x64/tsconfig.json +++ b/packages/@node-llama-cpp/win-x64/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": false, diff --git a/packages/create-node-llama-cpp/.gitignore b/packages/create-node-llama-cpp/.gitignore index 9b1c8b13..8c780bd4 100644 --- a/packages/create-node-llama-cpp/.gitignore +++ b/packages/create-node-llama-cpp/.gitignore @@ -1 +1,2 @@ /dist +/tsconfig.tsbuildinfo diff --git a/packages/create-node-llama-cpp/tsconfig.json b/packages/create-node-llama-cpp/tsconfig.json index 733d0ff7..dfe959cb 100644 --- a/packages/create-node-llama-cpp/tsconfig.json +++ b/packages/create-node-llama-cpp/tsconfig.json @@ -18,6 +18,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": true, diff --git a/templates/electron-typescript-react/.gitignore b/templates/electron-typescript-react/.gitignore index 047135f3..6cefdc03 100644 --- a/templates/electron-typescript-react/.gitignore +++ b/templates/electron-typescript-react/.gitignore @@ -4,6 +4,7 @@ node_modules .DS_Store /dist +/tsconfig.tsbuildinfo /dist-electron /release /models diff --git a/templates/electron-typescript-react/package.json b/templates/electron-typescript-react/package.json index b16be1a6..d291bb16 100644 --- a/templates/electron-typescript-react/package.json +++ b/templates/electron-typescript-react/package.json @@ -20,7 +20,7 @@ "lint": "npm run lint:eslint", "lint:eslint": "eslint --report-unused-disable-directives .", "format": "npm run lint:eslint -- --fix", - "clean": "rm -rf ./node_modules ./dist ./dist-electron ./release ./models" + "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo ./dist-electron ./release ./models" }, "dependencies": { "@fontsource-variable/inter": "^5.2.8", diff --git a/templates/node-typescript/.gitignore b/templates/node-typescript/.gitignore index 1c72b993..5981b699 100644 --- a/templates/node-typescript/.gitignore +++ b/templates/node-typescript/.gitignore @@ -4,4 +4,5 @@ node_modules .DS_Store /dist +/tsconfig.tsbuildinfo /models diff --git a/templates/node-typescript/tsconfig.json b/templates/node-typescript/tsconfig.json index eddee9ab..d5186a90 100644 --- a/templates/node-typescript/tsconfig.json +++ b/templates/node-typescript/tsconfig.json @@ -20,6 +20,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": true, diff --git a/tsconfig.json b/tsconfig.json index 74531cda..be0436ea 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -20,6 +20,7 @@ "strictNullChecks": true, "isolatedModules": true, "noEmit": false, + "rootDir": "./src", "outDir": "./dist", "strict": true, "sourceMap": true, From bbc87aea368c828eca0d5e75b9b4eee4e1e478d1 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sun, 19 Jul 2026 20:14:57 +0200 Subject: [PATCH 5/7] fix: context shift edge cases --- ...erRemovalCountToFitChatHistoryInContext.ts | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/utils/findCharacterRemovalCountToFitChatHistoryInContext.ts b/src/utils/findCharacterRemovalCountToFitChatHistoryInContext.ts index 27efd9b5..40c371d7 100644 --- a/src/utils/findCharacterRemovalCountToFitChatHistoryInContext.ts +++ b/src/utils/findCharacterRemovalCountToFitChatHistoryInContext.ts @@ -59,6 +59,7 @@ export async function findCharacterRemovalCountToFitChatHistoryInContext({ let latestCompressionAttempt = await getResultForCharacterRemovalCount(initialCharactersRemovalCount); const firstCompressionAttempt = latestCompressionAttempt; + let lastHelpfulCompressionAttempt = latestCompressionAttempt; let latestCompressionAttemptTokensCount = latestCompressionAttempt.tokensCount; let sameTokensCountRepetitions = 0; @@ -75,17 +76,15 @@ export async function findCharacterRemovalCountToFitChatHistoryInContext({ let compressionAttempts = 0, decompressionAttempts = 0; bestCompressionAttempt.tokensCount !== tokensCountToFit; ) { - if (compressionAttempts > 0) { - if (latestCompressionAttempt.tokensCount != firstCompressionAttempt.tokensCount && - latestCompressionAttempt.characterRemovalCount != firstCompressionAttempt.characterRemovalCount - ) - currentEstimatedCharactersPerToken = - Math.abs(latestCompressionAttempt.characterRemovalCount - firstCompressionAttempt.characterRemovalCount) / - Math.abs(latestCompressionAttempt.tokensCount - firstCompressionAttempt.tokensCount); - - if (!Number.isFinite(currentEstimatedCharactersPerToken) || currentEstimatedCharactersPerToken === 0) - currentEstimatedCharactersPerToken = estimatedCharactersPerToken; - } + if (lastHelpfulCompressionAttempt.tokensCount != firstCompressionAttempt.tokensCount && + lastHelpfulCompressionAttempt.characterRemovalCount != firstCompressionAttempt.characterRemovalCount + ) + currentEstimatedCharactersPerToken = + Math.abs(lastHelpfulCompressionAttempt.characterRemovalCount - firstCompressionAttempt.characterRemovalCount) / + Math.abs(lastHelpfulCompressionAttempt.tokensCount - firstCompressionAttempt.tokensCount); + + if (!Number.isFinite(currentEstimatedCharactersPerToken) || currentEstimatedCharactersPerToken === 0) + currentEstimatedCharactersPerToken = estimatedCharactersPerToken; const tokensLeftToRemove = latestCompressionAttempt.tokensCount - tokensCountToFit; let additionalCharactersToRemove = Math.round(tokensLeftToRemove * currentEstimatedCharactersPerToken); @@ -123,6 +122,17 @@ export async function findCharacterRemovalCountToFitChatHistoryInContext({ )) bestCompressionAttempt = latestCompressionAttempt; + const charactersDeltaFromLastHelpfulAttempt = ( + latestCompressionAttempt.characterRemovalCount - lastHelpfulCompressionAttempt.characterRemovalCount + ); + const tokensDeltaFromLastHelpfulAttempt = lastHelpfulCompressionAttempt.tokensCount - latestCompressionAttempt.tokensCount; + const attemptWasHelpful = + (charactersDeltaFromLastHelpfulAttempt > 0 && tokensDeltaFromLastHelpfulAttempt > 0) || + (charactersDeltaFromLastHelpfulAttempt < 0 && tokensDeltaFromLastHelpfulAttempt < 0); + + if (attemptWasHelpful) + lastHelpfulCompressionAttempt = latestCompressionAttempt; + if (latestCompressionAttempt.tokensCount === latestCompressionAttemptTokensCount) sameTokensCountRepetitions++; else { From c8a9e0db6f26703e6035f0a47ce731c6114ad329 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sun, 19 Jul 2026 20:24:55 +0200 Subject: [PATCH 6/7] fix: use stderr for partial logs of at least warn level --- src/bindings/Llama.ts | 44 +++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/bindings/Llama.ts b/src/bindings/Llama.ts index df6cdb45..bd7c5517 100644 --- a/src/bindings/Llama.ts +++ b/src/bindings/Llama.ts @@ -402,15 +402,15 @@ export class Llama { /** * Cap the amount of VRAM that this Llama instance is allowed to use in bytes. * This is useful for constraining the resource usage of models and contexts created with the Llama instance. - * + * * Capping to a value that's too low may cause model loads and context creations to either fail or not fully offload to VRAM, * causing inference to be significantly slower. - * + * * Setting a cap will only affect future model loads and context creations. - * + * * Use with caution. * Setting to `null` disables the cap. - * + * * Defaults to `null`. */ public async setVramCap(bytes: number | null) { @@ -425,7 +425,7 @@ export class Llama { /** * Get the current VRAM cap in bytes. See {@link setVramCap `setVramCap`} for more information. - * + * * Defaults to `null`, which means no cap is set. */ public getVramCap() { @@ -435,21 +435,21 @@ export class Llama { /** * Cap the amount of RAM that this Llama instance is allowed to use in bytes. * This is useful for constraining the resource usage of models and contexts created with the Llama instance. - * + * * Capping to a value that's too low may cause model loads and context creations to fail. * Capping to any value will exclude swap from the resource calculations, * so extremely large models may not load at all even if you have enough swap available. - * + * * Setting a cap will only affect future model loads and context creations. - * + * * On unified memory systems, capping the RAM may also effectively cap the VRAM, as they are shared. * On such systems, it's recommended to either cap the VRAM or the RAM (but not both), * and if you need to cap both then make sure to set the RAM cap to a value greater than the VRAM cap. * > **Note:** You can detect a unified memory system by checking whether `getVramState().unifiedSize` is greater than 0. - * + * * Use with caution. * Setting to `null` disables the cap. - * + * * Defaults to `null`. */ public async setRamCap(bytes: number | null) { @@ -459,14 +459,14 @@ export class Llama { throw new RangeError("RAM cap must be a non-negative number or null"); else if (bytes != null) bytes = Math.floor(bytes); - + this._ramOrchestrator.memoryCap = bytes; this._swapOrchestrator.memoryCap = bytes == null ? null : 0; // if RAM is capped, we can't count on swap for calculation } /** * Get the current RAM cap in bytes. See {@link setRamCap `setRamCap`} for more information. - * + * * Defaults to `null`, which means no cap is set. */ public getRamCap() { @@ -641,15 +641,23 @@ export class Llama { // llama.cpp uses dots to indicate progress, so we don't want to print them as different lines, // and instead, append to the same log line if (logMessageIsOnlyDots(message) && this._logger === Llama.defaultConsoleLogger) { + const stdout = LlamaLogLevelGreaterThanOrEqual(level, LlamaLogLevel.warn) + ? process.stderr + : process.stdout; + if (logMessageIsOnlyDots(this._previousLog) && level === this._previousLogLevel) { - process.stdout.write(message); + stdout.write(message); } else { this._nextLogNeedNewLine = true; - process.stdout.write(prefixAndColorMessage(message, getColorForLogLevel(level))); + stdout.write(prefixAndColorMessage(message, getColorForLogLevel(level))); } } else { if (this._nextLogNeedNewLine) { - process.stdout.write("\n"); + const stdout = LlamaLogLevelGreaterThanOrEqual(this._previousLogLevel ?? level, LlamaLogLevel.warn) + ? process.stderr + : process.stdout; + + stdout.write("\n"); this._nextLogNeedNewLine = false; } @@ -707,7 +715,7 @@ export class Llama { const vramOrchestrator = new MemoryOrchestrator(getBalancedVramState.bind(undefined, bindings, true)); const ramOrchestrator = new MemoryOrchestrator(async () => { const {total, wired} = await getBalancedRamState(bindings); - + return { total, free: total - wired, @@ -810,7 +818,7 @@ export class Llama { default: void (level satisfies never); console.warn(getConsoleLogPrefix() + getColorForLogLevel(LlamaLogLevel.warn)(`Unknown log level: ${level}`)); - console.log(prefixAndColorMessage(message, getColorForLogLevel(level))); + console.warn(prefixAndColorMessage(message, getColorForLogLevel(level))); } } } @@ -899,7 +907,7 @@ async function getBalancedVramState(bindings: BindingModule, balanceUnifiedMemor const nonUnifiedMemoryRam = systemMemoryInfo.total - unifiedSize; const lockedUnifiedVram = Math.max(0, Math.min(systemMemoryInfo.wired ?? 0, systemMemoryInfo.total) - nonUnifiedMemoryRam); - + currentUsed = Math.max(currentUsed, Math.min(lockedUnifiedVram, unifiedSize)); } catch (err) { // do nothing From 4b08e4e51446a1962ed34cde37b3d28af358dfe0 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sun, 19 Jul 2026 20:39:01 +0200 Subject: [PATCH 7/7] fix: docs fuzzy search --- .vitepress/config.ts | 64 +++++++++----------------------------------- 1 file changed, 13 insertions(+), 51 deletions(-) diff --git a/.vitepress/config.ts b/.vitepress/config.ts index 380ad8e9..ade9ae4c 100644 --- a/.vitepress/config.ts +++ b/.vitepress/config.ts @@ -468,65 +468,27 @@ export default defineConfig({ options: { detailedView: true, miniSearch: { - options: { - processTerm(term) { - function splitIdentifier(value: string): string[] { - return value - // dryRun → dry Run - .replace(/([a-z\d])([A-Z])/g, "$1 $2") - // XMLHttpRequest → XML Http Request - .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") - .toLowerCase() - .match(/[\p{L}\p{N}]+/gu) ?? []; - } - - function addBoundedCompounds(parts: readonly string[], maxParts = 3): string[] { - const terms = new Set([parts.join("")]); - - for (let start = 0; start < parts.length; start++) { - let compound = ""; - - for (let end = start; end < parts.length && end < start + maxParts; end++) { - compound += parts[end]; - terms.add(compound); - } - } - - return [...terms]; - } - - return addBoundedCompounds(splitIdentifier(term)); - } - }, searchOptions: { fuzzy: 0.05, - tokenize(query) { - function splitIdentifier(value: string): string[] { - return value - // dryRun → dry Run - .replace(/([a-z\d])([A-Z])/g, "$1 $2") - // XMLHttpRequest → XML Http Request - .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") - .toLowerCase() - .match(/[\p{L}\p{N}]+/gu) ?? []; - } + tokenize(query: string) { + const maxRemovedSpaces = 3; + const words = query.match(/[\p{L}\p{N}]+/gu) ?? []; + const terms = new Set(words); - function addBoundedCompounds(parts: readonly string[], maxParts = 3): string[] { - const terms = new Set([parts.join("")]); + for (let start = 0; start < words.length; start++) { + let joined = words[start]!; - for (let start = 0; start < parts.length; start++) { - let compound = ""; + for (let removedSpaces = 1; removedSpaces <= maxRemovedSpaces; removedSpaces++) { + const nextWord = words[start + removedSpaces]; + if (nextWord == null) + break; - for (let end = start; end < parts.length && end < start + maxParts; end++) { - compound += parts[end]; - terms.add(compound); - } + joined += nextWord; + terms.add(joined); } - - return [...terms]; } - return addBoundedCompounds((query.match(/[\p{L}\p{N}]+/gu) ?? []).flatMap(splitIdentifier)); + return [...terms]; }, boostDocument(term, documentId, storedFields) { const firstTitle = (storedFields?.titles as string[])?.[0];