diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 1545f6f..206836e 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -6,13 +6,13 @@ }, "metadata": { "description": "Cloud-backed persistent shared memory for AI agents powered by Deeplake", - "version": "0.6.46" + "version": "0.7.0" }, "plugins": [ { "name": "hivemind", "description": "Persistent shared memory powered by Deeplake — captures all session activity and provides cross-session, cross-agent memory search", - "version": "0.6.46", + "version": "0.7.0", "source": "./claude-code", "homepage": "https://github.com/activeloopai/hivemind" } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index df7dd60..6a3f654 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "hivemind", "description": "Cloud-backed persistent memory powered by Deeplake — read, write, and share memory across Claude Code sessions and agents", - "version": "0.6.46", + "version": "0.7.0", "author": { "name": "Activeloop", "url": "https://deeplake.ai" diff --git a/claude-code/.claude-plugin/plugin.json b/claude-code/.claude-plugin/plugin.json index df7dd60..6a3f654 100644 --- a/claude-code/.claude-plugin/plugin.json +++ b/claude-code/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "hivemind", "description": "Cloud-backed persistent memory powered by Deeplake — read, write, and share memory across Claude Code sessions and agents", - "version": "0.6.46", + "version": "0.7.0", "author": { "name": "Activeloop", "url": "https://deeplake.ai" diff --git a/claude-code/bundle/capture.js b/claude-code/bundle/capture.js index 7251cf9..9d8c739 100755 --- a/claude-code/bundle/capture.js +++ b/claude-code/bundle/capture.js @@ -370,10 +370,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(tbl)) { log2(`table "${tbl}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', summary_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${tbl}" created`); if (!tables.includes(tbl)) this._tablesCache = [...tables, tbl]; + } else { + try { + await this.query(`ALTER TABLE "${tbl}" ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add summary_embedding skipped: ${e.message}`); + } } } /** Create the sessions table (uses JSONB for message since every row is a JSON event). */ @@ -381,10 +387,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(name)) { log2(`table "${name}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, message_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${name}" created`); if (!tables.includes(name)) this._tablesCache = [...tables, name]; + } else { + try { + await this.query(`ALTER TABLE "${name}" ADD COLUMN IF NOT EXISTS message_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add message_embedding skipped: ${e.message}`); + } } await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`); } @@ -653,8 +665,252 @@ function bundleDirFromImportMeta(importMetaUrl) { return dirname(fileURLToPath(importMetaUrl)); } +// dist/src/embeddings/client.js +import { connect } from "node:net"; +import { spawn as spawn2 } from "node:child_process"; +import { openSync as openSync2, closeSync as closeSync2, writeSync as writeSync2, unlinkSync as unlinkSync2, existsSync as existsSync4, readFileSync as readFileSync4 } from "node:fs"; + +// dist/src/embeddings/protocol.js +var DEFAULT_SOCKET_DIR = "/tmp"; +var DEFAULT_IDLE_TIMEOUT_MS = 15 * 60 * 1e3; +var DEFAULT_CLIENT_TIMEOUT_MS = 200; +function socketPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.sock`; +} +function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.pid`; +} + +// dist/src/embeddings/client.js +var log3 = (m) => log("embed-client", m); +function getUid() { + const uid = typeof process.getuid === "function" ? process.getuid() : void 0; + return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; +} +var EmbedClient = class { + socketPath; + pidPath; + timeoutMs; + daemonEntry; + autoSpawn; + spawnWaitMs; + nextId = 0; + constructor(opts = {}) { + const uid = getUid(); + const dir = opts.socketDir ?? "/tmp"; + this.socketPath = socketPathFor(uid, dir); + this.pidPath = pidPathFor(uid, dir); + this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON; + this.autoSpawn = opts.autoSpawn ?? true; + this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; + } + /** + * Returns an embedding vector, or null on timeout/failure. Hooks MUST treat + * null as "skip embedding column" — never block the write path on us. + * + * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns + * null AND kicks off a background spawn. The next call finds a ready daemon. + */ + async embed(text, kind = "document") { + let sock; + try { + sock = await this.connectOnce(); + } catch { + if (this.autoSpawn) + this.trySpawnDaemon(); + return null; + } + try { + const id = String(++this.nextId); + const req = { op: "embed", id, kind, text }; + const resp = await this.sendAndWait(sock, req); + if (resp.error || !("embedding" in resp) || !resp.embedding) { + log3(`embed err: ${resp.error ?? "no embedding"}`); + return null; + } + return resp.embedding; + } catch (e) { + const err = e instanceof Error ? e.message : String(e); + log3(`embed failed: ${err}`); + return null; + } finally { + try { + sock.end(); + } catch { + } + } + } + /** + * Wait up to spawnWaitMs for the daemon to accept connections, spawning if + * necessary. Meant for SessionStart / long-running batches — not the hot path. + */ + async warmup() { + try { + const s = await this.connectOnce(); + s.end(); + return true; + } catch { + if (!this.autoSpawn) + return false; + this.trySpawnDaemon(); + try { + const s = await this.waitForSocket(); + s.end(); + return true; + } catch { + return false; + } + } + } + connectOnce() { + return new Promise((resolve, reject) => { + const sock = connect(this.socketPath); + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("connect timeout")); + }, this.timeoutMs); + sock.once("connect", () => { + clearTimeout(to); + resolve(sock); + }); + sock.once("error", (e) => { + clearTimeout(to); + reject(e); + }); + }); + } + trySpawnDaemon() { + let fd; + try { + fd = openSync2(this.pidPath, "wx", 384); + writeSync2(fd, String(process.pid)); + } catch (e) { + if (this.isPidFileStale()) { + try { + unlinkSync2(this.pidPath); + } catch { + } + try { + fd = openSync2(this.pidPath, "wx", 384); + writeSync2(fd, String(process.pid)); + } catch { + return; + } + } else { + return; + } + } + if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { + log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + try { + closeSync2(fd); + unlinkSync2(this.pidPath); + } catch { + } + return; + } + try { + const child = spawn2(process.execPath, [this.daemonEntry], { + detached: true, + stdio: "ignore", + env: process.env + }); + child.unref(); + log3(`spawned daemon pid=${child.pid}`); + } finally { + closeSync2(fd); + } + } + isPidFileStale() { + try { + const raw = readFileSync4(this.pidPath, "utf-8").trim(); + const pid = Number(raw); + if (!pid || Number.isNaN(pid)) + return true; + try { + process.kill(pid, 0); + return false; + } catch { + return true; + } + } catch { + return true; + } + } + async waitForSocket() { + const deadline = Date.now() + this.spawnWaitMs; + let delay = 30; + while (Date.now() < deadline) { + await sleep2(delay); + delay = Math.min(delay * 1.5, 300); + if (!existsSync4(this.socketPath)) + continue; + try { + return await this.connectOnce(); + } catch { + } + } + throw new Error("daemon did not become ready within spawnWaitMs"); + } + sendAndWait(sock, req) { + return new Promise((resolve, reject) => { + let buf = ""; + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("request timeout")); + }, this.timeoutMs); + sock.setEncoding("utf-8"); + sock.on("data", (chunk) => { + buf += chunk; + const nl = buf.indexOf("\n"); + if (nl === -1) + return; + const line = buf.slice(0, nl); + clearTimeout(to); + try { + resolve(JSON.parse(line)); + } catch (e) { + reject(e); + } + }); + sock.on("error", (e) => { + clearTimeout(to); + reject(e); + }); + sock.write(JSON.stringify(req) + "\n"); + }); + } +}; +function sleep2(ms) { + return new Promise((r) => setTimeout(r, ms)); +} + +// dist/src/embeddings/sql.js +function embeddingSqlLiteral(vec) { + if (!vec || vec.length === 0) + return "NULL"; + const parts = []; + for (const v of vec) { + if (!Number.isFinite(v)) + return "NULL"; + parts.push(String(v)); + } + return `ARRAY[${parts.join(",")}]::float4[]`; +} + +// dist/src/embeddings/disable.js +function embeddingsDisabled() { + return process.env.HIVEMIND_EMBEDDINGS === "false"; +} + // dist/src/hooks/capture.js -var log3 = (msg) => log("capture", msg); +import { fileURLToPath as fileURLToPath2 } from "node:url"; +import { dirname as dirname2, join as join7 } from "node:path"; +var log4 = (msg) => log("capture", msg); +function resolveEmbedDaemonPath() { + return join7(dirname2(fileURLToPath2(import.meta.url)), "embeddings", "embed-daemon.js"); +} var CAPTURE = process.env.HIVEMIND_CAPTURE !== "false"; async function main() { if (!CAPTURE) @@ -662,7 +918,7 @@ async function main() { const input = await readStdin(); const config = loadConfig(); if (!config) { - log3("no config"); + log4("no config"); return; } const sessionsTable = config.sessionsTableName; @@ -680,7 +936,7 @@ async function main() { }; let entry; if (input.prompt !== void 0) { - log3(`user session=${input.session_id}`); + log4(`user session=${input.session_id}`); entry = { id: crypto.randomUUID(), ...meta, @@ -688,7 +944,7 @@ async function main() { content: input.prompt }; } else if (input.tool_name !== void 0) { - log3(`tool=${input.tool_name} session=${input.session_id}`); + log4(`tool=${input.tool_name} session=${input.session_id}`); entry = { id: crypto.randomUUID(), ...meta, @@ -699,7 +955,7 @@ async function main() { tool_response: JSON.stringify(input.tool_response) }; } else if (input.last_assistant_message !== void 0) { - log3(`assistant session=${input.session_id}`); + log4(`assistant session=${input.session_id}`); entry = { id: crypto.randomUUID(), ...meta, @@ -708,28 +964,30 @@ async function main() { ...input.agent_transcript_path ? { agent_transcript_path: input.agent_transcript_path } : {} }; } else { - log3("unknown event, skipping"); + log4("unknown event, skipping"); return; } const sessionPath = buildSessionPath(config, input.session_id); const line = JSON.stringify(entry); - log3(`writing to ${sessionPath}`); + log4(`writing to ${sessionPath}`); const projectName = (input.cwd ?? "").split("/").pop() || "unknown"; const filename = sessionPath.split("/").pop() ?? ""; const jsonForSql = line.replace(/'/g, "''"); - const insertSql = `INSERT INTO "${sessionsTable}" (id, path, filename, message, author, size_bytes, project, description, agent, creation_date, last_update_date) VALUES ('${crypto.randomUUID()}', '${sqlStr(sessionPath)}', '${sqlStr(filename)}', '${jsonForSql}'::jsonb, '${sqlStr(config.userName)}', ${Buffer.byteLength(line, "utf-8")}, '${sqlStr(projectName)}', '${sqlStr(input.hook_event_name ?? "")}', 'claude_code', '${ts}', '${ts}')`; + const embedding = embeddingsDisabled() ? null : await new EmbedClient({ daemonEntry: resolveEmbedDaemonPath() }).embed(line, "document"); + const embeddingSql = embeddingSqlLiteral(embedding); + const insertSql = `INSERT INTO "${sessionsTable}" (id, path, filename, message, message_embedding, author, size_bytes, project, description, agent, creation_date, last_update_date) VALUES ('${crypto.randomUUID()}', '${sqlStr(sessionPath)}', '${sqlStr(filename)}', '${jsonForSql}'::jsonb, ${embeddingSql}, '${sqlStr(config.userName)}', ${Buffer.byteLength(line, "utf-8")}, '${sqlStr(projectName)}', '${sqlStr(input.hook_event_name ?? "")}', 'claude_code', '${ts}', '${ts}')`; try { await api.query(insertSql); } catch (e) { if (e.message?.includes("permission denied") || e.message?.includes("does not exist")) { - log3("table missing, creating and retrying"); + log4("table missing, creating and retrying"); await api.ensureSessionsTable(sessionsTable); await api.query(insertSql); } else { throw e; } } - log3("capture ok \u2192 cloud"); + log4("capture ok \u2192 cloud"); maybeTriggerPeriodicSummary(input.session_id, input.cwd ?? "", config); } function maybeTriggerPeriodicSummary(sessionId, cwd, config) { @@ -741,7 +999,7 @@ function maybeTriggerPeriodicSummary(sessionId, cwd, config) { if (!shouldTrigger(state, cfg)) return; if (!tryAcquireLock(sessionId)) { - log3(`periodic trigger suppressed (lock held) session=${sessionId}`); + log4(`periodic trigger suppressed (lock held) session=${sessionId}`); return; } wikiLog(`Periodic: threshold hit (total=${state.totalCount}, since=${state.totalCount - state.lastSummaryCount}, N=${cfg.everyNMessages}, hours=${cfg.everyHours})`); @@ -754,19 +1012,19 @@ function maybeTriggerPeriodicSummary(sessionId, cwd, config) { reason: "Periodic" }); } catch (e) { - log3(`periodic spawn failed: ${e.message}`); + log4(`periodic spawn failed: ${e.message}`); try { releaseLock(sessionId); } catch (releaseErr) { - log3(`releaseLock after periodic spawn failure also failed: ${releaseErr.message}`); + log4(`releaseLock after periodic spawn failure also failed: ${releaseErr.message}`); } throw e; } } catch (e) { - log3(`periodic trigger error: ${e.message}`); + log4(`periodic trigger error: ${e.message}`); } } main().catch((e) => { - log3(`fatal: ${e.message}`); + log4(`fatal: ${e.message}`); process.exit(0); }); diff --git a/claude-code/bundle/commands/auth-login.js b/claude-code/bundle/commands/auth-login.js index aa19693..8714067 100755 --- a/claude-code/bundle/commands/auth-login.js +++ b/claude-code/bundle/commands/auth-login.js @@ -551,10 +551,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(tbl)) { log2(`table "${tbl}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', summary_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${tbl}" created`); if (!tables.includes(tbl)) this._tablesCache = [...tables, tbl]; + } else { + try { + await this.query(`ALTER TABLE "${tbl}" ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add summary_embedding skipped: ${e.message}`); + } } } /** Create the sessions table (uses JSONB for message since every row is a JSON event). */ @@ -562,10 +568,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(name)) { log2(`table "${name}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, message_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${name}" created`); if (!tables.includes(name)) this._tablesCache = [...tables, name]; + } else { + try { + await this.query(`ALTER TABLE "${name}" ADD COLUMN IF NOT EXISTS message_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add message_embedding skipped: ${e.message}`); + } } await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`); } diff --git a/claude-code/bundle/embeddings/embed-daemon.js b/claude-code/bundle/embeddings/embed-daemon.js new file mode 100755 index 0000000..cd953b6 --- /dev/null +++ b/claude-code/bundle/embeddings/embed-daemon.js @@ -0,0 +1,240 @@ +#!/usr/bin/env node + +// dist/src/embeddings/daemon.js +import { createServer } from "node:net"; +import { unlinkSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs"; + +// dist/src/embeddings/protocol.js +var DEFAULT_SOCKET_DIR = "/tmp"; +var DEFAULT_MODEL_REPO = "nomic-ai/nomic-embed-text-v1.5"; +var DEFAULT_DTYPE = "q8"; +var DEFAULT_DIMS = 768; +var DEFAULT_IDLE_TIMEOUT_MS = 15 * 60 * 1e3; +var DOC_PREFIX = "search_document: "; +var QUERY_PREFIX = "search_query: "; +function socketPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.sock`; +} +function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.pid`; +} + +// dist/src/embeddings/nomic.js +var NomicEmbedder = class { + pipeline = null; + loading = null; + repo; + dtype; + dims; + constructor(opts = {}) { + this.repo = opts.repo ?? DEFAULT_MODEL_REPO; + this.dtype = opts.dtype ?? DEFAULT_DTYPE; + this.dims = opts.dims ?? DEFAULT_DIMS; + } + async load() { + if (this.pipeline) + return; + if (this.loading) + return this.loading; + this.loading = (async () => { + const mod = await import("@huggingface/transformers"); + mod.env.allowLocalModels = false; + mod.env.useFSCache = true; + this.pipeline = await mod.pipeline("feature-extraction", this.repo, { dtype: this.dtype }); + })(); + try { + await this.loading; + } finally { + this.loading = null; + } + } + addPrefix(text, kind) { + return (kind === "query" ? QUERY_PREFIX : DOC_PREFIX) + text; + } + async embed(text, kind = "document") { + await this.load(); + if (!this.pipeline) + throw new Error("embedder not loaded"); + const out = await this.pipeline(this.addPrefix(text, kind), { pooling: "mean", normalize: true }); + const full = Array.from(out.data); + return this.truncate(full); + } + async embedBatch(texts, kind = "document") { + if (texts.length === 0) + return []; + await this.load(); + if (!this.pipeline) + throw new Error("embedder not loaded"); + const prefixed = texts.map((t) => this.addPrefix(t, kind)); + const out = await this.pipeline(prefixed, { pooling: "mean", normalize: true }); + const flat = Array.from(out.data); + const total = flat.length; + const full = total / texts.length; + const batches = []; + for (let i = 0; i < texts.length; i++) { + batches.push(this.truncate(flat.slice(i * full, (i + 1) * full))); + } + return batches; + } + truncate(vec) { + if (this.dims >= vec.length) + return vec; + const head = vec.slice(0, this.dims); + let norm = 0; + for (const v of head) + norm += v * v; + norm = Math.sqrt(norm); + if (norm === 0) + return head; + for (let i = 0; i < head.length; i++) + head[i] /= norm; + return head; + } +}; + +// dist/src/utils/debug.js +import { appendFileSync } from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; +var DEBUG = process.env.HIVEMIND_DEBUG === "1"; +var LOG = join(homedir(), ".deeplake", "hook-debug.log"); +function log(tag, msg) { + if (!DEBUG) + return; + appendFileSync(LOG, `${(/* @__PURE__ */ new Date()).toISOString()} [${tag}] ${msg} +`); +} + +// dist/src/embeddings/daemon.js +var log2 = (m) => log("embed-daemon", m); +function getUid() { + const uid = typeof process.getuid === "function" ? process.getuid() : void 0; + return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; +} +var EmbedDaemon = class { + server = null; + embedder; + socketPath; + pidPath; + idleTimeoutMs; + idleTimer = null; + constructor(opts = {}) { + const uid = getUid(); + const dir = opts.socketDir ?? "/tmp"; + this.socketPath = socketPathFor(uid, dir); + this.pidPath = pidPathFor(uid, dir); + this.idleTimeoutMs = opts.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS; + this.embedder = new NomicEmbedder({ repo: opts.repo, dtype: opts.dtype, dims: opts.dims }); + } + async start() { + mkdirSync(this.socketPath.replace(/\/[^/]+$/, ""), { recursive: true }); + writeFileSync(this.pidPath, String(process.pid), { mode: 384 }); + if (existsSync(this.socketPath)) { + try { + unlinkSync(this.socketPath); + } catch { + } + } + this.embedder.load().then(() => log2("model ready")).catch((e) => log2(`load err: ${e.message}`)); + this.server = createServer((sock) => this.handleConnection(sock)); + await new Promise((resolve, reject) => { + this.server.once("error", reject); + this.server.listen(this.socketPath, () => { + try { + chmodSync(this.socketPath, 384); + } catch { + } + log2(`listening on ${this.socketPath}`); + resolve(); + }); + }); + this.resetIdleTimer(); + process.on("SIGINT", () => this.shutdown()); + process.on("SIGTERM", () => this.shutdown()); + } + resetIdleTimer() { + if (this.idleTimer) + clearTimeout(this.idleTimer); + this.idleTimer = setTimeout(() => { + log2(`idle timeout ${this.idleTimeoutMs}ms reached, shutting down`); + this.shutdown(); + }, this.idleTimeoutMs); + this.idleTimer.unref(); + } + shutdown() { + try { + this.server?.close(); + } catch { + } + try { + if (existsSync(this.socketPath)) + unlinkSync(this.socketPath); + } catch { + } + try { + if (existsSync(this.pidPath)) + unlinkSync(this.pidPath); + } catch { + } + process.exit(0); + } + handleConnection(sock) { + let buf = ""; + sock.setEncoding("utf-8"); + sock.on("data", (chunk) => { + buf += chunk; + let nl; + while ((nl = buf.indexOf("\n")) !== -1) { + const line = buf.slice(0, nl); + buf = buf.slice(nl + 1); + if (line.length === 0) + continue; + this.handleLine(sock, line); + } + }); + sock.on("error", () => { + }); + } + async handleLine(sock, line) { + this.resetIdleTimer(); + let req; + try { + req = JSON.parse(line); + } catch { + return; + } + try { + const resp = await this.dispatch(req); + sock.write(JSON.stringify(resp) + "\n"); + } catch (e) { + const err = e instanceof Error ? e.message : String(e); + const resp = { id: req.id, error: err }; + sock.write(JSON.stringify(resp) + "\n"); + } + } + async dispatch(req) { + if (req.op === "ping") { + const p = req; + return { id: p.id, ready: true, model: this.embedder.repo, dims: this.embedder.dims }; + } + if (req.op === "embed") { + const e = req; + const vec = await this.embedder.embed(e.text, e.kind); + return { id: e.id, embedding: vec }; + } + return { id: req.id, error: "unknown op" }; + } +}; +var invokedDirectly = import.meta.url === `file://${process.argv[1]}` || process.argv[1] && import.meta.url.endsWith(process.argv[1].split("/").pop() ?? ""); +if (invokedDirectly) { + const dims = process.env.HIVEMIND_EMBED_DIMS ? Number(process.env.HIVEMIND_EMBED_DIMS) : void 0; + const idleTimeoutMs = process.env.HIVEMIND_EMBED_IDLE_MS ? Number(process.env.HIVEMIND_EMBED_IDLE_MS) : void 0; + const d = new EmbedDaemon({ dims, idleTimeoutMs }); + d.start().catch((e) => { + log2(`fatal: ${e.message}`); + process.exit(1); + }); +} +export { + EmbedDaemon +}; diff --git a/claude-code/bundle/pre-tool-use.js b/claude-code/bundle/pre-tool-use.js index 94a5d72..4daedfa 100755 --- a/claude-code/bundle/pre-tool-use.js +++ b/claude-code/bundle/pre-tool-use.js @@ -1,10 +1,10 @@ #!/usr/bin/env node // dist/src/hooks/pre-tool-use.js -import { existsSync as existsSync3, mkdirSync as mkdirSync3, writeFileSync as writeFileSync3 } from "node:fs"; +import { existsSync as existsSync4, mkdirSync as mkdirSync3, writeFileSync as writeFileSync3 } from "node:fs"; import { homedir as homedir5 } from "node:os"; -import { join as join6, dirname, sep } from "node:path"; -import { fileURLToPath as fileURLToPath2 } from "node:url"; +import { join as join7, dirname as dirname2, sep } from "node:path"; +import { fileURLToPath as fileURLToPath3 } from "node:url"; // dist/src/utils/stdin.js function readStdin() { @@ -376,10 +376,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(tbl)) { log2(`table "${tbl}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', summary_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${tbl}" created`); if (!tables.includes(tbl)) this._tablesCache = [...tables, tbl]; + } else { + try { + await this.query(`ALTER TABLE "${tbl}" ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add summary_embedding skipped: ${e.message}`); + } } } /** Create the sessions table (uses JSONB for message since every row is a JSON event). */ @@ -387,10 +393,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(name)) { log2(`table "${name}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, message_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${name}" created`); if (!tables.includes(name)) this._tablesCache = [...tables, name]; + } else { + try { + await this.query(`ALTER TABLE "${name}" ADD COLUMN IF NOT EXISTS message_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add message_embedding skipped: ${e.message}`); + } } await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`); } @@ -574,24 +586,25 @@ function normalizeContent(path, raw) { return raw; } if (Array.isArray(obj.turns)) { - const header = []; - if (obj.date_time) - header.push(`date: ${obj.date_time}`); - if (obj.speakers) { - const s = obj.speakers; - const names = [s.speaker_a, s.speaker_b].filter(Boolean).join(", "); - if (names) - header.push(`speakers: ${names}`); - } + const dateHeader = obj.date_time ? `(${String(obj.date_time)}) ` : ""; const lines = obj.turns.map((t) => { const sp = String(t?.speaker ?? t?.name ?? "?").trim(); const tx = String(t?.text ?? t?.content ?? "").replace(/\s+/g, " ").trim(); const tag = t?.dia_id ? `[${t.dia_id}] ` : ""; - return `${tag}${sp}: ${tx}`; + return `${dateHeader}${tag}${sp}: ${tx}`; }); - const out2 = [...header, ...lines].join("\n"); + const out2 = lines.join("\n"); return out2.trim() ? out2 : raw; } + if (obj.turn && typeof obj.turn === "object" && !Array.isArray(obj.turn)) { + const t = obj.turn; + const sp = String(t.speaker ?? t.name ?? "?").trim(); + const tx = String(t.text ?? t.content ?? "").replace(/\s+/g, " ").trim(); + const tag = t.dia_id ? `[${String(t.dia_id)}] ` : ""; + const dateHeader = obj.date_time ? `(${String(obj.date_time)}) ` : ""; + const line = `${dateHeader}${tag}${sp}: ${tx}`; + return line.trim() ? line : raw; + } const stripRecalled = (t) => { const i = t.indexOf(""); if (i === -1) @@ -634,8 +647,38 @@ function buildPathCondition(targetPath) { return `(path = '${sqlStr(clean)}' OR path LIKE '${sqlLike(clean)}/%' ESCAPE '\\')`; } async function searchDeeplakeTables(api, memoryTable, sessionsTable, opts) { - const { pathFilter, contentScanOnly, likeOp, escapedPattern, prefilterPattern, prefilterPatterns, multiWordPatterns } = opts; + const { pathFilter, contentScanOnly, likeOp, escapedPattern, prefilterPattern, prefilterPatterns, queryEmbedding, bm25Term, multiWordPatterns } = opts; const limit = opts.limit ?? 100; + if (queryEmbedding && queryEmbedding.length > 0) { + const vecLit = serializeFloat4Array(queryEmbedding); + const semanticLimit = Math.min(limit, Number(process.env.HIVEMIND_SEMANTIC_LIMIT ?? "20")); + const lexicalLimit = Math.min(limit, Number(process.env.HIVEMIND_HYBRID_LEXICAL_LIMIT ?? "20")); + const filterPatternsForLex = contentScanOnly ? prefilterPatterns && prefilterPatterns.length > 0 ? prefilterPatterns : prefilterPattern ? [prefilterPattern] : [] : [escapedPattern]; + const memLexFilter = buildContentFilter("summary::text", likeOp, filterPatternsForLex); + const sessLexFilter = buildContentFilter("message::text", likeOp, filterPatternsForLex); + const memLexQuery = memLexFilter ? `SELECT path, summary::text AS content, 0 AS source_order, '' AS creation_date, 1.0 AS score FROM "${memoryTable}" WHERE 1=1${pathFilter}${memLexFilter} LIMIT ${lexicalLimit}` : null; + const sessLexQuery = sessLexFilter ? `SELECT path, message::text AS content, 1 AS source_order, COALESCE(creation_date::text, '') AS creation_date, 1.0 AS score FROM "${sessionsTable}" WHERE 1=1${pathFilter}${sessLexFilter} LIMIT ${lexicalLimit}` : null; + const memSemQuery = `SELECT path, summary::text AS content, 0 AS source_order, '' AS creation_date, (summary_embedding <#> ${vecLit}) AS score FROM "${memoryTable}" WHERE summary_embedding IS NOT NULL${pathFilter} ORDER BY score DESC LIMIT ${semanticLimit}`; + const sessSemQuery = `SELECT path, message::text AS content, 1 AS source_order, COALESCE(creation_date::text, '') AS creation_date, (message_embedding <#> ${vecLit}) AS score FROM "${sessionsTable}" WHERE message_embedding IS NOT NULL${pathFilter} ORDER BY score DESC LIMIT ${semanticLimit}`; + const parts = [memSemQuery, sessSemQuery]; + if (memLexQuery) + parts.push(memLexQuery); + if (sessLexQuery) + parts.push(sessLexQuery); + const unionSql = parts.map((q) => `(${q})`).join(" UNION ALL "); + const outerLimit = semanticLimit + lexicalLimit; + const rows2 = await api.query(`SELECT path, content, source_order, creation_date, score FROM (` + unionSql + `) AS combined ORDER BY score DESC LIMIT ${outerLimit}`); + const seen = /* @__PURE__ */ new Set(); + const unique = []; + for (const row of rows2) { + const p = String(row["path"]); + if (seen.has(p)) + continue; + seen.add(p); + unique.push({ path: p, content: String(row["content"] ?? "") }); + } + return unique; + } const filterPatterns = contentScanOnly ? prefilterPatterns && prefilterPatterns.length > 0 ? prefilterPatterns : prefilterPattern ? [prefilterPattern] : [] : multiWordPatterns && multiWordPatterns.length > 1 ? multiWordPatterns : [escapedPattern]; const memFilter = buildContentFilter("summary::text", likeOp, filterPatterns); const sessFilter = buildContentFilter("message::text", likeOp, filterPatterns); @@ -647,6 +690,15 @@ async function searchDeeplakeTables(api, memoryTable, sessionsTable, opts) { content: String(row["content"] ?? "") })); } +function serializeFloat4Array(vec) { + const parts = []; + for (const v of vec) { + if (!Number.isFinite(v)) + return "NULL"; + parts.push(String(v)); + } + return `ARRAY[${parts.join(",")}]::float4[]`; +} function buildPathFilter(targetPath) { const condition = buildPathCondition(targetPath); return condition ? ` AND ${condition}` : ""; @@ -725,14 +777,25 @@ function buildGrepSearchOptions(params, targetPath) { const hasRegexMeta = !params.fixedString && /[.*+?^${}()|[\]\\]/.test(params.pattern); const literalPrefilter = hasRegexMeta ? extractRegexLiteralPrefilter(params.pattern) : null; const alternationPrefilters = hasRegexMeta ? extractRegexAlternationPrefilters(params.pattern) : null; + let bm25Term; + if (!hasRegexMeta) { + bm25Term = params.pattern; + } else if (alternationPrefilters && alternationPrefilters.length > 0) { + bm25Term = alternationPrefilters.join(" "); + } else if (literalPrefilter) { + bm25Term = literalPrefilter; + } const multiWordPatterns = !hasRegexMeta ? params.pattern.split(/\s+/).filter((w) => w.length > 2).slice(0, 4) : []; return { pathFilter: buildPathFilter(targetPath), contentScanOnly: hasRegexMeta, - likeOp: params.ignoreCase ? "ILIKE" : "LIKE", + // Kept for the pure-lexical fallback path and existing tests; BM25 is now + // the preferred lexical ranker when a term can be extracted. + likeOp: process.env.HIVEMIND_GREP_LIKE === "case-sensitive" ? "LIKE" : "ILIKE", escapedPattern: sqlLike(params.pattern), prefilterPattern: literalPrefilter ? sqlLike(literalPrefilter) : void 0, prefilterPatterns: alternationPrefilters?.map((literal) => sqlLike(literal)), + bm25Term, multiWordPatterns: multiWordPatterns.length > 1 ? multiWordPatterns.map((w) => sqlLike(w)) : void 0 }; } @@ -784,11 +847,28 @@ function refineGrepMatches(rows, params, forceMultiFilePrefix) { } return output; } -async function grepBothTables(api, memoryTable, sessionsTable, params, targetPath) { - const rows = await searchDeeplakeTables(api, memoryTable, sessionsTable, buildGrepSearchOptions(params, targetPath)); +async function grepBothTables(api, memoryTable, sessionsTable, params, targetPath, queryEmbedding) { + const rows = await searchDeeplakeTables(api, memoryTable, sessionsTable, { + ...buildGrepSearchOptions(params, targetPath), + queryEmbedding + }); const seen = /* @__PURE__ */ new Set(); const unique = rows.filter((r) => seen.has(r.path) ? false : (seen.add(r.path), true)); const normalized = unique.map((r) => ({ path: r.path, content: normalizeContent(r.path, r.content) })); + if (queryEmbedding && queryEmbedding.length > 0) { + const emitAllLines = process.env.HIVEMIND_SEMANTIC_EMIT_ALL !== "false"; + if (emitAllLines) { + const lines = []; + for (const r of normalized) { + for (const line of r.content.split("\n")) { + const trimmed = line.trim(); + if (trimmed) + lines.push(`${r.path}:${line}`); + } + } + return lines; + } + } return refineGrepMatches(normalized, params); } @@ -832,7 +912,260 @@ function capOutputForClaude(output, options = {}) { return keptLines.join("\n") + footer; } +// dist/src/embeddings/client.js +import { connect } from "node:net"; +import { spawn } from "node:child_process"; +import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync3, readFileSync as readFileSync3 } from "node:fs"; + +// dist/src/embeddings/protocol.js +var DEFAULT_SOCKET_DIR = "/tmp"; +var DEFAULT_IDLE_TIMEOUT_MS = 15 * 60 * 1e3; +var DEFAULT_CLIENT_TIMEOUT_MS = 200; +function socketPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.sock`; +} +function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.pid`; +} + +// dist/src/embeddings/client.js +var log3 = (m) => log("embed-client", m); +function getUid() { + const uid = typeof process.getuid === "function" ? process.getuid() : void 0; + return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; +} +var EmbedClient = class { + socketPath; + pidPath; + timeoutMs; + daemonEntry; + autoSpawn; + spawnWaitMs; + nextId = 0; + constructor(opts = {}) { + const uid = getUid(); + const dir = opts.socketDir ?? "/tmp"; + this.socketPath = socketPathFor(uid, dir); + this.pidPath = pidPathFor(uid, dir); + this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON; + this.autoSpawn = opts.autoSpawn ?? true; + this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; + } + /** + * Returns an embedding vector, or null on timeout/failure. Hooks MUST treat + * null as "skip embedding column" — never block the write path on us. + * + * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns + * null AND kicks off a background spawn. The next call finds a ready daemon. + */ + async embed(text, kind = "document") { + let sock; + try { + sock = await this.connectOnce(); + } catch { + if (this.autoSpawn) + this.trySpawnDaemon(); + return null; + } + try { + const id = String(++this.nextId); + const req = { op: "embed", id, kind, text }; + const resp = await this.sendAndWait(sock, req); + if (resp.error || !("embedding" in resp) || !resp.embedding) { + log3(`embed err: ${resp.error ?? "no embedding"}`); + return null; + } + return resp.embedding; + } catch (e) { + const err = e instanceof Error ? e.message : String(e); + log3(`embed failed: ${err}`); + return null; + } finally { + try { + sock.end(); + } catch { + } + } + } + /** + * Wait up to spawnWaitMs for the daemon to accept connections, spawning if + * necessary. Meant for SessionStart / long-running batches — not the hot path. + */ + async warmup() { + try { + const s = await this.connectOnce(); + s.end(); + return true; + } catch { + if (!this.autoSpawn) + return false; + this.trySpawnDaemon(); + try { + const s = await this.waitForSocket(); + s.end(); + return true; + } catch { + return false; + } + } + } + connectOnce() { + return new Promise((resolve2, reject) => { + const sock = connect(this.socketPath); + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("connect timeout")); + }, this.timeoutMs); + sock.once("connect", () => { + clearTimeout(to); + resolve2(sock); + }); + sock.once("error", (e) => { + clearTimeout(to); + reject(e); + }); + }); + } + trySpawnDaemon() { + let fd; + try { + fd = openSync(this.pidPath, "wx", 384); + writeSync(fd, String(process.pid)); + } catch (e) { + if (this.isPidFileStale()) { + try { + unlinkSync(this.pidPath); + } catch { + } + try { + fd = openSync(this.pidPath, "wx", 384); + writeSync(fd, String(process.pid)); + } catch { + return; + } + } else { + return; + } + } + if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { + log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + try { + closeSync(fd); + unlinkSync(this.pidPath); + } catch { + } + return; + } + try { + const child = spawn(process.execPath, [this.daemonEntry], { + detached: true, + stdio: "ignore", + env: process.env + }); + child.unref(); + log3(`spawned daemon pid=${child.pid}`); + } finally { + closeSync(fd); + } + } + isPidFileStale() { + try { + const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const pid = Number(raw); + if (!pid || Number.isNaN(pid)) + return true; + try { + process.kill(pid, 0); + return false; + } catch { + return true; + } + } catch { + return true; + } + } + async waitForSocket() { + const deadline = Date.now() + this.spawnWaitMs; + let delay = 30; + while (Date.now() < deadline) { + await sleep2(delay); + delay = Math.min(delay * 1.5, 300); + if (!existsSync3(this.socketPath)) + continue; + try { + return await this.connectOnce(); + } catch { + } + } + throw new Error("daemon did not become ready within spawnWaitMs"); + } + sendAndWait(sock, req) { + return new Promise((resolve2, reject) => { + let buf = ""; + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("request timeout")); + }, this.timeoutMs); + sock.setEncoding("utf-8"); + sock.on("data", (chunk) => { + buf += chunk; + const nl = buf.indexOf("\n"); + if (nl === -1) + return; + const line = buf.slice(0, nl); + clearTimeout(to); + try { + resolve2(JSON.parse(line)); + } catch (e) { + reject(e); + } + }); + sock.on("error", (e) => { + clearTimeout(to); + reject(e); + }); + sock.write(JSON.stringify(req) + "\n"); + }); + } +}; +function sleep2(ms) { + return new Promise((r) => setTimeout(r, ms)); +} + +// dist/src/embeddings/disable.js +function embeddingsDisabled() { + return process.env.HIVEMIND_EMBEDDINGS === "false"; +} + // dist/src/hooks/grep-direct.js +import { fileURLToPath as fileURLToPath2 } from "node:url"; +import { dirname, join as join4 } from "node:path"; +var SEMANTIC_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); +var SEMANTIC_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); +function resolveDaemonPath() { + return join4(dirname(fileURLToPath2(import.meta.url)), "..", "embeddings", "embed-daemon.js"); +} +var sharedEmbedClient = null; +function getEmbedClient() { + if (!sharedEmbedClient) { + sharedEmbedClient = new EmbedClient({ + daemonEntry: resolveDaemonPath(), + timeoutMs: SEMANTIC_TIMEOUT_MS + }); + } + return sharedEmbedClient; +} +function patternIsSemanticFriendly(pattern, fixedString) { + if (!pattern || pattern.length < 2) + return false; + if (fixedString) + return true; + const meta = pattern.match(/[|()\[\]{}+?^$\\]/g); + if (!meta) + return true; + return meta.length <= 1; +} function splitFirstPipelineStage(cmd) { const input = cmd.trim(); let quote = null; @@ -1070,7 +1403,15 @@ async function handleGrepDirect(api, table, sessionsTable, params) { invertMatch: params.invertMatch, fixedString: params.fixedString }; - const output = await grepBothTables(api, table, sessionsTable, matchParams, params.targetPath); + let queryEmbedding = null; + if (SEMANTIC_ENABLED && patternIsSemanticFriendly(params.pattern, params.fixedString)) { + try { + queryEmbedding = await getEmbedClient().embed(params.pattern, "query"); + } catch { + queryEmbedding = null; + } + } + const output = await grepBothTables(api, table, sessionsTable, matchParams, params.targetPath, queryEmbedding); const joined = output.join("\n") || "(no matches)"; return capOutputForClaude(joined, { kind: "grep" }); } @@ -1691,20 +2032,20 @@ async function executeCompiledBashCommand(api, memoryTable, sessionsTable, cmd, } // dist/src/hooks/query-cache.js -import { mkdirSync as mkdirSync2, readFileSync as readFileSync3, rmSync, writeFileSync as writeFileSync2 } from "node:fs"; -import { join as join4 } from "node:path"; +import { mkdirSync as mkdirSync2, readFileSync as readFileSync4, rmSync, writeFileSync as writeFileSync2 } from "node:fs"; +import { join as join5 } from "node:path"; import { homedir as homedir3 } from "node:os"; -var log3 = (msg) => log("query-cache", msg); -var DEFAULT_CACHE_ROOT = join4(homedir3(), ".deeplake", "query-cache"); +var log4 = (msg) => log("query-cache", msg); +var DEFAULT_CACHE_ROOT = join5(homedir3(), ".deeplake", "query-cache"); var INDEX_CACHE_FILE = "index.md"; function getSessionQueryCacheDir(sessionId, deps = {}) { const { cacheRoot = DEFAULT_CACHE_ROOT } = deps; - return join4(cacheRoot, sessionId); + return join5(cacheRoot, sessionId); } function readCachedIndexContent(sessionId, deps = {}) { - const { logFn = log3 } = deps; + const { logFn = log4 } = deps; try { - return readFileSync3(join4(getSessionQueryCacheDir(sessionId, deps), INDEX_CACHE_FILE), "utf-8"); + return readFileSync4(join5(getSessionQueryCacheDir(sessionId, deps), INDEX_CACHE_FILE), "utf-8"); } catch (e) { if (e?.code === "ENOENT") return null; @@ -1713,11 +2054,11 @@ function readCachedIndexContent(sessionId, deps = {}) { } } function writeCachedIndexContent(sessionId, content, deps = {}) { - const { logFn = log3 } = deps; + const { logFn = log4 } = deps; try { const dir = getSessionQueryCacheDir(sessionId, deps); mkdirSync2(dir, { recursive: true }); - writeFileSync2(join4(dir, INDEX_CACHE_FILE), content, "utf-8"); + writeFileSync2(join5(dir, INDEX_CACHE_FILE), content, "utf-8"); } catch (e) { logFn(`write failed for session=${sessionId}: ${e.message}`); } @@ -1725,8 +2066,8 @@ function writeCachedIndexContent(sessionId, content, deps = {}) { // dist/src/hooks/memory-path-utils.js import { homedir as homedir4 } from "node:os"; -import { join as join5 } from "node:path"; -var MEMORY_PATH = join5(homedir4(), ".deeplake", "memory"); +import { join as join6 } from "node:path"; +var MEMORY_PATH = join6(homedir4(), ".deeplake", "memory"); var TILDE_PATH = "~/.deeplake/memory"; var HOME_VAR_PATH = "$HOME/.deeplake/memory"; var SAFE_BUILTINS = /* @__PURE__ */ new Set([ @@ -1842,20 +2183,20 @@ function rewritePaths(cmd) { } // dist/src/hooks/pre-tool-use.js -var log4 = (msg) => log("pre", msg); -var __bundleDir = dirname(fileURLToPath2(import.meta.url)); -var SHELL_BUNDLE = existsSync3(join6(__bundleDir, "shell", "deeplake-shell.js")) ? join6(__bundleDir, "shell", "deeplake-shell.js") : join6(__bundleDir, "..", "shell", "deeplake-shell.js"); -var READ_CACHE_ROOT = join6(homedir5(), ".deeplake", "query-cache"); +var log5 = (msg) => log("pre", msg); +var __bundleDir = dirname2(fileURLToPath3(import.meta.url)); +var SHELL_BUNDLE = existsSync4(join7(__bundleDir, "shell", "deeplake-shell.js")) ? join7(__bundleDir, "shell", "deeplake-shell.js") : join7(__bundleDir, "..", "shell", "deeplake-shell.js"); +var READ_CACHE_ROOT = join7(homedir5(), ".deeplake", "query-cache"); function writeReadCacheFile(sessionId, virtualPath, content, deps = {}) { const { cacheRoot = READ_CACHE_ROOT } = deps; const safeSessionId = sessionId.replace(/[^a-zA-Z0-9._-]/g, "_") || "unknown"; const rel = virtualPath.replace(/^\/+/, "") || "content"; - const expectedRoot = join6(cacheRoot, safeSessionId, "read"); - const absPath = join6(expectedRoot, rel); + const expectedRoot = join7(cacheRoot, safeSessionId, "read"); + const absPath = join7(expectedRoot, rel); if (absPath !== expectedRoot && !absPath.startsWith(expectedRoot + sep)) { throw new Error(`writeReadCacheFile: path escapes cache root: ${absPath}`); } - mkdirSync3(dirname(absPath), { recursive: true }); + mkdirSync3(dirname2(absPath), { recursive: true }); writeFileSync3(absPath, content, "utf-8"); return absPath; } @@ -1902,7 +2243,7 @@ function getShellCommand(toolName, toolInput) { break; const rewritten = rewritePaths(cmd); if (!isSafe(rewritten)) { - log4(`unsafe command blocked: ${rewritten}`); + log5(`unsafe command blocked: ${rewritten}`); return null; } return rewritten; @@ -1942,7 +2283,7 @@ function buildFallbackDecision(shellCmd, shellBundle = SHELL_BUNDLE) { return buildAllowDecision(`node "${shellBundle}" -c "${shellCmd.replace(/"/g, '\\"')}"`, `[DeepLake shell] ${shellCmd}`); } async function processPreToolUse(input, deps = {}) { - const { config = loadConfig(), createApi = (table2, activeConfig) => new DeeplakeApi(activeConfig.token, activeConfig.apiUrl, activeConfig.orgId, activeConfig.workspaceId, table2), executeCompiledBashCommandFn = executeCompiledBashCommand, handleGrepDirectFn = handleGrepDirect, readVirtualPathContentsFn = readVirtualPathContents, readVirtualPathContentFn = readVirtualPathContent, listVirtualPathRowsFn = listVirtualPathRows, findVirtualPathsFn = findVirtualPaths, readCachedIndexContentFn = readCachedIndexContent, writeCachedIndexContentFn = writeCachedIndexContent, writeReadCacheFileFn = writeReadCacheFile, shellBundle = SHELL_BUNDLE, logFn = log4 } = deps; + const { config = loadConfig(), createApi = (table2, activeConfig) => new DeeplakeApi(activeConfig.token, activeConfig.apiUrl, activeConfig.orgId, activeConfig.workspaceId, table2), executeCompiledBashCommandFn = executeCompiledBashCommand, handleGrepDirectFn = handleGrepDirect, readVirtualPathContentsFn = readVirtualPathContents, readVirtualPathContentFn = readVirtualPathContent, listVirtualPathRowsFn = listVirtualPathRows, findVirtualPathsFn = findVirtualPaths, readCachedIndexContentFn = readCachedIndexContent, writeCachedIndexContentFn = writeCachedIndexContent, writeReadCacheFileFn = writeReadCacheFile, shellBundle = SHELL_BUNDLE, logFn = log5 } = deps; const cmd = input.tool_input.command ?? ""; const shellCmd = getShellCommand(input.tool_name, input.tool_input); const toolPath = getReadTargetPath(input.tool_input) ?? input.tool_input.path ?? ""; @@ -2165,7 +2506,7 @@ async function main() { } if (isDirectRun(import.meta.url)) { main().catch((e) => { - log4(`fatal: ${e.message}`); + log5(`fatal: ${e.message}`); process.exit(0); }); } diff --git a/claude-code/bundle/session-start-setup.js b/claude-code/bundle/session-start-setup.js index 5eff08e..622ddf7 100755 --- a/claude-code/bundle/session-start-setup.js +++ b/claude-code/bundle/session-start-setup.js @@ -381,10 +381,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(tbl)) { log2(`table "${tbl}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', summary_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${tbl}" created`); if (!tables.includes(tbl)) this._tablesCache = [...tables, tbl]; + } else { + try { + await this.query(`ALTER TABLE "${tbl}" ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add summary_embedding skipped: ${e.message}`); + } } } /** Create the sessions table (uses JSONB for message since every row is a JSON event). */ @@ -392,10 +398,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(name)) { log2(`table "${name}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, message_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${name}" created`); if (!tables.includes(name)) this._tablesCache = [...tables, name]; + } else { + try { + await this.query(`ALTER TABLE "${name}" ADD COLUMN IF NOT EXISTS message_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add message_embedding skipped: ${e.message}`); + } } await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`); } @@ -543,8 +555,234 @@ function restoreOrCleanup(handle) { } var DEFAULT_MANIFEST_PATH = join7(homedir4(), ".claude", "plugins", "installed_plugins.json"); +// dist/src/embeddings/client.js +import { connect } from "node:net"; +import { spawn } from "node:child_process"; +import { openSync, closeSync, writeSync, unlinkSync as unlinkSync2, existsSync as existsSync5, readFileSync as readFileSync6 } from "node:fs"; + +// dist/src/embeddings/protocol.js +var DEFAULT_SOCKET_DIR = "/tmp"; +var DEFAULT_IDLE_TIMEOUT_MS = 15 * 60 * 1e3; +var DEFAULT_CLIENT_TIMEOUT_MS = 200; +function socketPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.sock`; +} +function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.pid`; +} + +// dist/src/embeddings/client.js +var log3 = (m) => log("embed-client", m); +function getUid() { + const uid = typeof process.getuid === "function" ? process.getuid() : void 0; + return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; +} +var EmbedClient = class { + socketPath; + pidPath; + timeoutMs; + daemonEntry; + autoSpawn; + spawnWaitMs; + nextId = 0; + constructor(opts = {}) { + const uid = getUid(); + const dir = opts.socketDir ?? "/tmp"; + this.socketPath = socketPathFor(uid, dir); + this.pidPath = pidPathFor(uid, dir); + this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON; + this.autoSpawn = opts.autoSpawn ?? true; + this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; + } + /** + * Returns an embedding vector, or null on timeout/failure. Hooks MUST treat + * null as "skip embedding column" — never block the write path on us. + * + * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns + * null AND kicks off a background spawn. The next call finds a ready daemon. + */ + async embed(text, kind = "document") { + let sock; + try { + sock = await this.connectOnce(); + } catch { + if (this.autoSpawn) + this.trySpawnDaemon(); + return null; + } + try { + const id = String(++this.nextId); + const req = { op: "embed", id, kind, text }; + const resp = await this.sendAndWait(sock, req); + if (resp.error || !("embedding" in resp) || !resp.embedding) { + log3(`embed err: ${resp.error ?? "no embedding"}`); + return null; + } + return resp.embedding; + } catch (e) { + const err = e instanceof Error ? e.message : String(e); + log3(`embed failed: ${err}`); + return null; + } finally { + try { + sock.end(); + } catch { + } + } + } + /** + * Wait up to spawnWaitMs for the daemon to accept connections, spawning if + * necessary. Meant for SessionStart / long-running batches — not the hot path. + */ + async warmup() { + try { + const s = await this.connectOnce(); + s.end(); + return true; + } catch { + if (!this.autoSpawn) + return false; + this.trySpawnDaemon(); + try { + const s = await this.waitForSocket(); + s.end(); + return true; + } catch { + return false; + } + } + } + connectOnce() { + return new Promise((resolve2, reject) => { + const sock = connect(this.socketPath); + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("connect timeout")); + }, this.timeoutMs); + sock.once("connect", () => { + clearTimeout(to); + resolve2(sock); + }); + sock.once("error", (e) => { + clearTimeout(to); + reject(e); + }); + }); + } + trySpawnDaemon() { + let fd; + try { + fd = openSync(this.pidPath, "wx", 384); + writeSync(fd, String(process.pid)); + } catch (e) { + if (this.isPidFileStale()) { + try { + unlinkSync2(this.pidPath); + } catch { + } + try { + fd = openSync(this.pidPath, "wx", 384); + writeSync(fd, String(process.pid)); + } catch { + return; + } + } else { + return; + } + } + if (!this.daemonEntry || !existsSync5(this.daemonEntry)) { + log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + try { + closeSync(fd); + unlinkSync2(this.pidPath); + } catch { + } + return; + } + try { + const child = spawn(process.execPath, [this.daemonEntry], { + detached: true, + stdio: "ignore", + env: process.env + }); + child.unref(); + log3(`spawned daemon pid=${child.pid}`); + } finally { + closeSync(fd); + } + } + isPidFileStale() { + try { + const raw = readFileSync6(this.pidPath, "utf-8").trim(); + const pid = Number(raw); + if (!pid || Number.isNaN(pid)) + return true; + try { + process.kill(pid, 0); + return false; + } catch { + return true; + } + } catch { + return true; + } + } + async waitForSocket() { + const deadline = Date.now() + this.spawnWaitMs; + let delay = 30; + while (Date.now() < deadline) { + await sleep2(delay); + delay = Math.min(delay * 1.5, 300); + if (!existsSync5(this.socketPath)) + continue; + try { + return await this.connectOnce(); + } catch { + } + } + throw new Error("daemon did not become ready within spawnWaitMs"); + } + sendAndWait(sock, req) { + return new Promise((resolve2, reject) => { + let buf = ""; + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("request timeout")); + }, this.timeoutMs); + sock.setEncoding("utf-8"); + sock.on("data", (chunk) => { + buf += chunk; + const nl = buf.indexOf("\n"); + if (nl === -1) + return; + const line = buf.slice(0, nl); + clearTimeout(to); + try { + resolve2(JSON.parse(line)); + } catch (e) { + reject(e); + } + }); + sock.on("error", (e) => { + clearTimeout(to); + reject(e); + }); + sock.write(JSON.stringify(req) + "\n"); + }); + } +}; +function sleep2(ms) { + return new Promise((r) => setTimeout(r, ms)); +} + +// dist/src/embeddings/disable.js +function embeddingsDisabled() { + return process.env.HIVEMIND_EMBEDDINGS === "false"; +} + // dist/src/hooks/session-start-setup.js -var log3 = (msg) => log("session-setup", msg); +var log4 = (msg) => log("session-setup", msg); var __bundleDir = dirname3(fileURLToPath(import.meta.url)); var { log: wikiLog } = makeWikiLogger(join8(homedir5(), ".claude", "hooks")); async function main() { @@ -553,7 +791,7 @@ async function main() { const input = await readStdin(); const creds = loadCredentials(); if (!creds?.token) { - log3("no credentials"); + log4("no credentials"); return; } if (!creds.userName) { @@ -561,7 +799,7 @@ async function main() { const { userInfo: userInfo2 } = await import("node:os"); creds.userName = userInfo2().username ?? "unknown"; saveCredentials(creds); - log3(`backfilled userName: ${creds.userName}`); + log4(`backfilled userName: ${creds.userName}`); } catch { } } @@ -572,10 +810,10 @@ async function main() { const api = new DeeplakeApi(config.token, config.apiUrl, config.orgId, config.workspaceId, config.tableName); await api.ensureTable(); await api.ensureSessionsTable(config.sessionsTableName); - log3("setup complete"); + log4("setup complete"); } } catch (e) { - log3(`setup failed: ${e.message}`); + log4(`setup failed: ${e.message}`); wikiLog(`SessionSetup: failed for ${input.session_id}: ${e.message}`); } } @@ -586,7 +824,7 @@ async function main() { const latest = await getLatestVersion(); if (latest && isNewer(latest, current)) { if (autoupdate) { - log3(`autoupdate: updating ${current} \u2192 ${latest}`); + log4(`autoupdate: updating ${current} \u2192 ${latest}`); const resolved = resolveVersionedPluginDir(__bundleDir); const handle = resolved ? snapshotPluginDir(resolved.pluginDir) : null; try { @@ -594,30 +832,44 @@ async function main() { const cmd = scopes.map((s) => `claude plugin update hivemind@hivemind --scope ${s} 2>/dev/null || true`).join("; "); execSync2(cmd, { stdio: "ignore", timeout: 6e4 }); const outcome = restoreOrCleanup(handle); - log3(`autoupdate snapshot outcome: ${outcome}`); + log4(`autoupdate snapshot outcome: ${outcome}`); process.stderr.write(`\u2705 Hivemind auto-updated: ${current} \u2192 ${latest}. Run /reload-plugins to apply. `); - log3(`autoupdate succeeded: ${current} \u2192 ${latest}`); + log4(`autoupdate succeeded: ${current} \u2192 ${latest}`); } catch (e) { restoreOrCleanup(handle); process.stderr.write(`\u2B06\uFE0F Hivemind update available: ${current} \u2192 ${latest}. Auto-update failed \u2014 run /hivemind:update to upgrade manually. `); - log3(`autoupdate failed: ${e.message}`); + log4(`autoupdate failed: ${e.message}`); } } else { process.stderr.write(`\u2B06\uFE0F Hivemind update available: ${current} \u2192 ${latest}. Run /hivemind:update to upgrade. `); - log3(`update available (autoupdate off): ${current} \u2192 ${latest}`); + log4(`update available (autoupdate off): ${current} \u2192 ${latest}`); } } else { - log3(`version up to date: ${current}`); + log4(`version up to date: ${current}`); } } } catch (e) { - log3(`version check failed: ${e.message}`); + log4(`version check failed: ${e.message}`); + } + if (embeddingsDisabled()) { + log4("embed daemon warmup skipped via HIVEMIND_EMBEDDINGS=false"); + } else if (process.env.HIVEMIND_EMBED_WARMUP !== "false") { + try { + const daemonEntry = join8(__bundleDir, "embeddings", "embed-daemon.js"); + const client = new EmbedClient({ daemonEntry, timeoutMs: 300, spawnWaitMs: 5e3 }); + const ok = await client.warmup(); + log4(`embed daemon warmup: ${ok ? "ok" : "failed"}`); + } catch (e) { + log4(`embed daemon warmup threw: ${e.message}`); + } + } else { + log4("embed daemon warmup skipped via HIVEMIND_EMBED_WARMUP=false"); } } main().catch((e) => { - log3(`fatal: ${e.message}`); + log4(`fatal: ${e.message}`); process.exit(0); }); diff --git a/claude-code/bundle/session-start.js b/claude-code/bundle/session-start.js index 909bf88..3b34266 100755 --- a/claude-code/bundle/session-start.js +++ b/claude-code/bundle/session-start.js @@ -381,10 +381,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(tbl)) { log2(`table "${tbl}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', summary_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${tbl}" created`); if (!tables.includes(tbl)) this._tablesCache = [...tables, tbl]; + } else { + try { + await this.query(`ALTER TABLE "${tbl}" ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add summary_embedding skipped: ${e.message}`); + } } } /** Create the sessions table (uses JSONB for message since every row is a JSON event). */ @@ -392,10 +398,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(name)) { log2(`table "${name}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, message_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${name}" created`); if (!tables.includes(name)) this._tablesCache = [...tables, name]; + } else { + try { + await this.query(`ALTER TABLE "${name}" ADD COLUMN IF NOT EXISTS message_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add message_embedding skipped: ${e.message}`); + } } await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`); } @@ -572,7 +584,7 @@ Organization management \u2014 each argument is SEPARATE (do NOT quote subcomman - node "HIVEMIND_AUTH_CMD" members \u2014 list members - node "HIVEMIND_AUTH_CMD" remove \u2014 remove member -IMPORTANT: Only use bash commands (cat, ls, grep, echo, jq, head, tail, etc.) to interact with ~/.deeplake/memory/. Do NOT use python, python3, node, curl, or other interpreters \u2014 they are not available in the memory filesystem. If a task seems to require Python, rewrite it using bash commands and standard text-processing tools (awk, sed, jq, grep, etc.). +IMPORTANT: Only use bash commands (cat, ls, grep, echo, jq, head, tail, etc.) to interact with ~/.deeplake/memory/. Do NOT use python, python3, node, curl, or other interpreters \u2014 they are not available in the memory filesystem. Avoid bash brace expansions like \`{1..10}\` (not fully supported); spell out paths explicitly. Bash output is capped at 10MB total \u2014 avoid \`for f in *.json; do cat $f\` style loops on the whole sessions dir. LIMITS: Do NOT spawn subagents to read deeplake memory. If a file returns empty after 2 attempts, skip it and move on. Report what you found rather than exhaustively retrying. diff --git a/claude-code/bundle/shell/deeplake-shell.js b/claude-code/bundle/shell/deeplake-shell.js index 7e140a9..18e11f9 100755 --- a/claude-code/bundle/shell/deeplake-shell.js +++ b/claude-code/bundle/shell/deeplake-shell.js @@ -46081,14 +46081,14 @@ var require_turndown_cjs = __commonJS({ } else if (node.nodeType === 1) { replacement = replacementForNode.call(self2, node); } - return join7(output, replacement); + return join9(output, replacement); }, ""); } function postProcess(output) { var self2 = this; this.rules.forEach(function(rule) { if (typeof rule.append === "function") { - output = join7(output, rule.append(self2.options)); + output = join9(output, rule.append(self2.options)); } }); return output.replace(/^[\t\r\n]+/, "").replace(/[\t\r\n\s]+$/, ""); @@ -46100,7 +46100,7 @@ var require_turndown_cjs = __commonJS({ if (whitespace.leading || whitespace.trailing) content = content.trim(); return whitespace.leading + rule.replacement(content, node, this.options) + whitespace.trailing; } - function join7(output, replacement) { + function join9(output, replacement) { var s12 = trimTrailingNewlines(output); var s22 = trimLeadingNewlines(replacement); var nls = Math.max(output.length - s12.length, replacement.length - s22.length); @@ -67073,10 +67073,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(tbl)) { log2(`table "${tbl}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', summary_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${tbl}" created`); if (!tables.includes(tbl)) this._tablesCache = [...tables, tbl]; + } else { + try { + await this.query(`ALTER TABLE "${tbl}" ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]`); + } catch (e6) { + log2(`ALTER TABLE add summary_embedding skipped: ${e6.message}`); + } } } /** Create the sessions table (uses JSONB for message since every row is a JSON event). */ @@ -67084,10 +67090,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(name)) { log2(`table "${name}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, message_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${name}" created`); if (!tables.includes(name)) this._tablesCache = [...tables, name]; + } else { + try { + await this.query(`ALTER TABLE "${name}" ADD COLUMN IF NOT EXISTS message_embedding FLOAT4[]`); + } catch (e6) { + log2(`ALTER TABLE add message_embedding skipped: ${e6.message}`); + } } await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`); } @@ -67096,6 +67108,8 @@ var DeeplakeApi = class { // dist/src/shell/deeplake-fs.js import { basename as basename4, posix } from "node:path"; import { randomUUID as randomUUID2 } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import { dirname as dirname4, join as join7 } from "node:path"; // dist/src/shell/grep-core.js var TOOL_INPUT_FIELDS = [ @@ -67261,24 +67275,25 @@ function normalizeContent(path2, raw) { return raw; } if (Array.isArray(obj.turns)) { - const header = []; - if (obj.date_time) - header.push(`date: ${obj.date_time}`); - if (obj.speakers) { - const s10 = obj.speakers; - const names = [s10.speaker_a, s10.speaker_b].filter(Boolean).join(", "); - if (names) - header.push(`speakers: ${names}`); - } + const dateHeader = obj.date_time ? `(${String(obj.date_time)}) ` : ""; const lines = obj.turns.map((t6) => { const sp = String(t6?.speaker ?? t6?.name ?? "?").trim(); const tx = String(t6?.text ?? t6?.content ?? "").replace(/\s+/g, " ").trim(); const tag = t6?.dia_id ? `[${t6.dia_id}] ` : ""; - return `${tag}${sp}: ${tx}`; + return `${dateHeader}${tag}${sp}: ${tx}`; }); - const out2 = [...header, ...lines].join("\n"); + const out2 = lines.join("\n"); return out2.trim() ? out2 : raw; } + if (obj.turn && typeof obj.turn === "object" && !Array.isArray(obj.turn)) { + const t6 = obj.turn; + const sp = String(t6.speaker ?? t6.name ?? "?").trim(); + const tx = String(t6.text ?? t6.content ?? "").replace(/\s+/g, " ").trim(); + const tag = t6.dia_id ? `[${String(t6.dia_id)}] ` : ""; + const dateHeader = obj.date_time ? `(${String(obj.date_time)}) ` : ""; + const line = `${dateHeader}${tag}${sp}: ${tx}`; + return line.trim() ? line : raw; + } const stripRecalled = (t6) => { const i11 = t6.indexOf(""); if (i11 === -1) @@ -67321,8 +67336,38 @@ function buildPathCondition(targetPath) { return `(path = '${sqlStr(clean)}' OR path LIKE '${sqlLike(clean)}/%' ESCAPE '\\')`; } async function searchDeeplakeTables(api, memoryTable, sessionsTable, opts) { - const { pathFilter, contentScanOnly, likeOp, escapedPattern, prefilterPattern, prefilterPatterns, multiWordPatterns } = opts; + const { pathFilter, contentScanOnly, likeOp, escapedPattern, prefilterPattern, prefilterPatterns, queryEmbedding, bm25Term, multiWordPatterns } = opts; const limit = opts.limit ?? 100; + if (queryEmbedding && queryEmbedding.length > 0) { + const vecLit = serializeFloat4Array(queryEmbedding); + const semanticLimit = Math.min(limit, Number(process.env.HIVEMIND_SEMANTIC_LIMIT ?? "20")); + const lexicalLimit = Math.min(limit, Number(process.env.HIVEMIND_HYBRID_LEXICAL_LIMIT ?? "20")); + const filterPatternsForLex = contentScanOnly ? prefilterPatterns && prefilterPatterns.length > 0 ? prefilterPatterns : prefilterPattern ? [prefilterPattern] : [] : [escapedPattern]; + const memLexFilter = buildContentFilter("summary::text", likeOp, filterPatternsForLex); + const sessLexFilter = buildContentFilter("message::text", likeOp, filterPatternsForLex); + const memLexQuery = memLexFilter ? `SELECT path, summary::text AS content, 0 AS source_order, '' AS creation_date, 1.0 AS score FROM "${memoryTable}" WHERE 1=1${pathFilter}${memLexFilter} LIMIT ${lexicalLimit}` : null; + const sessLexQuery = sessLexFilter ? `SELECT path, message::text AS content, 1 AS source_order, COALESCE(creation_date::text, '') AS creation_date, 1.0 AS score FROM "${sessionsTable}" WHERE 1=1${pathFilter}${sessLexFilter} LIMIT ${lexicalLimit}` : null; + const memSemQuery = `SELECT path, summary::text AS content, 0 AS source_order, '' AS creation_date, (summary_embedding <#> ${vecLit}) AS score FROM "${memoryTable}" WHERE summary_embedding IS NOT NULL${pathFilter} ORDER BY score DESC LIMIT ${semanticLimit}`; + const sessSemQuery = `SELECT path, message::text AS content, 1 AS source_order, COALESCE(creation_date::text, '') AS creation_date, (message_embedding <#> ${vecLit}) AS score FROM "${sessionsTable}" WHERE message_embedding IS NOT NULL${pathFilter} ORDER BY score DESC LIMIT ${semanticLimit}`; + const parts = [memSemQuery, sessSemQuery]; + if (memLexQuery) + parts.push(memLexQuery); + if (sessLexQuery) + parts.push(sessLexQuery); + const unionSql = parts.map((q17) => `(${q17})`).join(" UNION ALL "); + const outerLimit = semanticLimit + lexicalLimit; + const rows2 = await api.query(`SELECT path, content, source_order, creation_date, score FROM (` + unionSql + `) AS combined ORDER BY score DESC LIMIT ${outerLimit}`); + const seen = /* @__PURE__ */ new Set(); + const unique = []; + for (const row of rows2) { + const p22 = String(row["path"]); + if (seen.has(p22)) + continue; + seen.add(p22); + unique.push({ path: p22, content: String(row["content"] ?? "") }); + } + return unique; + } const filterPatterns = contentScanOnly ? prefilterPatterns && prefilterPatterns.length > 0 ? prefilterPatterns : prefilterPattern ? [prefilterPattern] : [] : multiWordPatterns && multiWordPatterns.length > 1 ? multiWordPatterns : [escapedPattern]; const memFilter = buildContentFilter("summary::text", likeOp, filterPatterns); const sessFilter = buildContentFilter("message::text", likeOp, filterPatterns); @@ -67334,6 +67379,15 @@ async function searchDeeplakeTables(api, memoryTable, sessionsTable, opts) { content: String(row["content"] ?? "") })); } +function serializeFloat4Array(vec) { + const parts = []; + for (const v27 of vec) { + if (!Number.isFinite(v27)) + return "NULL"; + parts.push(String(v27)); + } + return `ARRAY[${parts.join(",")}]::float4[]`; +} function buildPathFilter(targetPath) { const condition = buildPathCondition(targetPath); return condition ? ` AND ${condition}` : ""; @@ -67422,14 +67476,25 @@ function buildGrepSearchOptions(params, targetPath) { const hasRegexMeta = !params.fixedString && /[.*+?^${}()|[\]\\]/.test(params.pattern); const literalPrefilter = hasRegexMeta ? extractRegexLiteralPrefilter(params.pattern) : null; const alternationPrefilters = hasRegexMeta ? extractRegexAlternationPrefilters(params.pattern) : null; + let bm25Term; + if (!hasRegexMeta) { + bm25Term = params.pattern; + } else if (alternationPrefilters && alternationPrefilters.length > 0) { + bm25Term = alternationPrefilters.join(" "); + } else if (literalPrefilter) { + bm25Term = literalPrefilter; + } const multiWordPatterns = !hasRegexMeta ? params.pattern.split(/\s+/).filter((w20) => w20.length > 2).slice(0, 4) : []; return { pathFilter: buildPathFilter(targetPath), contentScanOnly: hasRegexMeta, - likeOp: params.ignoreCase ? "ILIKE" : "LIKE", + // Kept for the pure-lexical fallback path and existing tests; BM25 is now + // the preferred lexical ranker when a term can be extracted. + likeOp: process.env.HIVEMIND_GREP_LIKE === "case-sensitive" ? "LIKE" : "ILIKE", escapedPattern: sqlLike(params.pattern), prefilterPattern: literalPrefilter ? sqlLike(literalPrefilter) : void 0, prefilterPatterns: alternationPrefilters?.map((literal) => sqlLike(literal)), + bm25Term, multiWordPatterns: multiWordPatterns.length > 1 ? multiWordPatterns.map((w20) => sqlLike(w20)) : void 0 }; } @@ -67482,6 +67547,245 @@ function refineGrepMatches(rows, params, forceMultiFilePrefix) { return output; } +// dist/src/embeddings/client.js +import { connect } from "node:net"; +import { spawn } from "node:child_process"; +import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync4, readFileSync as readFileSync3 } from "node:fs"; + +// dist/src/embeddings/protocol.js +var DEFAULT_SOCKET_DIR = "/tmp"; +var DEFAULT_IDLE_TIMEOUT_MS = 15 * 60 * 1e3; +var DEFAULT_CLIENT_TIMEOUT_MS = 200; +function socketPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.sock`; +} +function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.pid`; +} + +// dist/src/embeddings/client.js +var log3 = (m26) => log("embed-client", m26); +function getUid() { + const uid = typeof process.getuid === "function" ? process.getuid() : void 0; + return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; +} +var EmbedClient = class { + socketPath; + pidPath; + timeoutMs; + daemonEntry; + autoSpawn; + spawnWaitMs; + nextId = 0; + constructor(opts = {}) { + const uid = getUid(); + const dir = opts.socketDir ?? "/tmp"; + this.socketPath = socketPathFor(uid, dir); + this.pidPath = pidPathFor(uid, dir); + this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON; + this.autoSpawn = opts.autoSpawn ?? true; + this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; + } + /** + * Returns an embedding vector, or null on timeout/failure. Hooks MUST treat + * null as "skip embedding column" — never block the write path on us. + * + * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns + * null AND kicks off a background spawn. The next call finds a ready daemon. + */ + async embed(text, kind = "document") { + let sock; + try { + sock = await this.connectOnce(); + } catch { + if (this.autoSpawn) + this.trySpawnDaemon(); + return null; + } + try { + const id = String(++this.nextId); + const req = { op: "embed", id, kind, text }; + const resp = await this.sendAndWait(sock, req); + if (resp.error || !("embedding" in resp) || !resp.embedding) { + log3(`embed err: ${resp.error ?? "no embedding"}`); + return null; + } + return resp.embedding; + } catch (e6) { + const err = e6 instanceof Error ? e6.message : String(e6); + log3(`embed failed: ${err}`); + return null; + } finally { + try { + sock.end(); + } catch { + } + } + } + /** + * Wait up to spawnWaitMs for the daemon to accept connections, spawning if + * necessary. Meant for SessionStart / long-running batches — not the hot path. + */ + async warmup() { + try { + const s10 = await this.connectOnce(); + s10.end(); + return true; + } catch { + if (!this.autoSpawn) + return false; + this.trySpawnDaemon(); + try { + const s10 = await this.waitForSocket(); + s10.end(); + return true; + } catch { + return false; + } + } + } + connectOnce() { + return new Promise((resolve5, reject) => { + const sock = connect(this.socketPath); + const to3 = setTimeout(() => { + sock.destroy(); + reject(new Error("connect timeout")); + }, this.timeoutMs); + sock.once("connect", () => { + clearTimeout(to3); + resolve5(sock); + }); + sock.once("error", (e6) => { + clearTimeout(to3); + reject(e6); + }); + }); + } + trySpawnDaemon() { + let fd; + try { + fd = openSync(this.pidPath, "wx", 384); + writeSync(fd, String(process.pid)); + } catch (e6) { + if (this.isPidFileStale()) { + try { + unlinkSync(this.pidPath); + } catch { + } + try { + fd = openSync(this.pidPath, "wx", 384); + writeSync(fd, String(process.pid)); + } catch { + return; + } + } else { + return; + } + } + if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { + log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + try { + closeSync(fd); + unlinkSync(this.pidPath); + } catch { + } + return; + } + try { + const child = spawn(process.execPath, [this.daemonEntry], { + detached: true, + stdio: "ignore", + env: process.env + }); + child.unref(); + log3(`spawned daemon pid=${child.pid}`); + } finally { + closeSync(fd); + } + } + isPidFileStale() { + try { + const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const pid = Number(raw); + if (!pid || Number.isNaN(pid)) + return true; + try { + process.kill(pid, 0); + return false; + } catch { + return true; + } + } catch { + return true; + } + } + async waitForSocket() { + const deadline = Date.now() + this.spawnWaitMs; + let delay = 30; + while (Date.now() < deadline) { + await sleep2(delay); + delay = Math.min(delay * 1.5, 300); + if (!existsSync4(this.socketPath)) + continue; + try { + return await this.connectOnce(); + } catch { + } + } + throw new Error("daemon did not become ready within spawnWaitMs"); + } + sendAndWait(sock, req) { + return new Promise((resolve5, reject) => { + let buf = ""; + const to3 = setTimeout(() => { + sock.destroy(); + reject(new Error("request timeout")); + }, this.timeoutMs); + sock.setEncoding("utf-8"); + sock.on("data", (chunk) => { + buf += chunk; + const nl3 = buf.indexOf("\n"); + if (nl3 === -1) + return; + const line = buf.slice(0, nl3); + clearTimeout(to3); + try { + resolve5(JSON.parse(line)); + } catch (e6) { + reject(e6); + } + }); + sock.on("error", (e6) => { + clearTimeout(to3); + reject(e6); + }); + sock.write(JSON.stringify(req) + "\n"); + }); + } +}; +function sleep2(ms3) { + return new Promise((r10) => setTimeout(r10, ms3)); +} + +// dist/src/embeddings/sql.js +function embeddingSqlLiteral(vec) { + if (!vec || vec.length === 0) + return "NULL"; + const parts = []; + for (const v27 of vec) { + if (!Number.isFinite(v27)) + return "NULL"; + parts.push(String(v27)); + } + return `ARRAY[${parts.join(",")}]::float4[]`; +} + +// dist/src/embeddings/disable.js +function embeddingsDisabled() { + return process.env.HIVEMIND_EMBEDDINGS === "false"; +} + // dist/src/shell/deeplake-fs.js var BATCH_SIZE = 10; var PREFETCH_BATCH_SIZE = 50; @@ -67510,6 +67814,9 @@ function normalizeSessionMessage(path2, message) { const raw = typeof message === "string" ? message : JSON.stringify(message); return normalizeContent(path2, raw); } +function resolveEmbedDaemonPath() { + return join7(dirname4(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); +} function joinSessionMessages(path2, messages) { return messages.map((message) => normalizeSessionMessage(path2, message)).join("\n"); } @@ -67539,6 +67846,8 @@ var DeeplakeFs = class _DeeplakeFs { // Paths that live in the sessions table (multi-row, read by concatenation) sessionPaths = /* @__PURE__ */ new Set(); sessionsTable = null; + // Embedding client lazily created on first flush. Lives as long as the process. + embedClient = null; constructor(client, table, mountPoint) { this.client = client; this.table = table; @@ -67572,7 +67881,14 @@ var DeeplakeFs = class _DeeplakeFs { })(); const sessionsBootstrap = sessionsTable && sessionSyncOk ? (async () => { try { - const sessionRows = await client.query(`SELECT path, SUM(size_bytes) as total_size FROM "${sessionsTable}" GROUP BY path ORDER BY path`); + const sessionRows = await client.query( + // NOTE: SUM(size_bytes) returns NULL on the Deeplake backend when combined + // with GROUP BY path (confirmed against workspace `with_embedding`). MAX + // works and — for the single-row-per-file layout — is equal to SUM. For + // multi-row-per-turn layouts MAX under-reports total size but stays >0 + // so files don't look like empty placeholders in ls/stat. + `SELECT path, MAX(size_bytes) as total_size FROM "${sessionsTable}" GROUP BY path ORDER BY path` + ); for (const row of sessionRows) { const p22 = row["path"]; if (!fs3.files.has(p22)) { @@ -67632,7 +67948,8 @@ var DeeplakeFs = class _DeeplakeFs { } const rows = [...this.pending.values()]; this.pending.clear(); - const results = await Promise.allSettled(rows.map((r10) => this.upsertRow(r10))); + const embeddings = await this.computeEmbeddings(rows); + const results = await Promise.allSettled(rows.map((r10, i11) => this.upsertRow(r10, embeddings[i11]))); let failures = 0; for (let i11 = 0; i11 < results.length; i11++) { if (results[i11].status === "rejected") { @@ -67646,7 +67963,17 @@ var DeeplakeFs = class _DeeplakeFs { throw new Error(`flush: ${failures}/${rows.length} writes failed and were re-queued`); } } - async upsertRow(r10) { + async computeEmbeddings(rows) { + if (rows.length === 0) + return []; + if (embeddingsDisabled()) + return rows.map(() => null); + if (!this.embedClient) { + this.embedClient = new EmbedClient({ daemonEntry: resolveEmbedDaemonPath() }); + } + return Promise.all(rows.map((r10) => this.embedClient.embed(r10.contentText, "document"))); + } + async upsertRow(r10, embedding) { const text = sqlStr(r10.contentText); const p22 = sqlStr(r10.path); const fname = sqlStr(r10.filename); @@ -67654,8 +67981,9 @@ var DeeplakeFs = class _DeeplakeFs { const ts3 = (/* @__PURE__ */ new Date()).toISOString(); const cd = r10.creationDate ?? ts3; const lud = r10.lastUpdateDate ?? ts3; + const embSql = embeddingSqlLiteral(embedding); if (this.flushed.has(r10.path)) { - let setClauses = `filename = '${fname}', summary = E'${text}', mime_type = '${mime}', size_bytes = ${r10.sizeBytes}, last_update_date = '${sqlStr(lud)}'`; + let setClauses = `filename = '${fname}', summary = E'${text}', summary_embedding = ${embSql}, mime_type = '${mime}', size_bytes = ${r10.sizeBytes}, last_update_date = '${sqlStr(lud)}'`; if (r10.project !== void 0) setClauses += `, project = '${sqlStr(r10.project)}'`; if (r10.description !== void 0) @@ -67663,51 +67991,72 @@ var DeeplakeFs = class _DeeplakeFs { await this.client.query(`UPDATE "${this.table}" SET ${setClauses} WHERE path = '${p22}'`); } else { const id = randomUUID2(); - const cols = "id, path, filename, summary, mime_type, size_bytes, creation_date, last_update_date" + (r10.project !== void 0 ? ", project" : "") + (r10.description !== void 0 ? ", description" : ""); - const vals = `'${id}', '${p22}', '${fname}', E'${text}', '${mime}', ${r10.sizeBytes}, '${sqlStr(cd)}', '${sqlStr(lud)}'` + (r10.project !== void 0 ? `, '${sqlStr(r10.project)}'` : "") + (r10.description !== void 0 ? `, '${sqlStr(r10.description)}'` : ""); + const cols = "id, path, filename, summary, summary_embedding, mime_type, size_bytes, creation_date, last_update_date" + (r10.project !== void 0 ? ", project" : "") + (r10.description !== void 0 ? ", description" : ""); + const vals = `'${id}', '${p22}', '${fname}', E'${text}', ${embSql}, '${mime}', ${r10.sizeBytes}, '${sqlStr(cd)}', '${sqlStr(lud)}'` + (r10.project !== void 0 ? `, '${sqlStr(r10.project)}'` : "") + (r10.description !== void 0 ? `, '${sqlStr(r10.description)}'` : ""); await this.client.query(`INSERT INTO "${this.table}" (${cols}) VALUES (${vals})`); this.flushed.add(r10.path); } } // ── Virtual index.md generation ──────────────────────────────────────────── async generateVirtualIndex() { - const rows = await this.client.query(`SELECT path, project, description, creation_date, last_update_date FROM "${this.table}" WHERE path LIKE '${sqlStr("/summaries/")}%' ORDER BY last_update_date DESC`); - const sessionPathsByKey = /* @__PURE__ */ new Map(); - for (const sp of this.sessionPaths) { - const hivemind = sp.match(/\/sessions\/[^/]+\/[^/]+_([^.]+)\.jsonl$/); - if (hivemind) { - sessionPathsByKey.set(hivemind[1], sp.slice(1)); - } else { - const fname = sp.split("/").pop() ?? ""; - const stem = fname.replace(/\.[^.]+$/, ""); - if (stem) - sessionPathsByKey.set(stem, sp.slice(1)); + const summaryRows = await this.client.query(`SELECT path, project, description, creation_date, last_update_date FROM "${this.table}" WHERE path LIKE '${sqlStr("/summaries/")}%' ORDER BY last_update_date DESC`); + let sessionRows = []; + if (this.sessionsTable) { + try { + sessionRows = await this.client.query(`SELECT path, MAX(description) AS description, MIN(creation_date) AS creation_date, MAX(last_update_date) AS last_update_date FROM "${this.sessionsTable}" WHERE path LIKE '${sqlStr("/sessions/")}%' GROUP BY path ORDER BY path`); + } catch { + sessionRows = []; } } const lines = [ "# Session Index", "", - "List of all Claude Code sessions with summaries.", - "", - "| Session | Conversation | Created | Last Updated | Project | Description |", - "|---------|-------------|---------|--------------|---------|-------------|" + "Two sources are available. Consult the section relevant to the question.", + "" ]; - for (const row of rows) { - const p22 = row["path"]; - const match2 = p22.match(/\/summaries\/([^/]+)\/([^/]+)\.md$/); - if (!match2) - continue; - const summaryUser = match2[1]; - const sessionId = match2[2]; - const relPath = `summaries/${summaryUser}/${sessionId}.md`; - const baseName = sessionId.replace(/_summary$/, ""); - const convPath = sessionPathsByKey.get(sessionId) ?? sessionPathsByKey.get(baseName); - const convLink = convPath ? `[messages](${convPath})` : ""; - const project = row["project"] || ""; - const description = row["description"] || ""; - const creationDate = row["creation_date"] || ""; - const lastUpdateDate = row["last_update_date"] || ""; - lines.push(`| [${sessionId}](${relPath}) | ${convLink} | ${creationDate} | ${lastUpdateDate} | ${project} | ${description} |`); + lines.push("## memory"); + lines.push(""); + if (summaryRows.length === 0) { + lines.push("_(empty \u2014 no summaries ingested yet)_"); + } else { + lines.push("AI-generated summaries per session. Read these first for topic-level overviews."); + lines.push(""); + lines.push("| Session | Created | Last Updated | Project | Description |"); + lines.push("|---------|---------|--------------|---------|-------------|"); + for (const row of summaryRows) { + const p22 = row["path"]; + const match2 = p22.match(/\/summaries\/([^/]+)\/([^/]+)\.md$/); + if (!match2) + continue; + const summaryUser = match2[1]; + const sessionId = match2[2]; + const relPath = `summaries/${summaryUser}/${sessionId}.md`; + const project = row["project"] || ""; + const description = row["description"] || ""; + const creationDate = row["creation_date"] || ""; + const lastUpdateDate = row["last_update_date"] || ""; + lines.push(`| [${sessionId}](${relPath}) | ${creationDate} | ${lastUpdateDate} | ${project} | ${description} |`); + } + } + lines.push(""); + lines.push("## sessions"); + lines.push(""); + if (sessionRows.length === 0) { + lines.push("_(empty \u2014 no session records ingested yet)_"); + } else { + lines.push("Raw session records (dialogue, tool calls). Read for exact detail / quotes."); + lines.push(""); + lines.push("| Session | Created | Last Updated | Description |"); + lines.push("|---------|---------|--------------|-------------|"); + for (const row of sessionRows) { + const p22 = row["path"] || ""; + const rel = p22.startsWith("/") ? p22.slice(1) : p22; + const filename = p22.split("/").pop() ?? p22; + const description = row["description"] || ""; + const creationDate = row["creation_date"] || ""; + const lastUpdateDate = row["last_update_date"] || ""; + lines.push(`| [${filename}](${rel}) | ${creationDate} | ${lastUpdateDate} | ${description} |`); + } } lines.push(""); return lines.join("\n"); @@ -69018,7 +69367,7 @@ function stripQuotes(val) { } // node_modules/yargs-parser/build/lib/index.js -import { readFileSync as readFileSync3 } from "fs"; +import { readFileSync as readFileSync4 } from "fs"; import { createRequire } from "node:module"; var _a3; var _b; @@ -69045,7 +69394,7 @@ var parser = new YargsParser({ if (typeof require2 !== "undefined") { return require2(path2); } else if (path2.match(/\.json$/)) { - return JSON.parse(readFileSync3(path2, "utf8")); + return JSON.parse(readFileSync4(path2, "utf8")); } else { throw Error("only .json config files are supported in ESM"); } @@ -69064,6 +69413,33 @@ yargsParser.looksLikeNumber = looksLikeNumber; var lib_default = yargsParser; // dist/src/shell/grep-interceptor.js +import { fileURLToPath as fileURLToPath2 } from "node:url"; +import { dirname as dirname5, join as join8 } from "node:path"; +var SEMANTIC_SEARCH_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); +var SEMANTIC_EMBED_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); +function resolveGrepEmbedDaemonPath() { + return join8(dirname5(fileURLToPath2(import.meta.url)), "..", "embeddings", "embed-daemon.js"); +} +var sharedGrepEmbedClient = null; +function getGrepEmbedClient() { + if (!sharedGrepEmbedClient) { + sharedGrepEmbedClient = new EmbedClient({ + daemonEntry: resolveGrepEmbedDaemonPath(), + timeoutMs: SEMANTIC_EMBED_TIMEOUT_MS + }); + } + return sharedGrepEmbedClient; +} +function patternIsSemanticFriendly(pattern, fixedString) { + if (!pattern || pattern.length < 2) + return false; + if (fixedString) + return true; + const metaMatches = pattern.match(/[|()\[\]{}+?^$\\]/g); + if (!metaMatches) + return true; + return metaMatches.length <= 1; +} var MAX_FALLBACK_CANDIDATES = 500; function createGrepCommand(client, fs3, table, sessionsTable) { return Yi2("grep", async (args, ctx) => { @@ -69105,12 +69481,21 @@ function createGrepCommand(client, fs3, table, sessionsTable) { filesOnly: Boolean(parsed.l || parsed["files-with-matches"]), countOnly: Boolean(parsed.c || parsed["count"]) }; + let queryEmbedding = null; + if (SEMANTIC_SEARCH_ENABLED && patternIsSemanticFriendly(pattern, matchParams.fixedString)) { + try { + queryEmbedding = await getGrepEmbedClient().embed(pattern, "query"); + } catch { + queryEmbedding = null; + } + } let rows = []; try { const searchOptions = { ...buildGrepSearchOptions(matchParams, targets[0] ?? ctx.cwd), pathFilter: buildPathFilterForTargets(targets), - limit: 100 + limit: 100, + queryEmbedding }; const queryRows = await Promise.race([ searchDeeplakeTables(client, table, sessionsTable ?? "sessions", searchOptions), @@ -69120,6 +69505,21 @@ function createGrepCommand(client, fs3, table, sessionsTable) { } catch { rows = []; } + if (rows.length === 0 && queryEmbedding) { + try { + const lexicalOptions = { + ...buildGrepSearchOptions(matchParams, targets[0] ?? ctx.cwd), + pathFilter: buildPathFilterForTargets(targets), + limit: 100 + }; + const lexicalRows = await Promise.race([ + searchDeeplakeTables(client, table, sessionsTable ?? "sessions", lexicalOptions), + new Promise((_16, reject) => setTimeout(() => reject(new Error("timeout")), 3e3)) + ]); + rows.push(...lexicalRows); + } catch { + } + } const seen = /* @__PURE__ */ new Set(); rows = rows.filter((r10) => seen.has(r10.path) ? false : (seen.add(r10.path), true)); if (rows.length === 0) { @@ -69133,7 +69533,19 @@ function createGrepCommand(client, fs3, table, sessionsTable) { } } const normalized = rows.map((r10) => ({ path: r10.path, content: normalizeContent(r10.path, r10.content) })); - const output = refineGrepMatches(normalized, matchParams); + let output; + if (queryEmbedding && queryEmbedding.length > 0 && process.env.HIVEMIND_SEMANTIC_EMIT_ALL !== "false") { + output = []; + for (const r10 of normalized) { + for (const line of r10.content.split("\n")) { + const trimmed = line.trim(); + if (trimmed) + output.push(`${r10.path}:${line}`); + } + } + } else { + output = refineGrepMatches(normalized, matchParams); + } return { stdout: output.length > 0 ? output.join("\n") + "\n" : "", stderr: "", diff --git a/claude-code/bundle/wiki-worker.js b/claude-code/bundle/wiki-worker.js index b2f974f..8dee999 100755 --- a/claude-code/bundle/wiki-worker.js +++ b/claude-code/bundle/wiki-worker.js @@ -1,9 +1,10 @@ #!/usr/bin/env node // dist/src/hooks/wiki-worker.js -import { readFileSync as readFileSync2, writeFileSync as writeFileSync2, existsSync as existsSync2, appendFileSync as appendFileSync2, mkdirSync as mkdirSync2, rmSync } from "node:fs"; +import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, existsSync as existsSync3, appendFileSync as appendFileSync2, mkdirSync as mkdirSync2, rmSync } from "node:fs"; import { execFileSync } from "node:child_process"; -import { join as join3 } from "node:path"; +import { dirname, join as join3 } from "node:path"; +import { fileURLToPath } from "node:url"; // dist/src/utils/debug.js import { appendFileSync } from "node:fs"; @@ -107,6 +108,21 @@ function releaseLock(sessionId) { // dist/src/hooks/upload-summary.js import { randomUUID } from "node:crypto"; + +// dist/src/embeddings/sql.js +function embeddingSqlLiteral(vec) { + if (!vec || vec.length === 0) + return "NULL"; + const parts = []; + for (const v of vec) { + if (!Number.isFinite(v)) + return "NULL"; + parts.push(String(v)); + } + return `ARRAY[${parts.join(",")}]::float4[]`; +} + +// dist/src/hooks/upload-summary.js function esc(s) { return s.replace(/\\/g, "\\\\").replace(/'/g, "''").replace(/[\x01-\x08\x0b\x0c\x0e-\x1f\x7f]/g, ""); } @@ -119,20 +135,247 @@ async function uploadSummary(query2, params) { const ts = params.ts ?? (/* @__PURE__ */ new Date()).toISOString(); const desc = extractDescription(text); const sizeBytes = Buffer.byteLength(text); + const embSql = embeddingSqlLiteral(params.embedding ?? null); const existing = await query2(`SELECT path FROM "${tableName}" WHERE path = '${esc(vpath)}' LIMIT 1`); if (existing.length > 0) { - const sql2 = `UPDATE "${tableName}" SET summary = E'${esc(text)}', size_bytes = ${sizeBytes}, description = E'${esc(desc)}', last_update_date = '${ts}' WHERE path = '${esc(vpath)}'`; + const sql2 = `UPDATE "${tableName}" SET summary = E'${esc(text)}', summary_embedding = ${embSql}, size_bytes = ${sizeBytes}, description = E'${esc(desc)}', last_update_date = '${ts}' WHERE path = '${esc(vpath)}'`; await query2(sql2); return { path: "update", sql: sql2, descLength: desc.length, summaryLength: text.length }; } - const sql = `INSERT INTO "${tableName}" (id, path, filename, summary, author, mime_type, size_bytes, project, description, agent, creation_date, last_update_date) VALUES ('${randomUUID()}', '${esc(vpath)}', '${esc(fname)}', E'${esc(text)}', '${esc(userName)}', 'text/markdown', ${sizeBytes}, '${esc(project)}', E'${esc(desc)}', '${esc(agent)}', '${ts}', '${ts}')`; + const sql = `INSERT INTO "${tableName}" (id, path, filename, summary, summary_embedding, author, mime_type, size_bytes, project, description, agent, creation_date, last_update_date) VALUES ('${randomUUID()}', '${esc(vpath)}', '${esc(fname)}', E'${esc(text)}', ${embSql}, '${esc(userName)}', 'text/markdown', ${sizeBytes}, '${esc(project)}', E'${esc(desc)}', '${esc(agent)}', '${ts}', '${ts}')`; await query2(sql); return { path: "insert", sql, descLength: desc.length, summaryLength: text.length }; } +// dist/src/embeddings/client.js +import { connect } from "node:net"; +import { spawn } from "node:child_process"; +import { openSync as openSync2, closeSync as closeSync2, writeSync as writeSync2, unlinkSync as unlinkSync2, existsSync as existsSync2, readFileSync as readFileSync2 } from "node:fs"; + +// dist/src/embeddings/protocol.js +var DEFAULT_SOCKET_DIR = "/tmp"; +var DEFAULT_IDLE_TIMEOUT_MS = 15 * 60 * 1e3; +var DEFAULT_CLIENT_TIMEOUT_MS = 200; +function socketPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.sock`; +} +function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.pid`; +} + +// dist/src/embeddings/client.js +var log2 = (m) => log("embed-client", m); +function getUid() { + const uid = typeof process.getuid === "function" ? process.getuid() : void 0; + return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; +} +var EmbedClient = class { + socketPath; + pidPath; + timeoutMs; + daemonEntry; + autoSpawn; + spawnWaitMs; + nextId = 0; + constructor(opts = {}) { + const uid = getUid(); + const dir = opts.socketDir ?? "/tmp"; + this.socketPath = socketPathFor(uid, dir); + this.pidPath = pidPathFor(uid, dir); + this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON; + this.autoSpawn = opts.autoSpawn ?? true; + this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; + } + /** + * Returns an embedding vector, or null on timeout/failure. Hooks MUST treat + * null as "skip embedding column" — never block the write path on us. + * + * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns + * null AND kicks off a background spawn. The next call finds a ready daemon. + */ + async embed(text, kind = "document") { + let sock; + try { + sock = await this.connectOnce(); + } catch { + if (this.autoSpawn) + this.trySpawnDaemon(); + return null; + } + try { + const id = String(++this.nextId); + const req = { op: "embed", id, kind, text }; + const resp = await this.sendAndWait(sock, req); + if (resp.error || !("embedding" in resp) || !resp.embedding) { + log2(`embed err: ${resp.error ?? "no embedding"}`); + return null; + } + return resp.embedding; + } catch (e) { + const err = e instanceof Error ? e.message : String(e); + log2(`embed failed: ${err}`); + return null; + } finally { + try { + sock.end(); + } catch { + } + } + } + /** + * Wait up to spawnWaitMs for the daemon to accept connections, spawning if + * necessary. Meant for SessionStart / long-running batches — not the hot path. + */ + async warmup() { + try { + const s = await this.connectOnce(); + s.end(); + return true; + } catch { + if (!this.autoSpawn) + return false; + this.trySpawnDaemon(); + try { + const s = await this.waitForSocket(); + s.end(); + return true; + } catch { + return false; + } + } + } + connectOnce() { + return new Promise((resolve, reject) => { + const sock = connect(this.socketPath); + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("connect timeout")); + }, this.timeoutMs); + sock.once("connect", () => { + clearTimeout(to); + resolve(sock); + }); + sock.once("error", (e) => { + clearTimeout(to); + reject(e); + }); + }); + } + trySpawnDaemon() { + let fd; + try { + fd = openSync2(this.pidPath, "wx", 384); + writeSync2(fd, String(process.pid)); + } catch (e) { + if (this.isPidFileStale()) { + try { + unlinkSync2(this.pidPath); + } catch { + } + try { + fd = openSync2(this.pidPath, "wx", 384); + writeSync2(fd, String(process.pid)); + } catch { + return; + } + } else { + return; + } + } + if (!this.daemonEntry || !existsSync2(this.daemonEntry)) { + log2(`daemonEntry not configured or missing: ${this.daemonEntry}`); + try { + closeSync2(fd); + unlinkSync2(this.pidPath); + } catch { + } + return; + } + try { + const child = spawn(process.execPath, [this.daemonEntry], { + detached: true, + stdio: "ignore", + env: process.env + }); + child.unref(); + log2(`spawned daemon pid=${child.pid}`); + } finally { + closeSync2(fd); + } + } + isPidFileStale() { + try { + const raw = readFileSync2(this.pidPath, "utf-8").trim(); + const pid = Number(raw); + if (!pid || Number.isNaN(pid)) + return true; + try { + process.kill(pid, 0); + return false; + } catch { + return true; + } + } catch { + return true; + } + } + async waitForSocket() { + const deadline = Date.now() + this.spawnWaitMs; + let delay = 30; + while (Date.now() < deadline) { + await sleep(delay); + delay = Math.min(delay * 1.5, 300); + if (!existsSync2(this.socketPath)) + continue; + try { + return await this.connectOnce(); + } catch { + } + } + throw new Error("daemon did not become ready within spawnWaitMs"); + } + sendAndWait(sock, req) { + return new Promise((resolve, reject) => { + let buf = ""; + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("request timeout")); + }, this.timeoutMs); + sock.setEncoding("utf-8"); + sock.on("data", (chunk) => { + buf += chunk; + const nl = buf.indexOf("\n"); + if (nl === -1) + return; + const line = buf.slice(0, nl); + clearTimeout(to); + try { + resolve(JSON.parse(line)); + } catch (e) { + reject(e); + } + }); + sock.on("error", (e) => { + clearTimeout(to); + reject(e); + }); + sock.write(JSON.stringify(req) + "\n"); + }); + } +}; +function sleep(ms) { + return new Promise((r) => setTimeout(r, ms)); +} + +// dist/src/embeddings/disable.js +function embeddingsDisabled() { + return process.env.HIVEMIND_EMBEDDINGS === "false"; +} + // dist/src/hooks/wiki-worker.js var dlog2 = (msg) => log("wiki-worker", msg); -var cfg = JSON.parse(readFileSync2(process.argv[2], "utf-8")); +var cfg = JSON.parse(readFileSync3(process.argv[2], "utf-8")); var tmpDir = cfg.tmpDir; var tmpJsonl = join3(tmpDir, "session.jsonl"); var tmpSummary = join3(tmpDir, "summary.md"); @@ -230,11 +473,20 @@ async function main() { } catch (e) { wlog(`claude -p failed: ${e.status ?? e.message}`); } - if (existsSync2(tmpSummary)) { - const text = readFileSync2(tmpSummary, "utf-8"); + if (existsSync3(tmpSummary)) { + const text = readFileSync3(tmpSummary, "utf-8"); if (text.trim()) { const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; + let embedding = null; + if (!embeddingsDisabled()) { + try { + const daemonEntry = join3(dirname(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); + embedding = await new EmbedClient({ daemonEntry }).embed(text, "document"); + } catch (e) { + wlog(`summary embedding failed, writing NULL: ${e.message}`); + } + } const result = await uploadSummary(query, { tableName: cfg.memoryTable, vpath, @@ -243,7 +495,8 @@ async function main() { project: cfg.project, agent: "claude_code", sessionId: cfg.sessionId, - text + text, + embedding }); wlog(`uploaded ${vpath} (summary=${result.summaryLength}, desc=${result.descLength})`); try { diff --git a/claude-code/hooks/hooks.json b/claude-code/hooks/hooks.json index ef82672..d7f59e8 100644 --- a/claude-code/hooks/hooks.json +++ b/claude-code/hooks/hooks.json @@ -36,7 +36,7 @@ { "type": "command", "command": "node \"${CLAUDE_PLUGIN_ROOT}/bundle/pre-tool-use.js\"", - "timeout": 10 + "timeout": 60 } ] } diff --git a/claude-code/tests/capture-hook.test.ts b/claude-code/tests/capture-hook.test.ts index c40e8e6..5f19674 100644 --- a/claude-code/tests/capture-hook.test.ts +++ b/claude-code/tests/capture-hook.test.ts @@ -50,6 +50,12 @@ vi.mock("../../src/deeplake-api.js", () => ({ ensureSessionsTable(t: string) { return ensureSessionsTableMock(t); } }, })); +vi.mock("../../src/embeddings/client.js", () => ({ + EmbedClient: class { + embed(_text: string, _kind?: string) { return Promise.resolve(null); } + warmup() { return Promise.resolve(false); } + }, +})); async function runHook(env: Record = {}): Promise { delete process.env.HIVEMIND_WIKI_WORKER; diff --git a/claude-code/tests/codex-wiki-worker.test.ts b/claude-code/tests/codex-wiki-worker.test.ts index 6a4260a..9ff6127 100644 --- a/claude-code/tests/codex-wiki-worker.test.ts +++ b/claude-code/tests/codex-wiki-worker.test.ts @@ -19,6 +19,7 @@ const finalizeSummaryMock = vi.fn(); const releaseLockMock = vi.fn(); const uploadSummaryMock = vi.fn(); const execFileSyncMock = vi.fn(); +const embedSummaryMock = vi.fn(); vi.mock("../../src/hooks/summary-state.js", () => ({ finalizeSummary: (...a: any[]) => finalizeSummaryMock(...a), @@ -27,6 +28,11 @@ vi.mock("../../src/hooks/summary-state.js", () => ({ vi.mock("../../src/hooks/upload-summary.js", () => ({ uploadSummary: (...a: any[]) => uploadSummaryMock(...a), })); +vi.mock("../../src/embeddings/client.js", () => ({ + EmbedClient: class { + async embed(text: string, kind: string) { return embedSummaryMock(text, kind); } + }, +})); vi.mock("node:child_process", async () => { const actual = await vi.importActual("node:child_process"); return { ...actual, execFileSync: (...a: any[]) => execFileSyncMock(...a) }; @@ -94,6 +100,7 @@ beforeEach(() => { finalizeSummaryMock.mockReset(); releaseLockMock.mockReset(); uploadSummaryMock.mockReset().mockResolvedValue({ path: "insert", summaryLength: 80, descLength: 15, sql: "..." }); + embedSummaryMock.mockReset().mockResolvedValue([0.1, 0.2, 0.3]); execFileSyncMock.mockReset(); }); diff --git a/claude-code/tests/deeplake-api.test.ts b/claude-code/tests/deeplake-api.test.ts index e3e1b59..a4b9099 100644 --- a/claude-code/tests/deeplake-api.test.ts +++ b/claude-code/tests/deeplake-api.test.ts @@ -406,15 +406,29 @@ describe("DeeplakeApi.ensureTable", () => { expect(createSql).toContain("USING deeplake"); }); - it("does nothing when table already exists", async () => { - // BM25 index creation is disabled (oid bug), so ensureTable only calls listTables + it("issues ALTER TABLE ADD COLUMN IF NOT EXISTS for the embedding column when the table already exists", async () => { mockFetch.mockResolvedValueOnce({ ok: true, status: 200, json: async () => ({ tables: [{ table_name: "my_table" }] }), }); + mockFetch.mockResolvedValueOnce(jsonResponse({})); const api = makeApi("my_table"); await api.ensureTable(); - expect(mockFetch).toHaveBeenCalledOnce(); // only listTables, no CREATE + expect(mockFetch).toHaveBeenCalledTimes(2); // listTables + ALTER + const alterSql = JSON.parse(mockFetch.mock.calls[1][1].body).query; + expect(alterSql).toContain("ALTER TABLE"); + expect(alterSql).toContain("my_table"); + expect(alterSql).toContain("ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]"); + }); + + it("swallows ALTER TABLE errors (older backend without ADD COLUMN IF NOT EXISTS)", async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, status: 200, + json: async () => ({ tables: [{ table_name: "my_table" }] }), + }); + mockFetch.mockResolvedValueOnce(jsonResponse("syntax error", 400)); + const api = makeApi("my_table"); + await expect(api.ensureTable()).resolves.toBeUndefined(); }); it("creates table with custom name", async () => { @@ -434,18 +448,22 @@ describe("DeeplakeApi.ensureTable", () => { ok: true, status: 200, json: async () => ({ tables: [{ table_name: "memory" }] }), }); - mockFetch.mockResolvedValueOnce(jsonResponse({})); - mockFetch.mockResolvedValueOnce(jsonResponse({})); + mockFetch.mockResolvedValueOnce(jsonResponse({})); // ALTER memory + mockFetch.mockResolvedValueOnce(jsonResponse({})); // CREATE sessions + mockFetch.mockResolvedValueOnce(jsonResponse({})); // CREATE INDEX const api = makeApi("memory"); await api.ensureTable(); await api.ensureSessionsTable("sessions"); - expect(mockFetch).toHaveBeenCalledTimes(3); - const createSql = JSON.parse(mockFetch.mock.calls[1][1].body).query; + expect(mockFetch).toHaveBeenCalledTimes(4); + const alterSql = JSON.parse(mockFetch.mock.calls[1][1].body).query; + expect(alterSql).toContain("ALTER TABLE"); + expect(alterSql).toContain("summary_embedding"); + const createSql = JSON.parse(mockFetch.mock.calls[2][1].body).query; expect(createSql).toContain("CREATE TABLE IF NOT EXISTS"); expect(createSql).toContain("sessions"); - const indexSql = JSON.parse(mockFetch.mock.calls[2][1].body).query; + const indexSql = JSON.parse(mockFetch.mock.calls[3][1].body).query; expect(indexSql).toContain("CREATE INDEX IF NOT EXISTS"); expect(indexSql).toContain("\"path\""); expect(indexSql).toContain("\"creation_date\""); @@ -475,16 +493,20 @@ describe("DeeplakeApi.ensureSessionsTable", () => { expect(indexSql).toContain("(\"path\", \"creation_date\")"); }); - it("ensures the lookup index when sessions table already exists", async () => { + it("adds message_embedding column and ensures the lookup index when sessions table already exists", async () => { mockFetch.mockResolvedValueOnce({ ok: true, status: 200, json: async () => ({ tables: [{ table_name: "sessions" }] }), }); - mockFetch.mockResolvedValueOnce(jsonResponse({})); + mockFetch.mockResolvedValueOnce(jsonResponse({})); // ALTER + mockFetch.mockResolvedValueOnce(jsonResponse({})); // CREATE INDEX const api = makeApi(); await api.ensureSessionsTable("sessions"); - expect(mockFetch).toHaveBeenCalledTimes(2); - const indexSql = JSON.parse(mockFetch.mock.calls[1][1].body).query; + expect(mockFetch).toHaveBeenCalledTimes(3); + const alterSql = JSON.parse(mockFetch.mock.calls[1][1].body).query; + expect(alterSql).toContain("ALTER TABLE"); + expect(alterSql).toContain("message_embedding FLOAT4[]"); + const indexSql = JSON.parse(mockFetch.mock.calls[2][1].body).query; expect(indexSql).toContain("CREATE INDEX IF NOT EXISTS"); }); @@ -493,11 +515,12 @@ describe("DeeplakeApi.ensureSessionsTable", () => { ok: true, status: 200, json: async () => ({ tables: [{ table_name: "sessions" }] }), }); + mockFetch.mockResolvedValueOnce(jsonResponse({})); // ALTER ok mockFetch.mockResolvedValueOnce(jsonResponse("forbidden", 403)); const api = makeApi(); await expect(api.ensureSessionsTable("sessions")).resolves.toBeUndefined(); - expect(mockFetch).toHaveBeenCalledTimes(2); + expect(mockFetch).toHaveBeenCalledTimes(3); }); it("treats duplicate concurrent index creation errors as success and records a local marker", async () => { @@ -505,15 +528,23 @@ describe("DeeplakeApi.ensureSessionsTable", () => { ok: true, status: 200, json: async () => ({ tables: [{ table_name: "sessions" }] }), }); + mockFetch.mockResolvedValueOnce(jsonResponse({})); // ALTER ok mockFetch.mockResolvedValueOnce(jsonResponse("duplicate key value violates unique constraint \"pg_class_relname_nsp_index\"", 400)); const api = makeApi(); await expect(api.ensureSessionsTable("sessions")).resolves.toBeUndefined(); mockFetch.mockReset(); + mockFetch.mockResolvedValueOnce(jsonResponse({})); // ALTER re-runs on 2nd call await api.ensureSessionsTable("sessions"); - expect(mockFetch).not.toHaveBeenCalled(); + // On the second call: listTables is cached, the index marker short- + // circuits the CREATE INDEX, but ALTER TABLE ADD COLUMN IF NOT EXISTS + // still fires once — it's cheap (the backend no-ops when the column + // already exists) and leaves the migration idempotent across versions. + expect(mockFetch).toHaveBeenCalledTimes(1); + const secondCallSql = JSON.parse(mockFetch.mock.calls[0][1].body).query; + expect(secondCallSql).toContain("ALTER TABLE"); }); }); diff --git a/claude-code/tests/deeplake-fs.test.ts b/claude-code/tests/deeplake-fs.test.ts index 455b86a..f3685e2 100644 --- a/claude-code/tests/deeplake-fs.test.ts +++ b/claude-code/tests/deeplake-fs.test.ts @@ -138,7 +138,9 @@ function makeClient(seed: Record = {}) { // Parse columns and values positionally const colsPart = sql.match(/\(([^)]+)\)\s+VALUES/)?.[1] ?? ""; const colsList = colsPart.split(",").map(c => c.trim()); - // Extract all quoted values from VALUES(...) + // Extract all values from VALUES(...): strings, integers, + // unquoted NULL, and ARRAY[...]::float4[] literals. Each value + // becomes one slot so positional column mapping stays correct. const valsStr = valuesMatch[1]; const allVals: string[] = []; let i = 0; @@ -158,6 +160,24 @@ function makeClient(seed: Record = {}) { const m = valsStr.slice(i).match(/^(\d+)/); if (m) { allVals.push(m[1]); i += m[1].length; } else i++; + } else if (valsStr.slice(i, i + 4).toUpperCase() === "NULL") { + allVals.push(""); + i += 4; + } else if (valsStr.slice(i, i + 6).toUpperCase() === "ARRAY[") { + // Consume up to the matching ']' and optional ::float4[] cast + let depth = 1; + let end = i + 6; + while (end < valsStr.length && depth > 0) { + if (valsStr[end] === "[") depth++; + else if (valsStr[end] === "]") depth--; + end++; + } + // Skip optional ::float4[] cast + const rest = valsStr.slice(end); + const castMatch = rest.match(/^::float4\[\]/i); + if (castMatch) end += castMatch[0].length; + allVals.push(valsStr.slice(i, end)); + i = end; } else { i++; } } // Map column names to values @@ -602,10 +622,10 @@ describe("prefetch", () => { ensureTable: vi.fn().mockResolvedValue(undefined), query: vi.fn(async (sql: string) => { if (sql.includes("SELECT path, size_bytes, mime_type")) return []; - if (sql.includes("SELECT path, SUM(size_bytes) as total_size")) { + if (sql.includes("SELECT path, MAX(size_bytes) as total_size")) { return [...sessionMessages.entries()].map(([path, rows]) => ({ path, - total_size: rows.reduce((sum, row) => sum + Buffer.byteLength(row.message, "utf-8"), 0), + total_size: Math.max(...rows.map((row) => Buffer.byteLength(row.message, "utf-8"))), })); } if (sql.includes("SELECT path, message, creation_date")) { @@ -809,7 +829,9 @@ describe("virtual index.md", () => { ]); const content = await fs.readFile("/index.md"); expect(content).toContain("# Session Index"); - expect(content).toContain("| Session | Conversation | Created | Last Updated | Project | Description |"); + expect(content).toContain("## memory"); + expect(content).toContain("## sessions"); + expect(content).toContain("| Session | Created | Last Updated | Project | Description |"); expect(content).toContain("aaa-111"); expect(content).toContain("bbb-222"); expect(content).toContain("my-project"); @@ -864,8 +886,9 @@ describe("virtual index.md", () => { const { fs } = await makeFs({}, "/"); const content = await fs.readFile("/index.md"); expect(content).toContain("# Session Index"); - expect(content).toContain("| Session | Conversation | Created | Last Updated | Project | Description |"); - // No data rows + expect(content).toContain("## memory"); + expect(content).toContain("_(empty — no summaries ingested yet)_"); + // No data rows in memory section const lines = content.split("\n").filter(l => l.startsWith("| [")); expect(lines.length).toBe(0); }); diff --git a/claude-code/tests/embedding-sql.test.ts b/claude-code/tests/embedding-sql.test.ts new file mode 100644 index 0000000..a4d469f --- /dev/null +++ b/claude-code/tests/embedding-sql.test.ts @@ -0,0 +1,42 @@ +import { describe, it, expect } from "vitest"; +import { embeddingSqlLiteral } from "../../src/embeddings/sql.js"; + +describe("embeddingSqlLiteral", () => { + it("returns NULL for null input", () => { + expect(embeddingSqlLiteral(null)).toBe("NULL"); + }); + + it("returns NULL for undefined", () => { + expect(embeddingSqlLiteral(undefined)).toBe("NULL"); + }); + + it("returns NULL for empty array", () => { + expect(embeddingSqlLiteral([])).toBe("NULL"); + }); + + it("returns ARRAY[...]::float4[] for a vector", () => { + expect(embeddingSqlLiteral([0.1, 0.2, -0.3])).toBe("ARRAY[0.1,0.2,-0.3]::float4[]"); + }); + + it("returns NULL if any element is NaN / Infinity", () => { + expect(embeddingSqlLiteral([0.1, NaN, 0.3])).toBe("NULL"); + expect(embeddingSqlLiteral([0.1, Infinity, 0.3])).toBe("NULL"); + expect(embeddingSqlLiteral([-Infinity, 0.1])).toBe("NULL"); + }); + + it("uses shortest round-trip representation (no toFixed truncation)", () => { + // A value that toFixed(6) would round is preserved + const vec = [0.123456789]; + expect(embeddingSqlLiteral(vec)).toBe("ARRAY[0.123456789]::float4[]"); + }); + + it("handles a realistic 768-dim vector without truncation", () => { + const vec = Array.from({ length: 768 }, (_, i) => i / 1000); + const sql = embeddingSqlLiteral(vec); + expect(sql.startsWith("ARRAY[")).toBe(true); + expect(sql.endsWith("]::float4[]")).toBe(true); + // Count commas → 767 separators → 768 elements + const commas = (sql.match(/,/g) ?? []).length; + expect(commas).toBe(767); + }); +}); diff --git a/claude-code/tests/embeddings-client.test.ts b/claude-code/tests/embeddings-client.test.ts new file mode 100644 index 0000000..07c0d57 --- /dev/null +++ b/claude-code/tests/embeddings-client.test.ts @@ -0,0 +1,330 @@ +// Unit tests for the embedding client — avoid loading the model by spinning up +// a tiny fake daemon that speaks the protocol. + +import { describe, it, expect, afterEach } from "vitest"; +import { createServer, type Server, type Socket } from "node:net"; +import { mkdtempSync, rmSync, existsSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { execSync } from "node:child_process"; +import { EmbedClient, getEmbedClient } from "../../src/embeddings/client.js"; +import type { DaemonRequest, DaemonResponse } from "../../src/embeddings/protocol.js"; + +let servers: Server[] = []; +let tmpDirs: string[] = []; + +afterEach(() => { + for (const s of servers) try { s.close(); } catch { /* */ } + servers = []; + for (const d of tmpDirs) try { rmSync(d, { recursive: true, force: true }); } catch { /* */ } + tmpDirs = []; +}); + +function makeTmpDir(): string { + const d = mkdtempSync(join(tmpdir(), "hvm-embed-test-")); + tmpDirs.push(d); + return d; +} + +async function startFakeDaemon(dir: string, handler: (req: DaemonRequest) => DaemonResponse): Promise { + const uid = String(process.getuid?.() ?? "test"); + const sockPath = join(dir, `hivemind-embed-${uid}.sock`); + const srv = createServer((sock: Socket) => { + let buf = ""; + sock.setEncoding("utf-8"); + sock.on("data", (chunk: string) => { + buf += chunk; + let nl: number; + while ((nl = buf.indexOf("\n")) !== -1) { + const line = buf.slice(0, nl); + buf = buf.slice(nl + 1); + if (!line) continue; + const req = JSON.parse(line) as DaemonRequest; + const resp = handler(req); + sock.write(JSON.stringify(resp) + "\n"); + } + }); + sock.on("error", () => { /* */ }); + }); + servers.push(srv); + await new Promise((resolve) => srv.listen(sockPath, resolve)); + return srv; +} + +describe("EmbedClient", () => { + it("returns the embedding vector when the daemon responds", async () => { + const dir = makeTmpDir(); + await startFakeDaemon(dir, (req) => { + if (req.op === "embed") return { id: req.id, embedding: [0.1, 0.2, 0.3] }; + return { id: req.id, ready: true }; + }); + const client = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false }); + const vec = await client.embed("hello", "document"); + expect(vec).toEqual([0.1, 0.2, 0.3]); + }); + + it("returns null when the daemon returns an error", async () => { + const dir = makeTmpDir(); + await startFakeDaemon(dir, (req) => ({ id: req.id, error: "boom" })); + const client = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false }); + const vec = await client.embed("hello"); + expect(vec).toBeNull(); + }); + + it("returns null when no daemon is running and autoSpawn is disabled", async () => { + const dir = makeTmpDir(); + const client = new EmbedClient({ socketDir: dir, timeoutMs: 100, autoSpawn: false }); + const vec = await client.embed("hello"); + expect(vec).toBeNull(); + }); + + it("does not create a duplicate pidfile under concurrent first-call race", async () => { + const dir = makeTmpDir(); + const client1 = new EmbedClient({ + socketDir: dir, + timeoutMs: 50, + autoSpawn: true, + daemonEntry: "/nonexistent/daemon.js", // guarantee spawn can't succeed + }); + const client2 = new EmbedClient({ + socketDir: dir, + timeoutMs: 50, + autoSpawn: true, + daemonEntry: "/nonexistent/daemon.js", + }); + // Both clients see no socket, both try spawnDaemon. O_EXCL guarantees only + // one actually tries to spawn. Both return null because no daemon comes up. + const [a, b] = await Promise.all([ + client1.embed("one"), + client2.embed("two"), + ]); + expect(a).toBeNull(); + expect(b).toBeNull(); + // pidfile should have been cleaned up when spawn couldn't find the entry. + const uid = String(process.getuid?.() ?? "test"); + expect(existsSync(join(dir, `hivemind-embed-${uid}.pid`))).toBe(false); + }); + + it("round-trips multiple requests on the same client without leaking sockets", async () => { + const dir = makeTmpDir(); + await startFakeDaemon(dir, (req) => ({ id: req.id, embedding: [Math.random()] })); + const client = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false }); + const results = await Promise.all([ + client.embed("a"), + client.embed("b"), + client.embed("c"), + ]); + expect(results.every((r) => r !== null && r.length === 1)).toBe(true); + }); + + it("warmup() returns true when the daemon is already listening", async () => { + const dir = makeTmpDir(); + await startFakeDaemon(dir, (req) => ({ id: req.id, ready: true })); + const client = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false }); + const ok = await client.warmup(); + expect(ok).toBe(true); + }); + + it("warmup() returns false when no daemon and autoSpawn is disabled", async () => { + const dir = makeTmpDir(); + const client = new EmbedClient({ socketDir: dir, timeoutMs: 100, autoSpawn: false }); + const ok = await client.warmup(); + expect(ok).toBe(false); + }); + + it("warmup() returns false when autoSpawn is on but entry cannot be launched", async () => { + const dir = makeTmpDir(); + const client = new EmbedClient({ + socketDir: dir, + timeoutMs: 100, + autoSpawn: true, + daemonEntry: "/nonexistent/daemon.js", + spawnWaitMs: 150, + }); + const ok = await client.warmup(); + expect(ok).toBe(false); + }); + + it("cleans up a stale pidfile (dead PID) before trying to spawn", async () => { + const dir = makeTmpDir(); + const uid = String(process.getuid?.() ?? "test"); + const pidPath = join(dir, `hivemind-embed-${uid}.pid`); + // Write a PID guaranteed-dead: 0x7FFFFFFF is not a plausible live PID on Linux. + writeFileSync(pidPath, "2147483646"); + + const client = new EmbedClient({ + socketDir: dir, + timeoutMs: 50, + autoSpawn: true, + daemonEntry: "/nonexistent/daemon.js", + }); + const vec = await client.embed("x"); + expect(vec).toBeNull(); + // Client should have cleaned up the pidfile after detecting the entry is missing. + expect(existsSync(pidPath)).toBe(false); + }); + + it("leaves an alive-PID pidfile alone (treats the daemon as still starting)", async () => { + const dir = makeTmpDir(); + const uid = String(process.getuid?.() ?? "test"); + const pidPath = join(dir, `hivemind-embed-${uid}.pid`); + // Our own PID is alive → isPidFileStale() should return false. + writeFileSync(pidPath, String(process.pid)); + + const client = new EmbedClient({ + socketDir: dir, + timeoutMs: 50, + autoSpawn: true, + daemonEntry: "/nonexistent/daemon.js", + }); + const vec = await client.embed("x"); + expect(vec).toBeNull(); + // Pidfile is still there because client saw it as a live owner, not stale. + expect(existsSync(pidPath)).toBe(true); + }); + + it("treats a garbage pidfile as stale and removes it", async () => { + const dir = makeTmpDir(); + const uid = String(process.getuid?.() ?? "test"); + const pidPath = join(dir, `hivemind-embed-${uid}.pid`); + writeFileSync(pidPath, "not-a-number"); + + const client = new EmbedClient({ + socketDir: dir, + timeoutMs: 50, + autoSpawn: true, + daemonEntry: "/nonexistent/daemon.js", + }); + const vec = await client.embed("x"); + expect(vec).toBeNull(); + expect(existsSync(pidPath)).toBe(false); + }); + + it("returns null when the socket closes mid-request", async () => { + const dir = makeTmpDir(); + const uid = String(process.getuid?.() ?? "test"); + const sockPath = join(dir, `hivemind-embed-${uid}.sock`); + const srv = createServer((sock: Socket) => { + // Immediately destroy the connection after accept so sendAndWait errors. + sock.destroy(); + }); + servers.push(srv); + await new Promise((resolve) => srv.listen(sockPath, resolve)); + + const client = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false }); + const vec = await client.embed("boom"); + expect(vec).toBeNull(); + }); + + it("returns null when the daemon writes malformed JSON", async () => { + const dir = makeTmpDir(); + const uid = String(process.getuid?.() ?? "test"); + const sockPath = join(dir, `hivemind-embed-${uid}.sock`); + const srv = createServer((sock: Socket) => { + sock.setEncoding("utf-8"); + sock.on("data", () => { + sock.write("not-json\n"); + }); + }); + servers.push(srv); + await new Promise((resolve) => srv.listen(sockPath, resolve)); + + const client = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false }); + const vec = await client.embed("boom"); + expect(vec).toBeNull(); + }); + + it("returns null on request timeout (daemon accepts but never replies)", async () => { + const dir = makeTmpDir(); + const uid = String(process.getuid?.() ?? "test"); + const sockPath = join(dir, `hivemind-embed-${uid}.sock`); + const srv = createServer((_sock: Socket) => { + // Accept the connection but never send anything back. + }); + servers.push(srv); + await new Promise((resolve) => srv.listen(sockPath, resolve)); + + const client = new EmbedClient({ socketDir: dir, timeoutMs: 50, autoSpawn: false }); + const vec = await client.embed("boom"); + expect(vec).toBeNull(); + }); + + it("getEmbedClient() returns a cached singleton", () => { + const a = getEmbedClient(); + const b = getEmbedClient(); + expect(a).toBe(b); + }); + + it("uses default option values when constructed with no arguments", () => { + // Just instantiating exercises every `opts.x ?? default` branch. + const c = new EmbedClient(); + expect(c).toBeInstanceOf(EmbedClient); + }); + + it("defaults the embed 'kind' argument to document when omitted", async () => { + const dir = makeTmpDir(); + const kinds: string[] = []; + await startFakeDaemon(dir, (req) => { + if (req.op === "embed") kinds.push(req.kind); + return { id: req.id, embedding: [0.5] }; + }); + const client = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false }); + await client.embed("hello"); // no kind + expect(kinds).toEqual(["document"]); + }); + + it("falls back to HIVEMIND_EMBED_DAEMON env when daemonEntry option is absent", () => { + const prev = process.env.HIVEMIND_EMBED_DAEMON; + process.env.HIVEMIND_EMBED_DAEMON = "/from/env.js"; + try { + const c = new EmbedClient({ socketDir: makeTmpDir(), autoSpawn: false }); + // We can't read the private field directly; just assert construction succeeded. + expect(c).toBeInstanceOf(EmbedClient); + } finally { + if (prev === undefined) delete process.env.HIVEMIND_EMBED_DAEMON; + else process.env.HIVEMIND_EMBED_DAEMON = prev; + } + }); + + it("warmup() succeeds after auto-spawning a fake daemon entry", async () => { + const dir = makeTmpDir(); + const uid = String(process.getuid?.() ?? "test"); + const sockPath = join(dir, `hivemind-embed-${uid}.sock`); + // Write a tiny daemon script that binds the expected socket and answers pings. + const daemonScript = join(dir, "fake-daemon.js"); + writeFileSync(daemonScript, ` + const net = require("node:net"); + const srv = net.createServer((s) => { + s.setEncoding("utf-8"); + let buf = ""; + s.on("data", (c) => { + buf += c; + let nl; + while ((nl = buf.indexOf("\\n")) !== -1) { + const line = buf.slice(0, nl); + buf = buf.slice(nl + 1); + try { + const req = JSON.parse(line); + s.write(JSON.stringify({ id: req.id, ready: true }) + "\\n"); + } catch {} + } + }); + }); + srv.listen(${JSON.stringify(sockPath)}); + setTimeout(() => srv.close(), 3000); + `); + + const client = new EmbedClient({ + socketDir: dir, + timeoutMs: 500, + autoSpawn: true, + daemonEntry: daemonScript, + spawnWaitMs: 2000, + }); + const ok = await client.warmup(); + expect(ok).toBe(true); + + // Cleanup the spawned daemon process. + try { execSync(`pkill -f ${daemonScript}`); } catch { /* already exited */ } + }); +}); diff --git a/claude-code/tests/embeddings-daemon.test.ts b/claude-code/tests/embeddings-daemon.test.ts new file mode 100644 index 0000000..2917816 --- /dev/null +++ b/claude-code/tests/embeddings-daemon.test.ts @@ -0,0 +1,263 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { connect } from "node:net"; +import { mkdtempSync, rmSync, existsSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +// Mock NomicEmbedder so the daemon doesn't pull in @huggingface/transformers. +// The daemon talks to the embedder via two methods only: load() and embed(). +// The `embedMode` global lets an individual test flip behavior: "ok" returns +// a vector, "throw" makes embed() reject — drives the dispatch-error branch. +(globalThis as any).__embedMode = "ok"; + +vi.mock("../../src/embeddings/nomic.js", () => { + class MockNomicEmbedder { + repo: string; + dims: number; + dtype: string; + constructor(opts: any = {}) { + this.repo = opts.repo ?? "mock-repo"; + this.dims = opts.dims ?? 768; + this.dtype = opts.dtype ?? "q8"; + } + async load() { /* no-op */ } + async embed(_text: string, _kind?: string) { + if ((globalThis as any).__embedMode === "throw") { + throw new Error("forced embed failure"); + } + return [0.1, 0.2, 0.3]; + } + async embedBatch(texts: string[], _kind?: string) { + return texts.map(() => [0.1, 0.2, 0.3]); + } + } + return { NomicEmbedder: MockNomicEmbedder }; +}); + +import { EmbedDaemon } from "../../src/embeddings/daemon.js"; + +function sendLine(socketPath: string, req: object): Promise { + return new Promise((resolve, reject) => { + const sock = connect(socketPath); + let buf = ""; + const to = setTimeout(() => { sock.destroy(); reject(new Error("timeout")); }, 2000); + sock.setEncoding("utf-8"); + sock.on("connect", () => sock.write(JSON.stringify(req) + "\n")); + sock.on("data", (chunk: string) => { + buf += chunk; + const nl = buf.indexOf("\n"); + if (nl === -1) return; + clearTimeout(to); + sock.end(); + try { resolve(JSON.parse(buf.slice(0, nl))); } catch (e) { reject(e); } + }); + sock.on("error", (e) => { clearTimeout(to); reject(e); }); + }); +} + +describe("EmbedDaemon", () => { + let dir: string; + let daemon: EmbedDaemon | null = null; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "hvm-daemon-test-")); + }); + + afterEach(() => { + try { daemon?.shutdown(); } catch { /* shutdown calls process.exit, ignore */ } + daemon = null; + try { rmSync(dir, { recursive: true, force: true }); } catch { /* */ } + (globalThis as any).__embedMode = "ok"; + }); + + it("answers a ping with the model + dims metadata", async () => { + // process.exit inside shutdown would terminate the test runner; stub it. + const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + daemon = new EmbedDaemon({ socketDir: dir, idleTimeoutMs: 60_000, dims: 128 }); + await daemon.start(); + + const uid = String(process.getuid?.() ?? "test"); + const sock = join(dir, `hivemind-embed-${uid}.sock`); + const resp = await sendLine(sock, { op: "ping", id: "p1" }); + expect(resp.id).toBe("p1"); + expect(resp.ready).toBe(true); + expect(resp.dims).toBe(128); + + exitSpy.mockRestore(); + }); + + it("answers an embed request with the vector from the mocked embedder", async () => { + const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + daemon = new EmbedDaemon({ socketDir: dir, idleTimeoutMs: 60_000 }); + await daemon.start(); + + const uid = String(process.getuid?.() ?? "test"); + const sock = join(dir, `hivemind-embed-${uid}.sock`); + const resp = await sendLine(sock, { op: "embed", id: "e1", kind: "document", text: "hello" }); + expect(resp.id).toBe("e1"); + expect(resp.embedding).toEqual([0.1, 0.2, 0.3]); + + exitSpy.mockRestore(); + }); + + it("returns { error } for unknown ops", async () => { + const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + daemon = new EmbedDaemon({ socketDir: dir, idleTimeoutMs: 60_000 }); + await daemon.start(); + + const uid = String(process.getuid?.() ?? "test"); + const sock = join(dir, `hivemind-embed-${uid}.sock`); + const resp = await sendLine(sock, { op: "bogus", id: "x" }); + expect(resp.error).toContain("unknown op"); + + exitSpy.mockRestore(); + }); + + it("writes a pidfile with the daemon's own PID on startup", async () => { + const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + daemon = new EmbedDaemon({ socketDir: dir, idleTimeoutMs: 60_000 }); + await daemon.start(); + + const uid = String(process.getuid?.() ?? "test"); + const pidPath = join(dir, `hivemind-embed-${uid}.pid`); + const { readFileSync } = await import("node:fs"); + const pid = Number(readFileSync(pidPath, "utf-8").trim()); + expect(pid).toBe(process.pid); + + exitSpy.mockRestore(); + }); + + it("unlinks a stale socket on startup before re-binding", async () => { + const uid = String(process.getuid?.() ?? "test"); + const sockPath = join(dir, `hivemind-embed-${uid}.sock`); + // Pre-create a stale file at the socket path. + writeFileSync(sockPath, "stale"); + expect(existsSync(sockPath)).toBe(true); + + const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + daemon = new EmbedDaemon({ socketDir: dir, idleTimeoutMs: 60_000 }); + await daemon.start(); + // Now it's a live Unix socket; stat would say it's a socket not a regular file. + expect(existsSync(sockPath)).toBe(true); + + exitSpy.mockRestore(); + }); + + it("idle timer triggers shutdown after the configured window", async () => { + const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + daemon = new EmbedDaemon({ socketDir: dir, idleTimeoutMs: 50 }); + await daemon.start(); + await new Promise(r => setTimeout(r, 120)); + // shutdown called via process.exit stub (spyed above) — our exit count > 0. + expect(exitSpy).toHaveBeenCalled(); + exitSpy.mockRestore(); + }); + + it("returns { error } when the embedder throws during an embed request", async () => { + const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + (globalThis as any).__embedMode = "throw"; + daemon = new EmbedDaemon({ socketDir: dir, idleTimeoutMs: 60_000 }); + await daemon.start(); + + const uid = String(process.getuid?.() ?? "test"); + const sock = join(dir, `hivemind-embed-${uid}.sock`); + const resp = await sendLine(sock, { op: "embed", id: "e2", kind: "document", text: "hi" }); + expect(resp.id).toBe("e2"); + expect(resp.error).toContain("forced embed failure"); + + exitSpy.mockRestore(); + }); + + it("constructs with default options (no opts object passed)", () => { + // Exercise the constructor's `opts = {}` default + every `??` fallback. + const d = new EmbedDaemon(); + expect(d).toBeInstanceOf(EmbedDaemon); + }); + + it("skips empty lines between valid requests", async () => { + const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + daemon = new EmbedDaemon({ socketDir: dir, idleTimeoutMs: 60_000 }); + await daemon.start(); + + const uid = String(process.getuid?.() ?? "test"); + const sockPath = join(dir, `hivemind-embed-${uid}.sock`); + await new Promise((resolve, reject) => { + const sock = connect(sockPath); + sock.setEncoding("utf-8"); + let buf = ""; + sock.on("connect", () => { + // Send blank lines first — they hit the `line.length === 0` branch. + sock.write("\n\n"); + sock.write(JSON.stringify({ op: "ping", id: "z" }) + "\n"); + }); + sock.on("data", (c: string) => { + buf += c; + const nl = buf.indexOf("\n"); + if (nl !== -1) { + const resp = JSON.parse(buf.slice(0, nl)); + expect(resp.id).toBe("z"); + sock.end(); + resolve(); + } + }); + sock.on("error", reject); + }); + exitSpy.mockRestore(); + }); + + it("survives a client that disconnects abruptly mid-session", async () => { + const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + daemon = new EmbedDaemon({ socketDir: dir, idleTimeoutMs: 60_000 }); + await daemon.start(); + + const uid = String(process.getuid?.() ?? "test"); + const sockPath = join(dir, `hivemind-embed-${uid}.sock`); + await new Promise((resolve) => { + const sock = connect(sockPath); + sock.on("error", () => { /* swallow — we intentionally destroy below */ }); + sock.on("connect", () => { + // Destroying a freshly connected socket should make the server's + // read side emit either `end` or `error` — either way the daemon + // should survive and keep serving. We test the survival below. + sock.destroy(); + resolve(); + }); + }); + // Follow-up ping should still work — the daemon didn't crash. + const sockPathStr = join(dir, `hivemind-embed-${uid}.sock`); + const resp = await sendLine(sockPathStr, { op: "ping", id: "after" }); + expect(resp.id).toBe("after"); + exitSpy.mockRestore(); + }); + + it("handles malformed JSON lines without crashing the daemon", async () => { + const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + daemon = new EmbedDaemon({ socketDir: dir, idleTimeoutMs: 60_000 }); + await daemon.start(); + + const uid = String(process.getuid?.() ?? "test"); + const sockPath = join(dir, `hivemind-embed-${uid}.sock`); + // Write a bad line then a good one on the same connection. + await new Promise((resolve, reject) => { + const sock = connect(sockPath); + sock.setEncoding("utf-8"); + let buf = ""; + sock.on("connect", () => { + sock.write("not-json\n"); + sock.write(JSON.stringify({ op: "ping", id: "ok" }) + "\n"); + }); + sock.on("data", (c: string) => { + buf += c; + const nl = buf.indexOf("\n"); + if (nl !== -1) { + const resp = JSON.parse(buf.slice(0, nl)); + expect(resp.id).toBe("ok"); + sock.end(); + resolve(); + } + }); + sock.on("error", reject); + }); + exitSpy.mockRestore(); + }); +}); diff --git a/claude-code/tests/embeddings-disable.test.ts b/claude-code/tests/embeddings-disable.test.ts new file mode 100644 index 0000000..9a8d527 --- /dev/null +++ b/claude-code/tests/embeddings-disable.test.ts @@ -0,0 +1,38 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { embeddingsDisabled } from "../../src/embeddings/disable.js"; + +describe("embeddingsDisabled()", () => { + const original = process.env.HIVEMIND_EMBEDDINGS; + + beforeEach(() => { + delete process.env.HIVEMIND_EMBEDDINGS; + }); + + afterEach(() => { + if (original === undefined) delete process.env.HIVEMIND_EMBEDDINGS; + else process.env.HIVEMIND_EMBEDDINGS = original; + }); + + it("returns false when the env var is unset (default behaviour)", () => { + expect(embeddingsDisabled()).toBe(false); + }); + + it("returns true only when explicitly set to the string 'false'", () => { + process.env.HIVEMIND_EMBEDDINGS = "false"; + expect(embeddingsDisabled()).toBe(true); + }); + + it("stays off for any non-'false' truthy value (intentional: avoid surprise kills)", () => { + process.env.HIVEMIND_EMBEDDINGS = "0"; + expect(embeddingsDisabled()).toBe(false); + + process.env.HIVEMIND_EMBEDDINGS = "no"; + expect(embeddingsDisabled()).toBe(false); + + process.env.HIVEMIND_EMBEDDINGS = "true"; + expect(embeddingsDisabled()).toBe(false); + + process.env.HIVEMIND_EMBEDDINGS = ""; + expect(embeddingsDisabled()).toBe(false); + }); +}); diff --git a/claude-code/tests/embeddings-nomic.test.ts b/claude-code/tests/embeddings-nomic.test.ts new file mode 100644 index 0000000..aa4300c --- /dev/null +++ b/claude-code/tests/embeddings-nomic.test.ts @@ -0,0 +1,149 @@ +import { describe, it, expect, vi } from "vitest"; +import { NomicEmbedder } from "../../src/embeddings/nomic.js"; + +// Mock the heavy transformers import so these tests don't pull in +// onnxruntime-node or download any model weights. `load()` uses +// `await import("@huggingface/transformers")` — vi.mock intercepts. +vi.mock("@huggingface/transformers", () => { + const embed = vi.fn((input: string | string[], _opts: Record) => { + const texts = Array.isArray(input) ? input : [input]; + // Return deterministic per-input vectors: 4 floats per text. + const out: number[] = []; + for (let i = 0; i < texts.length; i++) { + out.push(0.1 + i, 0.2 + i, 0.3 + i, 0.4 + i); + } + return Promise.resolve({ data: out }); + }); + return { + env: { allowLocalModels: false, useFSCache: false }, + pipeline: vi.fn(async () => embed), + }; +}); + +describe("NomicEmbedder", () => { + it("loads lazily and reuses the pipeline across calls", async () => { + const e = new NomicEmbedder({ dims: 4 }); + await e.load(); + await e.load(); // second call is a no-op (cached) + // If load() didn't memoize, pipeline() would be invoked twice; the + // mock would return a fresh spy whose call counts would differ. + const mod: any = await import("@huggingface/transformers"); + expect((mod.pipeline as any).mock.calls.length).toBe(1); + }); + + it("embeds a document with the search_document: prefix", async () => { + const e = new NomicEmbedder({ dims: 4 }); + const v = await e.embed("hello", "document"); + expect(v).toHaveLength(4); + const mod: any = await import("@huggingface/transformers"); + const pipeline = await (mod.pipeline as any).mock.results[0].value; + const callArg = (pipeline as any).mock.calls.at(-1)[0]; + expect(callArg).toBe("search_document: hello"); + }); + + it("embeds a query with the search_query: prefix", async () => { + const e = new NomicEmbedder({ dims: 4 }); + await e.embed("q", "query"); + const mod: any = await import("@huggingface/transformers"); + const pipeline = await (mod.pipeline as any).mock.results[0].value; + const callArg = (pipeline as any).mock.calls.at(-1)[0]; + expect(callArg).toBe("search_query: q"); + }); + + it("batches inputs and splits results back into per-text vectors", async () => { + const e = new NomicEmbedder({ dims: 4 }); + const out = await e.embedBatch(["a", "b", "c"], "document"); + expect(out).toHaveLength(3); + expect(out[0]).toHaveLength(4); + expect(out[0][0]).toBeCloseTo(0.1); + expect(out[1][0]).toBeCloseTo(1.1); + expect(out[2][0]).toBeCloseTo(2.1); + }); + + it("returns [] for an empty batch without touching the pipeline", async () => { + const e = new NomicEmbedder({ dims: 4 }); + expect(await e.embedBatch([])).toEqual([]); + }); + + it("applies Matryoshka truncation when dims < full length", async () => { + const e = new NomicEmbedder({ dims: 2 }); + const v = await e.embed("x"); + expect(v).toHaveLength(2); + // Truncated + re-normalized; the raw vector was [0.1,0.2,0.3,0.4]. + // After slicing to 2 and renormalizing, |v| === 1. + const norm = Math.sqrt(v[0] * v[0] + v[1] * v[1]); + expect(norm).toBeCloseTo(1.0, 5); + }); + + it("returns vector unchanged when requested dims >= vector length", async () => { + const e = new NomicEmbedder({ dims: 100 }); + const v = await e.embed("x"); + // Mock returns 4 dims; with target 100, truncate becomes a no-op and + // the raw vector is returned verbatim (no renormalization). + expect(v).toHaveLength(4); + }); + + it("handles a zero-norm truncation without dividing by zero", async () => { + // Reach through the private helper via a custom mock that returns zeros. + const mod: any = await import("@huggingface/transformers"); + const origPipeline = mod.pipeline; + const zeroPipe = vi.fn(async () => [0, 0, 0, 0]); + const wrapped = vi.fn(() => Promise.resolve(() => Promise.resolve({ data: [0, 0, 0, 0] }))); + (mod as any).pipeline = wrapped; + try { + const e = new NomicEmbedder({ dims: 2 }); + const v = await e.embed("z"); + expect(v).toEqual([0, 0]); + } finally { + (mod as any).pipeline = origPipeline; + } + }); + + it("throws if embed is called before load resolves (defensive)", async () => { + const e = new NomicEmbedder({ dims: 4 }); + // Call load once normally to populate the pipeline. + await e.load(); + // This is the happy path; the guard message fires only on a bug. + const v = await e.embed("x"); + expect(v).toHaveLength(4); + }); + + it("defaults repo + dtype + dims without explicit options", () => { + const e = new NomicEmbedder(); + expect(e.repo).toBe("nomic-ai/nomic-embed-text-v1.5"); + expect(e.dtype).toBe("q8"); + expect(e.dims).toBe(768); + }); + + it("coalesces concurrent load() calls onto a single pipeline build", async () => { + // Replace pipeline with a slow one so the two load() calls overlap and + // the second enters the `if (this.loading) return this.loading;` branch. + const mod: any = await import("@huggingface/transformers"); + const orig = mod.pipeline; + let calls = 0; + mod.pipeline = vi.fn(async () => { + calls++; + await new Promise((r) => setTimeout(r, 30)); + return async () => ({ data: [0, 0, 0, 0] }); + }); + try { + const e = new NomicEmbedder({ dims: 4 }); + // Kick off two loads without awaiting between them. + const [a, b] = await Promise.all([e.load(), e.load()]); + expect(a).toBeUndefined(); + expect(b).toBeUndefined(); + expect(calls).toBe(1); + } finally { + mod.pipeline = orig; + } + }); + + it("embeds a query in embedBatch with the search_query prefix", async () => { + const e = new NomicEmbedder({ dims: 4 }); + await e.embedBatch(["hi"], "query"); + const mod: any = await import("@huggingface/transformers"); + const pipeline = await (mod.pipeline as any).mock.results[0].value; + const lastCall = (pipeline as any).mock.calls.at(-1)[0]; + expect(lastCall).toEqual(["search_query: hi"]); + }); +}); diff --git a/claude-code/tests/embeddings-schema.test.ts b/claude-code/tests/embeddings-schema.test.ts new file mode 100644 index 0000000..44e1b6b --- /dev/null +++ b/claude-code/tests/embeddings-schema.test.ts @@ -0,0 +1,54 @@ +// Bundle-level guard: make sure the shipped hook bundles contain the new +// embedding columns in their INSERT statements. Catches regressions where +// the schema migration is done in src/ but a bundle referencing the old +// column list remains in the shipped artifact. + +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const BUNDLE_DIRS = [ + "claude-code/bundle", + "codex/bundle", +]; + +function read(path: string): string { + return readFileSync(path, "utf-8"); +} + +describe("shipped bundles include embedding columns", () => { + for (const dir of BUNDLE_DIRS) { + it(`${dir}/capture.js writes message_embedding`, () => { + const src = read(join(dir, "capture.js")); + expect(src).toMatch(/message_embedding/); + }); + + it(`${dir}/shell/deeplake-shell.js writes summary_embedding`, () => { + const src = read(join(dir, "shell/deeplake-shell.js")); + expect(src).toMatch(/summary_embedding/); + }); + + it(`${dir} has an embed-daemon bundle`, () => { + // Just check the file exists and is non-empty — not runnable without deps. + const src = read(join(dir, "embeddings/embed-daemon.js")); + expect(src.length).toBeGreaterThan(100); + }); + } +}); + +describe("src-level schema includes new embedding columns", () => { + const apiSrc = read("src/deeplake-api.ts"); + + it("memory table CREATE includes summary_embedding FLOAT4[]", () => { + expect(apiSrc).toMatch(/summary_embedding FLOAT4\[\]/); + }); + + it("sessions table CREATE includes message_embedding FLOAT4[]", () => { + expect(apiSrc).toMatch(/message_embedding FLOAT4\[\]/); + }); + + it("embedding columns do NOT use TEXT (regression guard)", () => { + expect(apiSrc).not.toMatch(/summary_embedding TEXT/); + expect(apiSrc).not.toMatch(/message_embedding TEXT/); + }); +}); diff --git a/claude-code/tests/grep-core.test.ts b/claude-code/tests/grep-core.test.ts index eb1ef79..20a8d1d 100644 --- a/claude-code/tests/grep-core.test.ts +++ b/claude-code/tests/grep-core.test.ts @@ -46,10 +46,13 @@ describe("normalizeContent: turn-array session shape", () => { ], }); - it("emits date and speakers header", () => { + it("prefixes every turn with the session date inline", () => { const out = normalizeContent("/sessions/alice/chat_1.json", raw); - expect(out).toContain("date: 1:56 pm on 8 May, 2023"); - expect(out).toContain("speakers: Avery, Jordan"); + // Date lives inline on every turn so it survives the refineGrepMatches + // line filter — a standalone `date:` header would be stripped whenever + // the regex didn't match the header line itself. + expect(out).toContain("(1:56 pm on 8 May, 2023) [D1:1] Avery: Hey Jordan!"); + expect(out).toContain("(1:56 pm on 8 May, 2023) [D1:2] Jordan: Hi Avery."); }); it("emits one line per turn with dia_id tag", () => { @@ -66,23 +69,24 @@ describe("normalizeContent: turn-array session shape", () => { expect(out).toContain("X: "); }); - it("omits speakers header when both speaker fields are empty", () => { + it("skips the date prefix when date_time is absent", () => { const raw = JSON.stringify({ turns: [{ speaker: "A", text: "hi" }], - speakers: { speaker_a: "", speaker_b: "" }, }); const out = normalizeContent("/sessions/alice/chat_1.json", raw); - expect(out).not.toContain("speakers:"); + // No leading "(...)" — the turn line starts with the dia_id or speaker. expect(out).toContain("A: hi"); + expect(out).not.toMatch(/^\(/); }); - it("emits only speaker_a when speaker_b is missing", () => { + it("still emits turn lines when only one speaker is set (date still inlined)", () => { const raw = JSON.stringify({ + date_time: "1:56 pm on 8 May, 2023", turns: [{ speaker: "A", text: "hi" }], speakers: { speaker_a: "Alice" }, }); const out = normalizeContent("/sessions/alice/chat_1.json", raw); - expect(out).toContain("speakers: Alice"); + expect(out).toContain("(1:56 pm on 8 May, 2023) A: hi"); }); it("falls back speaker->name when speaker field is absent on a turn", () => { @@ -782,22 +786,41 @@ describe("grepBothTables", () => { const [sql] = api.query.mock.calls.map((c: unknown[]) => c[0] as string); expect(sql).not.toContain("summary::text LIKE"); expect(sql).not.toContain("message::text LIKE"); + expect(sql).not.toContain("summary::text ILIKE"); + expect(sql).not.toContain("message::text ILIKE"); }); it("adds a safe literal prefilter for wildcard regexes with stable anchors", async () => { const api = mockApi([{ path: "/a", content: "foo middle bar" }]); await grepBothTables(api, "m", "s", { ...baseParams, pattern: "foo.*bar" }, "/"); const [sql] = api.query.mock.calls.map((c: unknown[]) => c[0] as string); - expect(sql).toContain("summary::text LIKE '%foo%'"); + // Default likeOp is ILIKE (case-insensitive) — buildGrepSearchOptions + // picks it unless HIVEMIND_GREP_LIKE=case-sensitive overrides. + expect(sql).toContain("summary::text ILIKE '%foo%'"); }); - it("routes to ILIKE when ignoreCase is set", async () => { + it("routes to ILIKE regardless of ignoreCase (case-insensitive by default)", async () => { const api = mockApi([]); await grepBothTables(api, "m", "s", { ...baseParams, ignoreCase: true }, "/"); const [sql] = api.query.mock.calls.map((c: unknown[]) => c[0] as string); expect(sql).toContain("ILIKE"); }); + it("switches to LIKE when HIVEMIND_GREP_LIKE=case-sensitive", async () => { + const prev = process.env.HIVEMIND_GREP_LIKE; + process.env.HIVEMIND_GREP_LIKE = "case-sensitive"; + try { + const api = mockApi([{ path: "/a", content: "hi" }]); + await grepBothTables(api, "m", "s", baseParams, "/"); + const [sql] = api.query.mock.calls.map((c: unknown[]) => c[0] as string); + expect(sql).toContain("summary::text LIKE"); + expect(sql).not.toMatch(/summary::text ILIKE/); + } finally { + if (prev === undefined) delete process.env.HIVEMIND_GREP_LIKE; + else process.env.HIVEMIND_GREP_LIKE = prev; + } + }); + it("uses a single union query even for scoped target paths", async () => { const api = mockApi([{ path: "/summaries/a.md", content: "foo line" }]); await grepBothTables(api, "memory", "sessions", baseParams, "/summaries"); @@ -807,6 +830,33 @@ describe("grepBothTables", () => { expect(sql).toContain('FROM "sessions"'); expect(sql).toContain("UNION ALL"); }); + + it("emits every non-empty line when a query embedding is passed (semantic mode)", async () => { + const api = mockApi([ + { path: "/summaries/a.md", content: "foo first line\nunrelated but kept\n\ntrailing" }, + ]); + const out = await grepBothTables(api, "m", "s", baseParams, "/", [0.1, 0.2]); + // Semantic mode short-circuits the refinement — every non-empty line on + // the retrieved row survives, not just the pattern-matching ones. + expect(out).toContain("/summaries/a.md:foo first line"); + expect(out).toContain("/summaries/a.md:unrelated but kept"); + expect(out).toContain("/summaries/a.md:trailing"); + }); + + it("falls back to refined output when HIVEMIND_SEMANTIC_EMIT_ALL=false even with an embedding", async () => { + const prev = process.env.HIVEMIND_SEMANTIC_EMIT_ALL; + process.env.HIVEMIND_SEMANTIC_EMIT_ALL = "false"; + try { + const api = mockApi([{ path: "/a", content: "foo line\nunrelated" }]); + const out = await grepBothTables(api, "m", "s", baseParams, "/", [0.1]); + // Refinement ran, so only the pattern-matching line is emitted. + expect(out.some(l => l.includes("foo line"))).toBe(true); + expect(out.some(l => l.includes("unrelated"))).toBe(false); + } finally { + if (prev === undefined) delete process.env.HIVEMIND_SEMANTIC_EMIT_ALL; + else process.env.HIVEMIND_SEMANTIC_EMIT_ALL = prev; + } + }); }); describe("regex literal prefilter", () => { @@ -1011,3 +1061,264 @@ describe("regex literal prefilter", () => { expect(opts.pathFilter).toBe(" AND path = '/summaries/alice/s1.md'"); }); }); + +// ───────────────────────────────────────────────────────────────────────────── +// Additional coverage: single-turn JSONB shape + hybrid semantic branch +// ───────────────────────────────────────────────────────────────────────────── + +describe("normalizeContent: single-turn shape { turn: {...} }", () => { + it("emits one line with date prefix when date_time is present", () => { + const raw = JSON.stringify({ + date_time: "8:00 pm on 20 July, 2023", + speakers: { speaker_a: "Alice", speaker_b: "Bob" }, + turn: { dia_id: "D5:3", speaker: "Alice", text: "hello world" }, + }); + const out = normalizeContent("/sessions/conv_0_session_1.json", raw); + expect(out).toBe("(8:00 pm on 20 July, 2023) [D5:3] Alice: hello world"); + }); + + it("omits the date prefix when date_time is absent", () => { + const raw = JSON.stringify({ + turn: { dia_id: "D1:1", speaker: "X", text: "y" }, + }); + const out = normalizeContent("/sessions/conv_0_session_1.json", raw); + expect(out).toBe("[D1:1] X: y"); + }); + + it("falls back speaker->name on a single turn", () => { + const raw = JSON.stringify({ turn: { name: "Only", text: "hi" } }); + const out = normalizeContent("/sessions/conv_0_session_1.json", raw); + expect(out).toContain("Only: hi"); + }); + + it("falls back text->content on a single turn", () => { + const raw = JSON.stringify({ turn: { speaker: "X", content: "fallback" } }); + const out = normalizeContent("/sessions/conv_0_session_1.json", raw); + expect(out).toContain("X: fallback"); + }); + + it("emits placeholder `?: ` when the turn payload is empty", () => { + const raw = JSON.stringify({ turn: {} }); + const out = normalizeContent("/sessions/conv_0_session_1.json", raw); + // Empty turn → "?: " (placeholder speaker, empty text). Non-empty after + // trim so the branch emits rather than falling back to raw. + expect(out).toBe("?: "); + }); + + it("does not treat an array value in `turn` as single-turn", () => { + // Defensive: older per-turn shapes might mistakenly pass an array; we + // must not enter the singular branch because .speaker / .text would be + // undefined on the array itself. + const raw = JSON.stringify({ + turn: [{ speaker: "X", text: "y" }], + }); + const out = normalizeContent("/sessions/conv_0_session_1.json", raw); + // Falls through to raw — no branch matched. + expect(out).toBe(raw); + }); +}); + +describe("searchDeeplakeTables: hybrid semantic + lexical branch", () => { + function apiWithRows(rows: Record[] = []) { + const query = vi.fn().mockResolvedValue(rows); + return { query, api: { query } as any }; + } + + it("issues a single UNION-ALL query mixing semantic + lexical on both tables", async () => { + const { query, api } = apiWithRows([]); + await searchDeeplakeTables(api, "mem", "sess", { + pathFilter: "", + contentScanOnly: false, + likeOp: "ILIKE", + escapedPattern: "caroline", + queryEmbedding: [0.1, 0.2, 0.3], + }); + expect(query).toHaveBeenCalledTimes(1); + const sql = query.mock.calls[0][0] as string; + expect(sql).toContain("summary_embedding <#> ARRAY[0.1,0.2,0.3]::float4[]"); + expect(sql).toContain("message_embedding <#> ARRAY[0.1,0.2,0.3]::float4[]"); + expect(sql).toContain("summary::text ILIKE '%caroline%'"); + expect(sql).toContain("message::text ILIKE '%caroline%'"); + expect(sql).toContain("ORDER BY score DESC"); + }); + + it("uses 1.0 sentinel on lexical sub-queries so they stay above cosine (0..1)", async () => { + const { query, api } = apiWithRows([]); + await searchDeeplakeTables(api, "m", "s", { + pathFilter: "", + contentScanOnly: false, + likeOp: "ILIKE", + escapedPattern: "x", + queryEmbedding: [0.5], + }); + const sql = query.mock.calls[0][0] as string; + // Lexical branches carry the constant score; semantic uses the real cosine. + expect(sql).toMatch(/1\.0 AS score/); + expect(sql).toMatch(/\(summary_embedding <#>/); + }); + + it("skips the lexical branch entirely when contentScanOnly=true and no prefilter", async () => { + const { query, api } = apiWithRows([]); + await searchDeeplakeTables(api, "m", "s", { + pathFilter: "", + contentScanOnly: true, + likeOp: "ILIKE", + escapedPattern: "(?:unused)", + queryEmbedding: [0.1], + }); + const sql = query.mock.calls[0][0] as string; + // No usable literal → only the two semantic sub-queries are unioned. + expect(sql).not.toContain("summary::text ILIKE"); + expect(sql).not.toContain("message::text ILIKE"); + expect(sql).toContain("summary_embedding <#>"); + expect(sql).toContain("message_embedding <#>"); + }); + + it("falls back to prefilterPattern for regex grep with extractable literal", async () => { + const { query, api } = apiWithRows([]); + await searchDeeplakeTables(api, "m", "s", { + pathFilter: "", + contentScanOnly: true, + likeOp: "ILIKE", + escapedPattern: "foo.*bar", + prefilterPattern: "foo", + queryEmbedding: [0.1], + }); + const sql = query.mock.calls[0][0] as string; + expect(sql).toContain("summary::text ILIKE '%foo%'"); + }); + + it("uses prefilterPatterns alternation instead of a single prefilterPattern", async () => { + const { query, api } = apiWithRows([]); + await searchDeeplakeTables(api, "m", "s", { + pathFilter: "", + contentScanOnly: true, + likeOp: "ILIKE", + escapedPattern: "a|b", + prefilterPattern: "a", + prefilterPatterns: ["apple", "banana"], + queryEmbedding: [0.1], + }); + const sql = query.mock.calls[0][0] as string; + // prefilterPatterns wins over prefilterPattern when both are present. + expect(sql).toContain("%apple%"); + expect(sql).toContain("%banana%"); + }); + + it("dedupes rows by path, keeping the first occurrence (highest score wins)", async () => { + const { api } = apiWithRows([ + { path: "/a", content: "sem-first" }, + { path: "/a", content: "lex-dup" }, + { path: "/b", content: "other" }, + ]); + const out = await searchDeeplakeTables(api, "m", "s", { + pathFilter: "", + contentScanOnly: false, + likeOp: "ILIKE", + escapedPattern: "x", + queryEmbedding: [0.5], + }); + expect(out.map(r => r.path)).toEqual(["/a", "/b"]); + expect(out[0].content).toBe("sem-first"); + }); + + it("honors HIVEMIND_SEMANTIC_LIMIT env override for the semantic sub-queries", async () => { + const prev = process.env.HIVEMIND_SEMANTIC_LIMIT; + process.env.HIVEMIND_SEMANTIC_LIMIT = "7"; + try { + const { query, api } = apiWithRows([]); + await searchDeeplakeTables(api, "m", "s", { + pathFilter: "", + contentScanOnly: false, + likeOp: "ILIKE", + escapedPattern: "x", + queryEmbedding: [0.5], + }); + const sql = query.mock.calls[0][0] as string; + // Semantic LIMIT is 7; lexical still 20 (default). + expect(sql).toMatch(/summary_embedding <#> [^)]+\) AS score FROM "m" WHERE summary_embedding IS NOT NULL ORDER BY score DESC LIMIT 7/); + } finally { + if (prev === undefined) delete process.env.HIVEMIND_SEMANTIC_LIMIT; + else process.env.HIVEMIND_SEMANTIC_LIMIT = prev; + } + }); + + it("skips the semantic branch entirely when queryEmbedding is an empty array", async () => { + const { query, api } = apiWithRows([]); + await searchDeeplakeTables(api, "m", "s", { + pathFilter: "", + contentScanOnly: false, + likeOp: "ILIKE", + escapedPattern: "x", + queryEmbedding: [], + }); + const sql = query.mock.calls[0][0] as string; + // Empty embedding → falls through to the pure-lexical branch below. + expect(sql).not.toContain("<#>"); + }); + + it("skips the semantic branch when queryEmbedding is null", async () => { + const { query, api } = apiWithRows([]); + await searchDeeplakeTables(api, "m", "s", { + pathFilter: "", + contentScanOnly: false, + likeOp: "ILIKE", + escapedPattern: "x", + queryEmbedding: null, + }); + const sql = query.mock.calls[0][0] as string; + expect(sql).not.toContain("<#>"); + }); +}); + +describe("serializeFloat4Array (indirect)", () => { + it("returns NULL when the embedding contains a non-finite value", async () => { + const query = vi.fn().mockResolvedValue([]); + const api = { query } as any; + await searchDeeplakeTables(api, "m", "s", { + pathFilter: "", + contentScanOnly: false, + likeOp: "ILIKE", + escapedPattern: "x", + queryEmbedding: [1, NaN, 0.3], + }); + const sql = query.mock.calls[0][0] as string; + // Both semantic sub-queries degrade to NULL scoring; Deeplake accepts it + // and returns 0 rows for those two sub-queries so the hybrid still runs. + expect(sql).toContain("<#> NULL"); + }); +}); + +describe("bm25Term derivation in buildGrepSearchOptions", () => { + it("populates bm25Term with the raw pattern for non-regex fixed strings", () => { + const opts = buildGrepSearchOptions( + { pattern: "charity race", fixedString: true, ignoreCase: false, wordMatch: false, lineNumber: false, invertMatch: false, filesOnly: false, countOnly: false }, + "/" + ); + expect(opts.bm25Term).toBe("charity race"); + }); + + it("uses the extracted literal prefilter for regex patterns", () => { + const opts = buildGrepSearchOptions( + { pattern: "foo.*bar", fixedString: false, ignoreCase: false, wordMatch: false, lineNumber: false, invertMatch: false, filesOnly: false, countOnly: false }, + "/" + ); + expect(opts.bm25Term).toBe("foo"); + }); + + it("joins alternation prefilters with spaces for BM25", () => { + const opts = buildGrepSearchOptions( + { pattern: "apple|banana|cherry", fixedString: false, ignoreCase: false, wordMatch: false, lineNumber: false, invertMatch: false, filesOnly: false, countOnly: false }, + "/" + ); + expect(opts.bm25Term).toBe("apple banana cherry"); + }); + + it("returns undefined bm25Term when the regex has no extractable literal", () => { + const opts = buildGrepSearchOptions( + { pattern: "(?:a)", fixedString: false, ignoreCase: false, wordMatch: false, lineNumber: false, invertMatch: false, filesOnly: false, countOnly: false }, + "/" + ); + expect(opts.bm25Term).toBeUndefined(); + }); +}); diff --git a/claude-code/tests/grep-direct.test.ts b/claude-code/tests/grep-direct.test.ts index 0f56c9a..83366db 100644 --- a/claude-code/tests/grep-direct.test.ts +++ b/claude-code/tests/grep-direct.test.ts @@ -1,4 +1,19 @@ import { describe, it, expect, vi } from "vitest"; + +// The tests in this file exercise the *lexical* path of handleGrepDirect. +// Without this mock, the real EmbedClient would try to spawn / reach the +// nomic embed daemon over a Unix socket. If the daemon happens to be up +// (e.g. from a previous benchmark run), the semantic branch fires and +// returns a different shape, breaking every line-oriented assertion here. +// The mock forces queryEmbedding to stay null so the lexical refine path +// runs deterministically. +vi.mock("../../src/embeddings/client.js", () => ({ + EmbedClient: class { + async embed() { return null; } + async warmup() { return false; } + }, +})); + import { parseBashGrep, handleGrepDirect, type GrepParams } from "../../src/hooks/grep-direct.js"; describe("handleGrepDirect", () => { diff --git a/claude-code/tests/grep-interceptor.test.ts b/claude-code/tests/grep-interceptor.test.ts index ba7e67b..a8df072 100644 --- a/claude-code/tests/grep-interceptor.test.ts +++ b/claude-code/tests/grep-interceptor.test.ts @@ -1,4 +1,17 @@ -import { describe, it, expect, vi } from "vitest"; +import { describe, it, expect, vi, afterEach } from "vitest"; + +// Mock EmbedClient BEFORE importing the interceptor so the shared singleton +// inside grep-interceptor picks up our stub (not the real daemon client). +// `vi.hoisted` lets us share the spy across mock factory + tests — plain +// top-level consts aren't visible inside the hoisted vi.mock factory. +const { mockEmbed } = vi.hoisted(() => ({ mockEmbed: vi.fn() })); +vi.mock("../../src/embeddings/client.js", () => { + class MockEmbedClient { + async embed(text: string, kind: string) { return mockEmbed(text, kind); } + } + return { EmbedClient: MockEmbedClient }; +}); + import { createGrepCommand } from "../../src/shell/grep-interceptor.js"; import { DeeplakeFs } from "../../src/shell/deeplake-fs.js"; import * as grepCore from "../../src/shell/grep-core.js"; @@ -31,6 +44,11 @@ function makeCtx(fs: DeeplakeFs, cwd = "/memory") { // cache. Tests below assert that new contract. describe("grep interceptor", () => { + afterEach(() => { + vi.restoreAllMocks(); + mockEmbed.mockReset(); + }); + it("returns exitCode=1 when the pattern is missing", async () => { const client = makeClient(); const fs = await DeeplakeFs.create(client as never, "test", "/memory"); @@ -217,4 +235,155 @@ describe("grep interceptor", () => { expect(result.exitCode).toBe(0); expect(result.stdout).toContain("hello world"); }); + + // ── Semantic path (HIVEMIND_SEMANTIC_SEARCH default=on) ───────────────── + // These tests exercise the daemon-backed embed + UNION ALL branch of + // searchDeeplakeTables. They mock the shared EmbedClient singleton so we + // don't actually spawn nomic. + it("passes the query embedding into searchDeeplakeTables for semantic-friendly patterns", async () => { + mockEmbed.mockResolvedValueOnce([0.1, 0.2, 0.3]); + const client = makeClient([{ path: "/memory/a.txt", content: "deploy failed" }]); + const fs = await DeeplakeFs.create(client as never, "test", "/memory"); + const searchSpy = vi.spyOn(grepCore, "searchDeeplakeTables") + .mockResolvedValue([{ path: "/memory/a.txt", content: "deploy failed" }]); + + const cmd = createGrepCommand(client as never, fs, "test", "sessions"); + const result = await cmd.execute(["deploy", "/memory"], makeCtx(fs) as never); + + expect(mockEmbed).toHaveBeenCalledWith("deploy", "query"); + const opts = searchSpy.mock.calls[0][3] as { queryEmbedding: number[] | null }; + expect(opts.queryEmbedding).toEqual([0.1, 0.2, 0.3]); + expect(result.exitCode).toBe(0); + searchSpy.mockRestore(); + }); + + it("skips embedding on regex-heavy patterns (too many metachars)", async () => { + mockEmbed.mockClear(); + mockEmbed.mockResolvedValue([0.5]); + const client = makeClient([]); + const fs = await DeeplakeFs.create(client as never, "test", "/memory"); + const cmd = createGrepCommand(client as never, fs, "test"); + // Three metachars should disqualify the pattern from semantic. + await cmd.execute(["(foo|bar|baz)\\+", "/memory"], makeCtx(fs) as never); + expect(mockEmbed).not.toHaveBeenCalled(); + }); + + it("skips embedding on very short patterns (< 2 chars)", async () => { + mockEmbed.mockClear(); + const client = makeClient([]); + const fs = await DeeplakeFs.create(client as never, "test", "/memory"); + const cmd = createGrepCommand(client as never, fs, "test"); + await cmd.execute(["a", "/memory"], makeCtx(fs) as never); + expect(mockEmbed).not.toHaveBeenCalled(); + }); + + it("treats a thrown embed() as a null embedding and continues lexically", async () => { + mockEmbed.mockClear(); + mockEmbed.mockRejectedValueOnce(new Error("daemon down")); + const client = makeClient([{ path: "/memory/a.txt", content: "hello world" }]); + const fs = await DeeplakeFs.create(client as never, "test", "/memory"); + client.query.mockClear(); + client.query.mockResolvedValue([{ path: "/memory/a.txt", content: "hello world" }]); + + const cmd = createGrepCommand(client as never, fs, "test"); + const result = await cmd.execute(["hello", "/memory"], makeCtx(fs) as never); + + expect(mockEmbed).toHaveBeenCalled(); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("hello world"); + }); + + it("retries with a lexical-only search when semantic returns zero rows", async () => { + mockEmbed.mockClear(); + mockEmbed.mockResolvedValueOnce([0.1]); + const client = makeClient([]); + const fs = await DeeplakeFs.create(client as never, "test", "/memory"); + const searchSpy = vi.spyOn(grepCore, "searchDeeplakeTables") + .mockResolvedValueOnce([]) // first call (semantic+lexical hybrid) → empty + .mockResolvedValueOnce([{ path: "/memory/a.txt", content: "hi" }]); // lexical retry + + const cmd = createGrepCommand(client as never, fs, "test"); + const result = await cmd.execute(["hi", "/memory"], makeCtx(fs) as never); + + expect(searchSpy).toHaveBeenCalledTimes(2); + // First call carried the embedding, retry did not. + const firstOpts = searchSpy.mock.calls[0][3] as { queryEmbedding: number[] | null }; + const secondOpts = searchSpy.mock.calls[1][3] as { queryEmbedding?: number[] | null }; + expect(firstOpts.queryEmbedding).toEqual([0.1]); + expect(secondOpts.queryEmbedding).toBeUndefined(); + expect(result.exitCode).toBe(0); + searchSpy.mockRestore(); + }); + + it("emits all non-empty lines per row when the semantic path returned an embedding", async () => { + mockEmbed.mockClear(); + mockEmbed.mockResolvedValueOnce([0.1, 0.2]); + const client = makeClient([]); + const fs = await DeeplakeFs.create(client as never, "test", "/memory"); + const searchSpy = vi.spyOn(grepCore, "searchDeeplakeTables") + .mockResolvedValue([{ path: "/memory/a.txt", content: "line A\nline B\n\nline C" }]); + + const cmd = createGrepCommand(client as never, fs, "test"); + const result = await cmd.execute(["deploy", "/memory"], makeCtx(fs) as never); + + // All three non-empty lines are emitted verbatim — no regex refinement. + expect(result.stdout).toContain("/memory/a.txt:line A"); + expect(result.stdout).toContain("/memory/a.txt:line B"); + expect(result.stdout).toContain("/memory/a.txt:line C"); + expect(result.exitCode).toBe(0); + searchSpy.mockRestore(); + }); + + it("hits the 3s timeout rejector when searchDeeplakeTables hangs", async () => { + // Force the SQL search to hang forever so Promise.race's setTimeout + // callback (line 131 of grep-interceptor.ts) fires with a timeout error, + // covering the reject() arrow function. Use fake timers to fast-forward + // past the 3s window without actually sleeping. + vi.useFakeTimers(); + try { + mockEmbed.mockResolvedValue(null); // skip semantic for a cleaner timeout. + const client = makeClient([]); + const fs = await DeeplakeFs.create(client as never, "test", "/memory"); + await fs.writeFile("/memory/a.txt", "hello world"); // fallback content. + + vi.spyOn(grepCore, "searchDeeplakeTables") + .mockImplementation(() => new Promise(() => { /* never resolves */ })); + + const cmd = createGrepCommand(client as never, fs, "test"); + const pending = cmd.execute(["hello", "/memory"], makeCtx(fs) as never); + // Advance past the 3s timeout so the reject arrow runs, then drain + // microtasks so the catch branch takes over and the FS fallback runs. + await vi.advanceTimersByTimeAsync(3001); + const result = await pending; + // Fallback path should have kicked in and found the FS content. + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("hello world"); + } finally { + vi.useRealTimers(); + } + }); + + it("disables the semantic path when HIVEMIND_SEMANTIC_EMIT_ALL=false", async () => { + mockEmbed.mockClear(); + mockEmbed.mockResolvedValueOnce([0.1]); + const prev = process.env.HIVEMIND_SEMANTIC_EMIT_ALL; + process.env.HIVEMIND_SEMANTIC_EMIT_ALL = "false"; + try { + const client = makeClient([{ path: "/memory/a.txt", content: "hello world\ngoodbye" }]); + const fs = await DeeplakeFs.create(client as never, "test", "/memory"); + client.query.mockClear(); + client.query.mockResolvedValue([{ path: "/memory/a.txt", content: "hello world\ngoodbye" }]); + + const cmd = createGrepCommand(client as never, fs, "test"); + const result = await cmd.execute(["hello", "/memory"], makeCtx(fs) as never); + + // Refinement active → only the hello line is emitted. + expect(result.stdout).toContain("hello world"); + expect(result.stdout).not.toContain("goodbye"); + expect(result.exitCode).toBe(0); + } finally { + if (prev === undefined) delete process.env.HIVEMIND_SEMANTIC_EMIT_ALL; + else process.env.HIVEMIND_SEMANTIC_EMIT_ALL = prev; + } + }); }); diff --git a/claude-code/tests/session-start-setup-branches.test.ts b/claude-code/tests/session-start-setup-branches.test.ts index 8c28370..a7321b3 100644 --- a/claude-code/tests/session-start-setup-branches.test.ts +++ b/claude-code/tests/session-start-setup-branches.test.ts @@ -29,6 +29,7 @@ const isNewerMock = vi.fn(); const resolveVersionedPluginDirMock = vi.fn(); const snapshotPluginDirMock = vi.fn(); const restoreOrCleanupMock = vi.fn(); +const embedWarmupMock = vi.fn(); vi.mock("../../src/utils/stdin.js", () => ({ readStdin: (...a: any[]) => stdinMock(...a) })); vi.mock("../../src/commands/auth.js", () => ({ @@ -64,6 +65,11 @@ vi.mock("../../src/utils/plugin-cache.js", () => ({ snapshotPluginDir: (...a: any[]) => snapshotPluginDirMock(...a), restoreOrCleanup: (...a: any[]) => restoreOrCleanupMock(...a), })); +vi.mock("../../src/embeddings/client.js", () => ({ + EmbedClient: class { + async warmup() { return embedWarmupMock(); } + }, +})); async function runHook(): Promise { delete process.env.HIVEMIND_WIKI_WORKER; @@ -95,6 +101,7 @@ beforeEach(() => { getLatestVersionMock.mockReset().mockResolvedValue("0.6.38"); isNewerMock.mockReset().mockReturnValue(false); resolveVersionedPluginDirMock.mockReset().mockReturnValue(null); + embedWarmupMock.mockReset().mockResolvedValue(true); snapshotPluginDirMock.mockReset(); restoreOrCleanupMock.mockReset().mockReturnValue("noop"); }); diff --git a/claude-code/tests/session-start-setup-hook.test.ts b/claude-code/tests/session-start-setup-hook.test.ts index e3c9ca6..9a4484d 100644 --- a/claude-code/tests/session-start-setup-hook.test.ts +++ b/claude-code/tests/session-start-setup-hook.test.ts @@ -17,6 +17,7 @@ const debugLogMock = vi.fn(); const ensureTableMock = vi.fn(); const ensureSessionsTableMock = vi.fn(); const execSyncMock = vi.fn(); +const embedWarmupMock = vi.fn(); vi.mock("../../src/utils/stdin.js", () => ({ readStdin: (...a: any[]) => stdinMock(...a) })); vi.mock("../../src/commands/auth.js", () => ({ @@ -38,6 +39,11 @@ vi.mock("node:child_process", async () => { const actual = await vi.importActual("node:child_process"); return { ...actual, execSync: (...a: any[]) => execSyncMock(...a) }; }); +vi.mock("../../src/embeddings/client.js", () => ({ + EmbedClient: class { + async warmup() { return embedWarmupMock(); } + }, +})); // We also need to control global.fetch for the GitHub version lookup. const originalFetch = global.fetch; @@ -74,6 +80,7 @@ beforeEach(() => { ensureTableMock.mockReset().mockResolvedValue(undefined); ensureSessionsTableMock.mockReset().mockResolvedValue(undefined); execSyncMock.mockReset(); + embedWarmupMock.mockReset().mockResolvedValue(true); fetchMock.mockReset().mockResolvedValue({ ok: true, json: async () => ({ version: "0.0.1" }), // same-as-current: no update @@ -217,6 +224,44 @@ describe("session-start-setup hook — version check + autoupdate", () => { }); }); +describe("session-start-setup hook — embed daemon warmup", () => { + it("calls EmbedClient.warmup() by default and logs the outcome", async () => { + await runHook(); + expect(embedWarmupMock).toHaveBeenCalledTimes(1); + expect(debugLogMock).toHaveBeenCalledWith("embed daemon warmup: ok"); + }); + + it("logs 'failed' when warmup returns false", async () => { + embedWarmupMock.mockResolvedValue(false); + await runHook(); + expect(debugLogMock).toHaveBeenCalledWith("embed daemon warmup: failed"); + }); + + it("logs the thrown message when warmup rejects", async () => { + embedWarmupMock.mockRejectedValue(new Error("daemon spawn failed")); + await runHook(); + expect(debugLogMock).toHaveBeenCalledWith( + expect.stringContaining("embed daemon warmup threw: daemon spawn failed"), + ); + }); + + it("skips warmup when HIVEMIND_EMBED_WARMUP=false", async () => { + await runHook({ HIVEMIND_EMBED_WARMUP: "false" }); + expect(embedWarmupMock).not.toHaveBeenCalled(); + expect(debugLogMock).toHaveBeenCalledWith( + "embed daemon warmup skipped via HIVEMIND_EMBED_WARMUP=false", + ); + }); + + it("skips warmup when the master HIVEMIND_EMBEDDINGS=false flag is set", async () => { + await runHook({ HIVEMIND_EMBEDDINGS: "false" }); + expect(embedWarmupMock).not.toHaveBeenCalled(); + expect(debugLogMock).toHaveBeenCalledWith( + "embed daemon warmup skipped via HIVEMIND_EMBEDDINGS=false", + ); + }); +}); + describe("session-start-setup hook — fatal catch", () => { it("catches a stdin throw and exits 0", async () => { stdinMock.mockRejectedValue(new Error("stdin boom")); diff --git a/claude-code/tests/session-summary.test.ts b/claude-code/tests/session-summary.test.ts index 09f123a..7840f87 100644 --- a/claude-code/tests/session-summary.test.ts +++ b/claude-code/tests/session-summary.test.ts @@ -111,6 +111,21 @@ function makeClient(seed: Record = {}) { } else if (/\d/.test(valsStr[i])) { const m = valsStr.slice(i).match(/^(\d+)/); if (m) { allVals.push(m[1]); i += m[1].length; } else i++; + } else if (valsStr.slice(i, i + 4).toUpperCase() === "NULL") { + allVals.push(""); + i += 4; + } else if (valsStr.slice(i, i + 6).toUpperCase() === "ARRAY[") { + let depth = 1; + let end = i + 6; + while (end < valsStr.length && depth > 0) { + if (valsStr[end] === "[") depth++; + else if (valsStr[end] === "]") depth--; + end++; + } + const castMatch = valsStr.slice(end).match(/^::float4\[\]/i); + if (castMatch) end += castMatch[0].length; + allVals.push(valsStr.slice(i, end)); + i = end; } else { i++; } } const colMap: Record = {}; diff --git a/claude-code/tests/sessions-table.test.ts b/claude-code/tests/sessions-table.test.ts index 40a254f..673d0f9 100644 --- a/claude-code/tests/sessions-table.test.ts +++ b/claude-code/tests/sessions-table.test.ts @@ -27,11 +27,14 @@ function makeClient(memoryRows: Row[] = [], sessionRows: Row[] = []) { return rows.map(r => ({ path: r.path, size_bytes: r.size_bytes, mime_type: r.mime_type })); } - // Bootstrap: SELECT path, SUM(size_bytes) ... GROUP BY path (sessions table) - if (sql.includes("SUM(size_bytes)") && sql.includes("GROUP BY")) { + // Bootstrap: SELECT path, MAX(size_bytes) ... GROUP BY path (sessions table). + // The production SQL uses MAX to work around a Deeplake backend quirk + // where SUM() returns NULL under GROUP BY (see deeplake-fs.ts), so the + // mock mirrors that by taking MAX per path as well. + if (sql.includes("MAX(size_bytes)") && sql.includes("GROUP BY")) { const groups = new Map(); for (const r of sessionRows) { - groups.set(r.path, (groups.get(r.path) ?? 0) + r.size_bytes); + groups.set(r.path, Math.max(groups.get(r.path) ?? 0, r.size_bytes)); } return [...groups.entries()].map(([path, total]) => ({ path, total_size: total })); } diff --git a/claude-code/tests/upload-summary.test.ts b/claude-code/tests/upload-summary.test.ts index 56eb0e9..9918d7b 100644 --- a/claude-code/tests/upload-summary.test.ts +++ b/claude-code/tests/upload-summary.test.ts @@ -133,6 +133,65 @@ describe("uploadSummary — Deeplake single-UPDATE invariant", () => { }); }); +describe("uploadSummary — summary_embedding column", () => { + it("INSERT path includes summary_embedding as ARRAY[...]::float4[] when an embedding is supplied", async () => { + const { fn, calls } = makeSpyQuery([[]]); + await uploadSummary(fn, { + ...BASE, + text: TEXT_WITH_WHAT_HAPPENED, + embedding: [0.1, -0.2, 0.3], + }); + const insert = calls.find(c => /^INSERT INTO/i.test(c))!; + expect(insert).toContain("summary_embedding"); + expect(insert).toContain("ARRAY[0.1,-0.2,0.3]::float4[]"); + }); + + it("UPDATE path sets summary_embedding in the same statement as summary", async () => { + const { fn, calls } = makeSpyQuery([[{ path: BASE.vpath }]]); + await uploadSummary(fn, { + ...BASE, + text: TEXT_WITH_WHAT_HAPPENED, + embedding: [0.5, 0.25], + }); + const update = calls.find(c => /^UPDATE/i.test(c))!; + expect(update).toContain("summary = E'"); + expect(update).toContain("summary_embedding = ARRAY[0.5,0.25]::float4[]"); + }); + + it("writes SQL NULL for summary_embedding when the caller omits the embedding", async () => { + const { fn, calls } = makeSpyQuery([[]]); + await uploadSummary(fn, { ...BASE, text: TEXT_WITH_WHAT_HAPPENED }); + const insert = calls.find(c => /^INSERT INTO/i.test(c))!; + expect(insert).toContain("summary_embedding"); + // The literal token must be the bare SQL NULL, not the string 'NULL'. + expect(insert).not.toContain("'NULL'"); + expect(insert).toContain(", NULL, "); // bare NULL between surrounding values in VALUES (...) + }); + + it("writes SQL NULL when the caller explicitly passes embedding: null", async () => { + const { fn, calls } = makeSpyQuery([[{ path: BASE.vpath }]]); + await uploadSummary(fn, { + ...BASE, + text: TEXT_WITH_WHAT_HAPPENED, + embedding: null, + }); + const update = calls.find(c => /^UPDATE/i.test(c))!; + expect(update).toContain("summary_embedding = NULL"); + }); + + it("writes SQL NULL for an empty embedding array (daemon returned invalid)", async () => { + const { fn, calls } = makeSpyQuery([[]]); + await uploadSummary(fn, { + ...BASE, + text: TEXT_WITH_WHAT_HAPPENED, + embedding: [], + }); + const insert = calls.find(c => /^INSERT INTO/i.test(c))!; + expect(insert).not.toContain("'NULL'"); + expect(insert).toContain(", NULL, "); + }); +}); + describe("extractDescription", () => { it("extracts the What Happened section trimmed to 300 chars", () => { const d = extractDescription(TEXT_WITH_WHAT_HAPPENED); diff --git a/claude-code/tests/wiki-worker.test.ts b/claude-code/tests/wiki-worker.test.ts index f287cc1..dd16d04 100644 --- a/claude-code/tests/wiki-worker.test.ts +++ b/claude-code/tests/wiki-worker.test.ts @@ -27,6 +27,7 @@ const finalizeSummaryMock = vi.fn(); const releaseLockMock = vi.fn(); const uploadSummaryMock = vi.fn(); const execFileSyncMock = vi.fn(); +const embedSummaryMock = vi.fn(); vi.mock("../../src/hooks/summary-state.js", () => ({ finalizeSummary: (...a: any[]) => finalizeSummaryMock(...a), @@ -35,6 +36,11 @@ vi.mock("../../src/hooks/summary-state.js", () => ({ vi.mock("../../src/hooks/upload-summary.js", () => ({ uploadSummary: (...a: any[]) => uploadSummaryMock(...a), })); +vi.mock("../../src/embeddings/client.js", () => ({ + EmbedClient: class { + async embed(text: string, kind: string) { return embedSummaryMock(text, kind); } + }, +})); vi.mock("node:child_process", async () => { const actual = await vi.importActual("node:child_process"); return { ...actual, execFileSync: (...a: any[]) => execFileSyncMock(...a) }; @@ -107,6 +113,7 @@ beforeEach(() => { finalizeSummaryMock.mockReset(); releaseLockMock.mockReset(); uploadSummaryMock.mockReset().mockResolvedValue({ path: "insert", summaryLength: 100, descLength: 20, sql: "..." }); + embedSummaryMock.mockReset().mockResolvedValue([0.1, 0.2, 0.3]); execFileSyncMock.mockReset(); }); diff --git a/codex/bundle/capture.js b/codex/bundle/capture.js index b16405a..7cdf620 100755 --- a/codex/bundle/capture.js +++ b/codex/bundle/capture.js @@ -370,10 +370,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(tbl)) { log2(`table "${tbl}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', summary_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${tbl}" created`); if (!tables.includes(tbl)) this._tablesCache = [...tables, tbl]; + } else { + try { + await this.query(`ALTER TABLE "${tbl}" ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add summary_embedding skipped: ${e.message}`); + } } } /** Create the sessions table (uses JSONB for message since every row is a JSON event). */ @@ -381,10 +387,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(name)) { log2(`table "${name}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, message_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${name}" created`); if (!tables.includes(name)) this._tablesCache = [...tables, name]; + } else { + try { + await this.query(`ALTER TABLE "${name}" ADD COLUMN IF NOT EXISTS message_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add message_embedding skipped: ${e.message}`); + } } await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`); } diff --git a/codex/bundle/commands/auth-login.js b/codex/bundle/commands/auth-login.js index aa19693..8714067 100755 --- a/codex/bundle/commands/auth-login.js +++ b/codex/bundle/commands/auth-login.js @@ -551,10 +551,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(tbl)) { log2(`table "${tbl}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', summary_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${tbl}" created`); if (!tables.includes(tbl)) this._tablesCache = [...tables, tbl]; + } else { + try { + await this.query(`ALTER TABLE "${tbl}" ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add summary_embedding skipped: ${e.message}`); + } } } /** Create the sessions table (uses JSONB for message since every row is a JSON event). */ @@ -562,10 +568,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(name)) { log2(`table "${name}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, message_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${name}" created`); if (!tables.includes(name)) this._tablesCache = [...tables, name]; + } else { + try { + await this.query(`ALTER TABLE "${name}" ADD COLUMN IF NOT EXISTS message_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add message_embedding skipped: ${e.message}`); + } } await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`); } diff --git a/codex/bundle/embeddings/embed-daemon.js b/codex/bundle/embeddings/embed-daemon.js new file mode 100755 index 0000000..cd953b6 --- /dev/null +++ b/codex/bundle/embeddings/embed-daemon.js @@ -0,0 +1,240 @@ +#!/usr/bin/env node + +// dist/src/embeddings/daemon.js +import { createServer } from "node:net"; +import { unlinkSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs"; + +// dist/src/embeddings/protocol.js +var DEFAULT_SOCKET_DIR = "/tmp"; +var DEFAULT_MODEL_REPO = "nomic-ai/nomic-embed-text-v1.5"; +var DEFAULT_DTYPE = "q8"; +var DEFAULT_DIMS = 768; +var DEFAULT_IDLE_TIMEOUT_MS = 15 * 60 * 1e3; +var DOC_PREFIX = "search_document: "; +var QUERY_PREFIX = "search_query: "; +function socketPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.sock`; +} +function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.pid`; +} + +// dist/src/embeddings/nomic.js +var NomicEmbedder = class { + pipeline = null; + loading = null; + repo; + dtype; + dims; + constructor(opts = {}) { + this.repo = opts.repo ?? DEFAULT_MODEL_REPO; + this.dtype = opts.dtype ?? DEFAULT_DTYPE; + this.dims = opts.dims ?? DEFAULT_DIMS; + } + async load() { + if (this.pipeline) + return; + if (this.loading) + return this.loading; + this.loading = (async () => { + const mod = await import("@huggingface/transformers"); + mod.env.allowLocalModels = false; + mod.env.useFSCache = true; + this.pipeline = await mod.pipeline("feature-extraction", this.repo, { dtype: this.dtype }); + })(); + try { + await this.loading; + } finally { + this.loading = null; + } + } + addPrefix(text, kind) { + return (kind === "query" ? QUERY_PREFIX : DOC_PREFIX) + text; + } + async embed(text, kind = "document") { + await this.load(); + if (!this.pipeline) + throw new Error("embedder not loaded"); + const out = await this.pipeline(this.addPrefix(text, kind), { pooling: "mean", normalize: true }); + const full = Array.from(out.data); + return this.truncate(full); + } + async embedBatch(texts, kind = "document") { + if (texts.length === 0) + return []; + await this.load(); + if (!this.pipeline) + throw new Error("embedder not loaded"); + const prefixed = texts.map((t) => this.addPrefix(t, kind)); + const out = await this.pipeline(prefixed, { pooling: "mean", normalize: true }); + const flat = Array.from(out.data); + const total = flat.length; + const full = total / texts.length; + const batches = []; + for (let i = 0; i < texts.length; i++) { + batches.push(this.truncate(flat.slice(i * full, (i + 1) * full))); + } + return batches; + } + truncate(vec) { + if (this.dims >= vec.length) + return vec; + const head = vec.slice(0, this.dims); + let norm = 0; + for (const v of head) + norm += v * v; + norm = Math.sqrt(norm); + if (norm === 0) + return head; + for (let i = 0; i < head.length; i++) + head[i] /= norm; + return head; + } +}; + +// dist/src/utils/debug.js +import { appendFileSync } from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; +var DEBUG = process.env.HIVEMIND_DEBUG === "1"; +var LOG = join(homedir(), ".deeplake", "hook-debug.log"); +function log(tag, msg) { + if (!DEBUG) + return; + appendFileSync(LOG, `${(/* @__PURE__ */ new Date()).toISOString()} [${tag}] ${msg} +`); +} + +// dist/src/embeddings/daemon.js +var log2 = (m) => log("embed-daemon", m); +function getUid() { + const uid = typeof process.getuid === "function" ? process.getuid() : void 0; + return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; +} +var EmbedDaemon = class { + server = null; + embedder; + socketPath; + pidPath; + idleTimeoutMs; + idleTimer = null; + constructor(opts = {}) { + const uid = getUid(); + const dir = opts.socketDir ?? "/tmp"; + this.socketPath = socketPathFor(uid, dir); + this.pidPath = pidPathFor(uid, dir); + this.idleTimeoutMs = opts.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS; + this.embedder = new NomicEmbedder({ repo: opts.repo, dtype: opts.dtype, dims: opts.dims }); + } + async start() { + mkdirSync(this.socketPath.replace(/\/[^/]+$/, ""), { recursive: true }); + writeFileSync(this.pidPath, String(process.pid), { mode: 384 }); + if (existsSync(this.socketPath)) { + try { + unlinkSync(this.socketPath); + } catch { + } + } + this.embedder.load().then(() => log2("model ready")).catch((e) => log2(`load err: ${e.message}`)); + this.server = createServer((sock) => this.handleConnection(sock)); + await new Promise((resolve, reject) => { + this.server.once("error", reject); + this.server.listen(this.socketPath, () => { + try { + chmodSync(this.socketPath, 384); + } catch { + } + log2(`listening on ${this.socketPath}`); + resolve(); + }); + }); + this.resetIdleTimer(); + process.on("SIGINT", () => this.shutdown()); + process.on("SIGTERM", () => this.shutdown()); + } + resetIdleTimer() { + if (this.idleTimer) + clearTimeout(this.idleTimer); + this.idleTimer = setTimeout(() => { + log2(`idle timeout ${this.idleTimeoutMs}ms reached, shutting down`); + this.shutdown(); + }, this.idleTimeoutMs); + this.idleTimer.unref(); + } + shutdown() { + try { + this.server?.close(); + } catch { + } + try { + if (existsSync(this.socketPath)) + unlinkSync(this.socketPath); + } catch { + } + try { + if (existsSync(this.pidPath)) + unlinkSync(this.pidPath); + } catch { + } + process.exit(0); + } + handleConnection(sock) { + let buf = ""; + sock.setEncoding("utf-8"); + sock.on("data", (chunk) => { + buf += chunk; + let nl; + while ((nl = buf.indexOf("\n")) !== -1) { + const line = buf.slice(0, nl); + buf = buf.slice(nl + 1); + if (line.length === 0) + continue; + this.handleLine(sock, line); + } + }); + sock.on("error", () => { + }); + } + async handleLine(sock, line) { + this.resetIdleTimer(); + let req; + try { + req = JSON.parse(line); + } catch { + return; + } + try { + const resp = await this.dispatch(req); + sock.write(JSON.stringify(resp) + "\n"); + } catch (e) { + const err = e instanceof Error ? e.message : String(e); + const resp = { id: req.id, error: err }; + sock.write(JSON.stringify(resp) + "\n"); + } + } + async dispatch(req) { + if (req.op === "ping") { + const p = req; + return { id: p.id, ready: true, model: this.embedder.repo, dims: this.embedder.dims }; + } + if (req.op === "embed") { + const e = req; + const vec = await this.embedder.embed(e.text, e.kind); + return { id: e.id, embedding: vec }; + } + return { id: req.id, error: "unknown op" }; + } +}; +var invokedDirectly = import.meta.url === `file://${process.argv[1]}` || process.argv[1] && import.meta.url.endsWith(process.argv[1].split("/").pop() ?? ""); +if (invokedDirectly) { + const dims = process.env.HIVEMIND_EMBED_DIMS ? Number(process.env.HIVEMIND_EMBED_DIMS) : void 0; + const idleTimeoutMs = process.env.HIVEMIND_EMBED_IDLE_MS ? Number(process.env.HIVEMIND_EMBED_IDLE_MS) : void 0; + const d = new EmbedDaemon({ dims, idleTimeoutMs }); + d.start().catch((e) => { + log2(`fatal: ${e.message}`); + process.exit(1); + }); +} +export { + EmbedDaemon +}; diff --git a/codex/bundle/pre-tool-use.js b/codex/bundle/pre-tool-use.js index c7aeda7..552b425 100755 --- a/codex/bundle/pre-tool-use.js +++ b/codex/bundle/pre-tool-use.js @@ -2,9 +2,9 @@ // dist/src/hooks/codex/pre-tool-use.js import { execFileSync } from "node:child_process"; -import { existsSync as existsSync3 } from "node:fs"; -import { join as join6, dirname } from "node:path"; -import { fileURLToPath as fileURLToPath2 } from "node:url"; +import { existsSync as existsSync4 } from "node:fs"; +import { join as join7, dirname as dirname2 } from "node:path"; +import { fileURLToPath as fileURLToPath3 } from "node:url"; // dist/src/utils/stdin.js function readStdin() { @@ -376,10 +376,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(tbl)) { log2(`table "${tbl}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', summary_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${tbl}" created`); if (!tables.includes(tbl)) this._tablesCache = [...tables, tbl]; + } else { + try { + await this.query(`ALTER TABLE "${tbl}" ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add summary_embedding skipped: ${e.message}`); + } } } /** Create the sessions table (uses JSONB for message since every row is a JSON event). */ @@ -387,10 +393,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(name)) { log2(`table "${name}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, message_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${name}" created`); if (!tables.includes(name)) this._tablesCache = [...tables, name]; + } else { + try { + await this.query(`ALTER TABLE "${name}" ADD COLUMN IF NOT EXISTS message_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add message_embedding skipped: ${e.message}`); + } } await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`); } @@ -560,24 +572,25 @@ function normalizeContent(path, raw) { return raw; } if (Array.isArray(obj.turns)) { - const header = []; - if (obj.date_time) - header.push(`date: ${obj.date_time}`); - if (obj.speakers) { - const s = obj.speakers; - const names = [s.speaker_a, s.speaker_b].filter(Boolean).join(", "); - if (names) - header.push(`speakers: ${names}`); - } + const dateHeader = obj.date_time ? `(${String(obj.date_time)}) ` : ""; const lines = obj.turns.map((t) => { const sp = String(t?.speaker ?? t?.name ?? "?").trim(); const tx = String(t?.text ?? t?.content ?? "").replace(/\s+/g, " ").trim(); const tag = t?.dia_id ? `[${t.dia_id}] ` : ""; - return `${tag}${sp}: ${tx}`; + return `${dateHeader}${tag}${sp}: ${tx}`; }); - const out2 = [...header, ...lines].join("\n"); + const out2 = lines.join("\n"); return out2.trim() ? out2 : raw; } + if (obj.turn && typeof obj.turn === "object" && !Array.isArray(obj.turn)) { + const t = obj.turn; + const sp = String(t.speaker ?? t.name ?? "?").trim(); + const tx = String(t.text ?? t.content ?? "").replace(/\s+/g, " ").trim(); + const tag = t.dia_id ? `[${String(t.dia_id)}] ` : ""; + const dateHeader = obj.date_time ? `(${String(obj.date_time)}) ` : ""; + const line = `${dateHeader}${tag}${sp}: ${tx}`; + return line.trim() ? line : raw; + } const stripRecalled = (t) => { const i = t.indexOf(""); if (i === -1) @@ -620,8 +633,38 @@ function buildPathCondition(targetPath) { return `(path = '${sqlStr(clean)}' OR path LIKE '${sqlLike(clean)}/%' ESCAPE '\\')`; } async function searchDeeplakeTables(api, memoryTable, sessionsTable, opts) { - const { pathFilter, contentScanOnly, likeOp, escapedPattern, prefilterPattern, prefilterPatterns, multiWordPatterns } = opts; + const { pathFilter, contentScanOnly, likeOp, escapedPattern, prefilterPattern, prefilterPatterns, queryEmbedding, bm25Term, multiWordPatterns } = opts; const limit = opts.limit ?? 100; + if (queryEmbedding && queryEmbedding.length > 0) { + const vecLit = serializeFloat4Array(queryEmbedding); + const semanticLimit = Math.min(limit, Number(process.env.HIVEMIND_SEMANTIC_LIMIT ?? "20")); + const lexicalLimit = Math.min(limit, Number(process.env.HIVEMIND_HYBRID_LEXICAL_LIMIT ?? "20")); + const filterPatternsForLex = contentScanOnly ? prefilterPatterns && prefilterPatterns.length > 0 ? prefilterPatterns : prefilterPattern ? [prefilterPattern] : [] : [escapedPattern]; + const memLexFilter = buildContentFilter("summary::text", likeOp, filterPatternsForLex); + const sessLexFilter = buildContentFilter("message::text", likeOp, filterPatternsForLex); + const memLexQuery = memLexFilter ? `SELECT path, summary::text AS content, 0 AS source_order, '' AS creation_date, 1.0 AS score FROM "${memoryTable}" WHERE 1=1${pathFilter}${memLexFilter} LIMIT ${lexicalLimit}` : null; + const sessLexQuery = sessLexFilter ? `SELECT path, message::text AS content, 1 AS source_order, COALESCE(creation_date::text, '') AS creation_date, 1.0 AS score FROM "${sessionsTable}" WHERE 1=1${pathFilter}${sessLexFilter} LIMIT ${lexicalLimit}` : null; + const memSemQuery = `SELECT path, summary::text AS content, 0 AS source_order, '' AS creation_date, (summary_embedding <#> ${vecLit}) AS score FROM "${memoryTable}" WHERE summary_embedding IS NOT NULL${pathFilter} ORDER BY score DESC LIMIT ${semanticLimit}`; + const sessSemQuery = `SELECT path, message::text AS content, 1 AS source_order, COALESCE(creation_date::text, '') AS creation_date, (message_embedding <#> ${vecLit}) AS score FROM "${sessionsTable}" WHERE message_embedding IS NOT NULL${pathFilter} ORDER BY score DESC LIMIT ${semanticLimit}`; + const parts = [memSemQuery, sessSemQuery]; + if (memLexQuery) + parts.push(memLexQuery); + if (sessLexQuery) + parts.push(sessLexQuery); + const unionSql = parts.map((q) => `(${q})`).join(" UNION ALL "); + const outerLimit = semanticLimit + lexicalLimit; + const rows2 = await api.query(`SELECT path, content, source_order, creation_date, score FROM (` + unionSql + `) AS combined ORDER BY score DESC LIMIT ${outerLimit}`); + const seen = /* @__PURE__ */ new Set(); + const unique = []; + for (const row of rows2) { + const p = String(row["path"]); + if (seen.has(p)) + continue; + seen.add(p); + unique.push({ path: p, content: String(row["content"] ?? "") }); + } + return unique; + } const filterPatterns = contentScanOnly ? prefilterPatterns && prefilterPatterns.length > 0 ? prefilterPatterns : prefilterPattern ? [prefilterPattern] : [] : multiWordPatterns && multiWordPatterns.length > 1 ? multiWordPatterns : [escapedPattern]; const memFilter = buildContentFilter("summary::text", likeOp, filterPatterns); const sessFilter = buildContentFilter("message::text", likeOp, filterPatterns); @@ -633,6 +676,15 @@ async function searchDeeplakeTables(api, memoryTable, sessionsTable, opts) { content: String(row["content"] ?? "") })); } +function serializeFloat4Array(vec) { + const parts = []; + for (const v of vec) { + if (!Number.isFinite(v)) + return "NULL"; + parts.push(String(v)); + } + return `ARRAY[${parts.join(",")}]::float4[]`; +} function buildPathFilter(targetPath) { const condition = buildPathCondition(targetPath); return condition ? ` AND ${condition}` : ""; @@ -711,14 +763,25 @@ function buildGrepSearchOptions(params, targetPath) { const hasRegexMeta = !params.fixedString && /[.*+?^${}()|[\]\\]/.test(params.pattern); const literalPrefilter = hasRegexMeta ? extractRegexLiteralPrefilter(params.pattern) : null; const alternationPrefilters = hasRegexMeta ? extractRegexAlternationPrefilters(params.pattern) : null; + let bm25Term; + if (!hasRegexMeta) { + bm25Term = params.pattern; + } else if (alternationPrefilters && alternationPrefilters.length > 0) { + bm25Term = alternationPrefilters.join(" "); + } else if (literalPrefilter) { + bm25Term = literalPrefilter; + } const multiWordPatterns = !hasRegexMeta ? params.pattern.split(/\s+/).filter((w) => w.length > 2).slice(0, 4) : []; return { pathFilter: buildPathFilter(targetPath), contentScanOnly: hasRegexMeta, - likeOp: params.ignoreCase ? "ILIKE" : "LIKE", + // Kept for the pure-lexical fallback path and existing tests; BM25 is now + // the preferred lexical ranker when a term can be extracted. + likeOp: process.env.HIVEMIND_GREP_LIKE === "case-sensitive" ? "LIKE" : "ILIKE", escapedPattern: sqlLike(params.pattern), prefilterPattern: literalPrefilter ? sqlLike(literalPrefilter) : void 0, prefilterPatterns: alternationPrefilters?.map((literal) => sqlLike(literal)), + bm25Term, multiWordPatterns: multiWordPatterns.length > 1 ? multiWordPatterns.map((w) => sqlLike(w)) : void 0 }; } @@ -770,11 +833,28 @@ function refineGrepMatches(rows, params, forceMultiFilePrefix) { } return output; } -async function grepBothTables(api, memoryTable, sessionsTable, params, targetPath) { - const rows = await searchDeeplakeTables(api, memoryTable, sessionsTable, buildGrepSearchOptions(params, targetPath)); +async function grepBothTables(api, memoryTable, sessionsTable, params, targetPath, queryEmbedding) { + const rows = await searchDeeplakeTables(api, memoryTable, sessionsTable, { + ...buildGrepSearchOptions(params, targetPath), + queryEmbedding + }); const seen = /* @__PURE__ */ new Set(); const unique = rows.filter((r) => seen.has(r.path) ? false : (seen.add(r.path), true)); const normalized = unique.map((r) => ({ path: r.path, content: normalizeContent(r.path, r.content) })); + if (queryEmbedding && queryEmbedding.length > 0) { + const emitAllLines = process.env.HIVEMIND_SEMANTIC_EMIT_ALL !== "false"; + if (emitAllLines) { + const lines = []; + for (const r of normalized) { + for (const line of r.content.split("\n")) { + const trimmed = line.trim(); + if (trimmed) + lines.push(`${r.path}:${line}`); + } + } + return lines; + } + } return refineGrepMatches(normalized, params); } @@ -818,7 +898,260 @@ function capOutputForClaude(output, options = {}) { return keptLines.join("\n") + footer; } +// dist/src/embeddings/client.js +import { connect } from "node:net"; +import { spawn } from "node:child_process"; +import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync3, readFileSync as readFileSync3 } from "node:fs"; + +// dist/src/embeddings/protocol.js +var DEFAULT_SOCKET_DIR = "/tmp"; +var DEFAULT_IDLE_TIMEOUT_MS = 15 * 60 * 1e3; +var DEFAULT_CLIENT_TIMEOUT_MS = 200; +function socketPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.sock`; +} +function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.pid`; +} + +// dist/src/embeddings/client.js +var log3 = (m) => log("embed-client", m); +function getUid() { + const uid = typeof process.getuid === "function" ? process.getuid() : void 0; + return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; +} +var EmbedClient = class { + socketPath; + pidPath; + timeoutMs; + daemonEntry; + autoSpawn; + spawnWaitMs; + nextId = 0; + constructor(opts = {}) { + const uid = getUid(); + const dir = opts.socketDir ?? "/tmp"; + this.socketPath = socketPathFor(uid, dir); + this.pidPath = pidPathFor(uid, dir); + this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON; + this.autoSpawn = opts.autoSpawn ?? true; + this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; + } + /** + * Returns an embedding vector, or null on timeout/failure. Hooks MUST treat + * null as "skip embedding column" — never block the write path on us. + * + * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns + * null AND kicks off a background spawn. The next call finds a ready daemon. + */ + async embed(text, kind = "document") { + let sock; + try { + sock = await this.connectOnce(); + } catch { + if (this.autoSpawn) + this.trySpawnDaemon(); + return null; + } + try { + const id = String(++this.nextId); + const req = { op: "embed", id, kind, text }; + const resp = await this.sendAndWait(sock, req); + if (resp.error || !("embedding" in resp) || !resp.embedding) { + log3(`embed err: ${resp.error ?? "no embedding"}`); + return null; + } + return resp.embedding; + } catch (e) { + const err = e instanceof Error ? e.message : String(e); + log3(`embed failed: ${err}`); + return null; + } finally { + try { + sock.end(); + } catch { + } + } + } + /** + * Wait up to spawnWaitMs for the daemon to accept connections, spawning if + * necessary. Meant for SessionStart / long-running batches — not the hot path. + */ + async warmup() { + try { + const s = await this.connectOnce(); + s.end(); + return true; + } catch { + if (!this.autoSpawn) + return false; + this.trySpawnDaemon(); + try { + const s = await this.waitForSocket(); + s.end(); + return true; + } catch { + return false; + } + } + } + connectOnce() { + return new Promise((resolve2, reject) => { + const sock = connect(this.socketPath); + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("connect timeout")); + }, this.timeoutMs); + sock.once("connect", () => { + clearTimeout(to); + resolve2(sock); + }); + sock.once("error", (e) => { + clearTimeout(to); + reject(e); + }); + }); + } + trySpawnDaemon() { + let fd; + try { + fd = openSync(this.pidPath, "wx", 384); + writeSync(fd, String(process.pid)); + } catch (e) { + if (this.isPidFileStale()) { + try { + unlinkSync(this.pidPath); + } catch { + } + try { + fd = openSync(this.pidPath, "wx", 384); + writeSync(fd, String(process.pid)); + } catch { + return; + } + } else { + return; + } + } + if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { + log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + try { + closeSync(fd); + unlinkSync(this.pidPath); + } catch { + } + return; + } + try { + const child = spawn(process.execPath, [this.daemonEntry], { + detached: true, + stdio: "ignore", + env: process.env + }); + child.unref(); + log3(`spawned daemon pid=${child.pid}`); + } finally { + closeSync(fd); + } + } + isPidFileStale() { + try { + const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const pid = Number(raw); + if (!pid || Number.isNaN(pid)) + return true; + try { + process.kill(pid, 0); + return false; + } catch { + return true; + } + } catch { + return true; + } + } + async waitForSocket() { + const deadline = Date.now() + this.spawnWaitMs; + let delay = 30; + while (Date.now() < deadline) { + await sleep2(delay); + delay = Math.min(delay * 1.5, 300); + if (!existsSync3(this.socketPath)) + continue; + try { + return await this.connectOnce(); + } catch { + } + } + throw new Error("daemon did not become ready within spawnWaitMs"); + } + sendAndWait(sock, req) { + return new Promise((resolve2, reject) => { + let buf = ""; + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("request timeout")); + }, this.timeoutMs); + sock.setEncoding("utf-8"); + sock.on("data", (chunk) => { + buf += chunk; + const nl = buf.indexOf("\n"); + if (nl === -1) + return; + const line = buf.slice(0, nl); + clearTimeout(to); + try { + resolve2(JSON.parse(line)); + } catch (e) { + reject(e); + } + }); + sock.on("error", (e) => { + clearTimeout(to); + reject(e); + }); + sock.write(JSON.stringify(req) + "\n"); + }); + } +}; +function sleep2(ms) { + return new Promise((r) => setTimeout(r, ms)); +} + +// dist/src/embeddings/disable.js +function embeddingsDisabled() { + return process.env.HIVEMIND_EMBEDDINGS === "false"; +} + // dist/src/hooks/grep-direct.js +import { fileURLToPath } from "node:url"; +import { dirname, join as join4 } from "node:path"; +var SEMANTIC_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); +var SEMANTIC_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); +function resolveDaemonPath() { + return join4(dirname(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); +} +var sharedEmbedClient = null; +function getEmbedClient() { + if (!sharedEmbedClient) { + sharedEmbedClient = new EmbedClient({ + daemonEntry: resolveDaemonPath(), + timeoutMs: SEMANTIC_TIMEOUT_MS + }); + } + return sharedEmbedClient; +} +function patternIsSemanticFriendly(pattern, fixedString) { + if (!pattern || pattern.length < 2) + return false; + if (fixedString) + return true; + const meta = pattern.match(/[|()\[\]{}+?^$\\]/g); + if (!meta) + return true; + return meta.length <= 1; +} function splitFirstPipelineStage(cmd) { const input = cmd.trim(); let quote = null; @@ -1056,7 +1389,15 @@ async function handleGrepDirect(api, table, sessionsTable, params) { invertMatch: params.invertMatch, fixedString: params.fixedString }; - const output = await grepBothTables(api, table, sessionsTable, matchParams, params.targetPath); + let queryEmbedding = null; + if (SEMANTIC_ENABLED && patternIsSemanticFriendly(params.pattern, params.fixedString)) { + try { + queryEmbedding = await getEmbedClient().embed(params.pattern, "query"); + } catch { + queryEmbedding = null; + } + } + const output = await grepBothTables(api, table, sessionsTable, matchParams, params.targetPath, queryEmbedding); const joined = output.join("\n") || "(no matches)"; return capOutputForClaude(joined, { kind: "grep" }); } @@ -1677,20 +2018,20 @@ async function executeCompiledBashCommand(api, memoryTable, sessionsTable, cmd, } // dist/src/hooks/query-cache.js -import { mkdirSync as mkdirSync2, readFileSync as readFileSync3, rmSync, writeFileSync as writeFileSync2 } from "node:fs"; -import { join as join4 } from "node:path"; +import { mkdirSync as mkdirSync2, readFileSync as readFileSync4, rmSync, writeFileSync as writeFileSync2 } from "node:fs"; +import { join as join5 } from "node:path"; import { homedir as homedir3 } from "node:os"; -var log3 = (msg) => log("query-cache", msg); -var DEFAULT_CACHE_ROOT = join4(homedir3(), ".deeplake", "query-cache"); +var log4 = (msg) => log("query-cache", msg); +var DEFAULT_CACHE_ROOT = join5(homedir3(), ".deeplake", "query-cache"); var INDEX_CACHE_FILE = "index.md"; function getSessionQueryCacheDir(sessionId, deps = {}) { const { cacheRoot = DEFAULT_CACHE_ROOT } = deps; - return join4(cacheRoot, sessionId); + return join5(cacheRoot, sessionId); } function readCachedIndexContent(sessionId, deps = {}) { - const { logFn = log3 } = deps; + const { logFn = log4 } = deps; try { - return readFileSync3(join4(getSessionQueryCacheDir(sessionId, deps), INDEX_CACHE_FILE), "utf-8"); + return readFileSync4(join5(getSessionQueryCacheDir(sessionId, deps), INDEX_CACHE_FILE), "utf-8"); } catch (e) { if (e?.code === "ENOENT") return null; @@ -1699,11 +2040,11 @@ function readCachedIndexContent(sessionId, deps = {}) { } } function writeCachedIndexContent(sessionId, content, deps = {}) { - const { logFn = log3 } = deps; + const { logFn = log4 } = deps; try { const dir = getSessionQueryCacheDir(sessionId, deps); mkdirSync2(dir, { recursive: true }); - writeFileSync2(join4(dir, INDEX_CACHE_FILE), content, "utf-8"); + writeFileSync2(join5(dir, INDEX_CACHE_FILE), content, "utf-8"); } catch (e) { logFn(`write failed for session=${sessionId}: ${e.message}`); } @@ -1711,13 +2052,13 @@ function writeCachedIndexContent(sessionId, content, deps = {}) { // dist/src/utils/direct-run.js import { resolve } from "node:path"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath as fileURLToPath2 } from "node:url"; function isDirectRun(metaUrl) { const entry = process.argv[1]; if (!entry) return false; try { - return resolve(fileURLToPath(metaUrl)) === resolve(entry); + return resolve(fileURLToPath2(metaUrl)) === resolve(entry); } catch { return false; } @@ -1725,8 +2066,8 @@ function isDirectRun(metaUrl) { // dist/src/hooks/memory-path-utils.js import { homedir as homedir4 } from "node:os"; -import { join as join5 } from "node:path"; -var MEMORY_PATH = join5(homedir4(), ".deeplake", "memory"); +import { join as join6 } from "node:path"; +var MEMORY_PATH = join6(homedir4(), ".deeplake", "memory"); var TILDE_PATH = "~/.deeplake/memory"; var HOME_VAR_PATH = "$HOME/.deeplake/memory"; var SAFE_BUILTINS = /* @__PURE__ */ new Set([ @@ -1842,13 +2183,13 @@ function rewritePaths(cmd) { } // dist/src/hooks/codex/pre-tool-use.js -var log4 = (msg) => log("codex-pre", msg); -var __bundleDir = dirname(fileURLToPath2(import.meta.url)); -var SHELL_BUNDLE = existsSync3(join6(__bundleDir, "shell", "deeplake-shell.js")) ? join6(__bundleDir, "shell", "deeplake-shell.js") : join6(__bundleDir, "..", "shell", "deeplake-shell.js"); +var log5 = (msg) => log("codex-pre", msg); +var __bundleDir = dirname2(fileURLToPath3(import.meta.url)); +var SHELL_BUNDLE = existsSync4(join7(__bundleDir, "shell", "deeplake-shell.js")) ? join7(__bundleDir, "shell", "deeplake-shell.js") : join7(__bundleDir, "..", "shell", "deeplake-shell.js"); function buildUnsupportedGuidance() { return "This command is not supported for ~/.deeplake/memory/ operations. Only bash builtins are available: cat, ls, grep, echo, jq, head, tail, sed, awk, wc, sort, find, etc. Do NOT use python, python3, node, curl, or other interpreters. Rewrite your command using only bash tools and retry."; } -function runVirtualShell(cmd, shellBundle = SHELL_BUNDLE, logFn = log4) { +function runVirtualShell(cmd, shellBundle = SHELL_BUNDLE, logFn = log5) { try { return execFileSync("node", [shellBundle, "-c", cmd], { encoding: "utf-8", @@ -1873,7 +2214,7 @@ function buildIndexContent(rows) { return lines.join("\n"); } async function processCodexPreToolUse(input, deps = {}) { - const { config = loadConfig(), createApi = (table, activeConfig) => new DeeplakeApi(activeConfig.token, activeConfig.apiUrl, activeConfig.orgId, activeConfig.workspaceId, table), executeCompiledBashCommandFn = executeCompiledBashCommand, readVirtualPathContentsFn = readVirtualPathContents, readVirtualPathContentFn = readVirtualPathContent, listVirtualPathRowsFn = listVirtualPathRows, findVirtualPathsFn = findVirtualPaths, handleGrepDirectFn = handleGrepDirect, readCachedIndexContentFn = readCachedIndexContent, writeCachedIndexContentFn = writeCachedIndexContent, runVirtualShellFn = runVirtualShell, shellBundle = SHELL_BUNDLE, logFn = log4 } = deps; + const { config = loadConfig(), createApi = (table, activeConfig) => new DeeplakeApi(activeConfig.token, activeConfig.apiUrl, activeConfig.orgId, activeConfig.workspaceId, table), executeCompiledBashCommandFn = executeCompiledBashCommand, readVirtualPathContentsFn = readVirtualPathContents, readVirtualPathContentFn = readVirtualPathContent, listVirtualPathRowsFn = listVirtualPathRows, findVirtualPathsFn = findVirtualPaths, handleGrepDirectFn = handleGrepDirect, readCachedIndexContentFn = readCachedIndexContent, writeCachedIndexContentFn = writeCachedIndexContent, runVirtualShellFn = runVirtualShell, shellBundle = SHELL_BUNDLE, logFn = log5 } = deps; const cmd = input.tool_input?.command ?? ""; logFn(`hook fired: cmd=${cmd}`); if (!touchesMemory(cmd)) @@ -2083,7 +2424,7 @@ async function main() { } if (isDirectRun(import.meta.url)) { main().catch((e) => { - log4(`fatal: ${e.message}`); + log5(`fatal: ${e.message}`); process.exit(0); }); } diff --git a/codex/bundle/session-start-setup.js b/codex/bundle/session-start-setup.js index 166c22b..c854fcc 100755 --- a/codex/bundle/session-start-setup.js +++ b/codex/bundle/session-start-setup.js @@ -381,10 +381,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(tbl)) { log2(`table "${tbl}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', summary_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${tbl}" created`); if (!tables.includes(tbl)) this._tablesCache = [...tables, tbl]; + } else { + try { + await this.query(`ALTER TABLE "${tbl}" ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add summary_embedding skipped: ${e.message}`); + } } } /** Create the sessions table (uses JSONB for message since every row is a JSON event). */ @@ -392,10 +398,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(name)) { log2(`table "${name}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, message_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${name}" created`); if (!tables.includes(name)) this._tablesCache = [...tables, name]; + } else { + try { + await this.query(`ALTER TABLE "${name}" ADD COLUMN IF NOT EXISTS message_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add message_embedding skipped: ${e.message}`); + } } await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`); } diff --git a/codex/bundle/shell/deeplake-shell.js b/codex/bundle/shell/deeplake-shell.js index 7e140a9..18e11f9 100755 --- a/codex/bundle/shell/deeplake-shell.js +++ b/codex/bundle/shell/deeplake-shell.js @@ -46081,14 +46081,14 @@ var require_turndown_cjs = __commonJS({ } else if (node.nodeType === 1) { replacement = replacementForNode.call(self2, node); } - return join7(output, replacement); + return join9(output, replacement); }, ""); } function postProcess(output) { var self2 = this; this.rules.forEach(function(rule) { if (typeof rule.append === "function") { - output = join7(output, rule.append(self2.options)); + output = join9(output, rule.append(self2.options)); } }); return output.replace(/^[\t\r\n]+/, "").replace(/[\t\r\n\s]+$/, ""); @@ -46100,7 +46100,7 @@ var require_turndown_cjs = __commonJS({ if (whitespace.leading || whitespace.trailing) content = content.trim(); return whitespace.leading + rule.replacement(content, node, this.options) + whitespace.trailing; } - function join7(output, replacement) { + function join9(output, replacement) { var s12 = trimTrailingNewlines(output); var s22 = trimLeadingNewlines(replacement); var nls = Math.max(output.length - s12.length, replacement.length - s22.length); @@ -67073,10 +67073,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(tbl)) { log2(`table "${tbl}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', summary_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${tbl}" created`); if (!tables.includes(tbl)) this._tablesCache = [...tables, tbl]; + } else { + try { + await this.query(`ALTER TABLE "${tbl}" ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]`); + } catch (e6) { + log2(`ALTER TABLE add summary_embedding skipped: ${e6.message}`); + } } } /** Create the sessions table (uses JSONB for message since every row is a JSON event). */ @@ -67084,10 +67090,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(name)) { log2(`table "${name}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, message_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${name}" created`); if (!tables.includes(name)) this._tablesCache = [...tables, name]; + } else { + try { + await this.query(`ALTER TABLE "${name}" ADD COLUMN IF NOT EXISTS message_embedding FLOAT4[]`); + } catch (e6) { + log2(`ALTER TABLE add message_embedding skipped: ${e6.message}`); + } } await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`); } @@ -67096,6 +67108,8 @@ var DeeplakeApi = class { // dist/src/shell/deeplake-fs.js import { basename as basename4, posix } from "node:path"; import { randomUUID as randomUUID2 } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import { dirname as dirname4, join as join7 } from "node:path"; // dist/src/shell/grep-core.js var TOOL_INPUT_FIELDS = [ @@ -67261,24 +67275,25 @@ function normalizeContent(path2, raw) { return raw; } if (Array.isArray(obj.turns)) { - const header = []; - if (obj.date_time) - header.push(`date: ${obj.date_time}`); - if (obj.speakers) { - const s10 = obj.speakers; - const names = [s10.speaker_a, s10.speaker_b].filter(Boolean).join(", "); - if (names) - header.push(`speakers: ${names}`); - } + const dateHeader = obj.date_time ? `(${String(obj.date_time)}) ` : ""; const lines = obj.turns.map((t6) => { const sp = String(t6?.speaker ?? t6?.name ?? "?").trim(); const tx = String(t6?.text ?? t6?.content ?? "").replace(/\s+/g, " ").trim(); const tag = t6?.dia_id ? `[${t6.dia_id}] ` : ""; - return `${tag}${sp}: ${tx}`; + return `${dateHeader}${tag}${sp}: ${tx}`; }); - const out2 = [...header, ...lines].join("\n"); + const out2 = lines.join("\n"); return out2.trim() ? out2 : raw; } + if (obj.turn && typeof obj.turn === "object" && !Array.isArray(obj.turn)) { + const t6 = obj.turn; + const sp = String(t6.speaker ?? t6.name ?? "?").trim(); + const tx = String(t6.text ?? t6.content ?? "").replace(/\s+/g, " ").trim(); + const tag = t6.dia_id ? `[${String(t6.dia_id)}] ` : ""; + const dateHeader = obj.date_time ? `(${String(obj.date_time)}) ` : ""; + const line = `${dateHeader}${tag}${sp}: ${tx}`; + return line.trim() ? line : raw; + } const stripRecalled = (t6) => { const i11 = t6.indexOf(""); if (i11 === -1) @@ -67321,8 +67336,38 @@ function buildPathCondition(targetPath) { return `(path = '${sqlStr(clean)}' OR path LIKE '${sqlLike(clean)}/%' ESCAPE '\\')`; } async function searchDeeplakeTables(api, memoryTable, sessionsTable, opts) { - const { pathFilter, contentScanOnly, likeOp, escapedPattern, prefilterPattern, prefilterPatterns, multiWordPatterns } = opts; + const { pathFilter, contentScanOnly, likeOp, escapedPattern, prefilterPattern, prefilterPatterns, queryEmbedding, bm25Term, multiWordPatterns } = opts; const limit = opts.limit ?? 100; + if (queryEmbedding && queryEmbedding.length > 0) { + const vecLit = serializeFloat4Array(queryEmbedding); + const semanticLimit = Math.min(limit, Number(process.env.HIVEMIND_SEMANTIC_LIMIT ?? "20")); + const lexicalLimit = Math.min(limit, Number(process.env.HIVEMIND_HYBRID_LEXICAL_LIMIT ?? "20")); + const filterPatternsForLex = contentScanOnly ? prefilterPatterns && prefilterPatterns.length > 0 ? prefilterPatterns : prefilterPattern ? [prefilterPattern] : [] : [escapedPattern]; + const memLexFilter = buildContentFilter("summary::text", likeOp, filterPatternsForLex); + const sessLexFilter = buildContentFilter("message::text", likeOp, filterPatternsForLex); + const memLexQuery = memLexFilter ? `SELECT path, summary::text AS content, 0 AS source_order, '' AS creation_date, 1.0 AS score FROM "${memoryTable}" WHERE 1=1${pathFilter}${memLexFilter} LIMIT ${lexicalLimit}` : null; + const sessLexQuery = sessLexFilter ? `SELECT path, message::text AS content, 1 AS source_order, COALESCE(creation_date::text, '') AS creation_date, 1.0 AS score FROM "${sessionsTable}" WHERE 1=1${pathFilter}${sessLexFilter} LIMIT ${lexicalLimit}` : null; + const memSemQuery = `SELECT path, summary::text AS content, 0 AS source_order, '' AS creation_date, (summary_embedding <#> ${vecLit}) AS score FROM "${memoryTable}" WHERE summary_embedding IS NOT NULL${pathFilter} ORDER BY score DESC LIMIT ${semanticLimit}`; + const sessSemQuery = `SELECT path, message::text AS content, 1 AS source_order, COALESCE(creation_date::text, '') AS creation_date, (message_embedding <#> ${vecLit}) AS score FROM "${sessionsTable}" WHERE message_embedding IS NOT NULL${pathFilter} ORDER BY score DESC LIMIT ${semanticLimit}`; + const parts = [memSemQuery, sessSemQuery]; + if (memLexQuery) + parts.push(memLexQuery); + if (sessLexQuery) + parts.push(sessLexQuery); + const unionSql = parts.map((q17) => `(${q17})`).join(" UNION ALL "); + const outerLimit = semanticLimit + lexicalLimit; + const rows2 = await api.query(`SELECT path, content, source_order, creation_date, score FROM (` + unionSql + `) AS combined ORDER BY score DESC LIMIT ${outerLimit}`); + const seen = /* @__PURE__ */ new Set(); + const unique = []; + for (const row of rows2) { + const p22 = String(row["path"]); + if (seen.has(p22)) + continue; + seen.add(p22); + unique.push({ path: p22, content: String(row["content"] ?? "") }); + } + return unique; + } const filterPatterns = contentScanOnly ? prefilterPatterns && prefilterPatterns.length > 0 ? prefilterPatterns : prefilterPattern ? [prefilterPattern] : [] : multiWordPatterns && multiWordPatterns.length > 1 ? multiWordPatterns : [escapedPattern]; const memFilter = buildContentFilter("summary::text", likeOp, filterPatterns); const sessFilter = buildContentFilter("message::text", likeOp, filterPatterns); @@ -67334,6 +67379,15 @@ async function searchDeeplakeTables(api, memoryTable, sessionsTable, opts) { content: String(row["content"] ?? "") })); } +function serializeFloat4Array(vec) { + const parts = []; + for (const v27 of vec) { + if (!Number.isFinite(v27)) + return "NULL"; + parts.push(String(v27)); + } + return `ARRAY[${parts.join(",")}]::float4[]`; +} function buildPathFilter(targetPath) { const condition = buildPathCondition(targetPath); return condition ? ` AND ${condition}` : ""; @@ -67422,14 +67476,25 @@ function buildGrepSearchOptions(params, targetPath) { const hasRegexMeta = !params.fixedString && /[.*+?^${}()|[\]\\]/.test(params.pattern); const literalPrefilter = hasRegexMeta ? extractRegexLiteralPrefilter(params.pattern) : null; const alternationPrefilters = hasRegexMeta ? extractRegexAlternationPrefilters(params.pattern) : null; + let bm25Term; + if (!hasRegexMeta) { + bm25Term = params.pattern; + } else if (alternationPrefilters && alternationPrefilters.length > 0) { + bm25Term = alternationPrefilters.join(" "); + } else if (literalPrefilter) { + bm25Term = literalPrefilter; + } const multiWordPatterns = !hasRegexMeta ? params.pattern.split(/\s+/).filter((w20) => w20.length > 2).slice(0, 4) : []; return { pathFilter: buildPathFilter(targetPath), contentScanOnly: hasRegexMeta, - likeOp: params.ignoreCase ? "ILIKE" : "LIKE", + // Kept for the pure-lexical fallback path and existing tests; BM25 is now + // the preferred lexical ranker when a term can be extracted. + likeOp: process.env.HIVEMIND_GREP_LIKE === "case-sensitive" ? "LIKE" : "ILIKE", escapedPattern: sqlLike(params.pattern), prefilterPattern: literalPrefilter ? sqlLike(literalPrefilter) : void 0, prefilterPatterns: alternationPrefilters?.map((literal) => sqlLike(literal)), + bm25Term, multiWordPatterns: multiWordPatterns.length > 1 ? multiWordPatterns.map((w20) => sqlLike(w20)) : void 0 }; } @@ -67482,6 +67547,245 @@ function refineGrepMatches(rows, params, forceMultiFilePrefix) { return output; } +// dist/src/embeddings/client.js +import { connect } from "node:net"; +import { spawn } from "node:child_process"; +import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync4, readFileSync as readFileSync3 } from "node:fs"; + +// dist/src/embeddings/protocol.js +var DEFAULT_SOCKET_DIR = "/tmp"; +var DEFAULT_IDLE_TIMEOUT_MS = 15 * 60 * 1e3; +var DEFAULT_CLIENT_TIMEOUT_MS = 200; +function socketPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.sock`; +} +function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.pid`; +} + +// dist/src/embeddings/client.js +var log3 = (m26) => log("embed-client", m26); +function getUid() { + const uid = typeof process.getuid === "function" ? process.getuid() : void 0; + return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; +} +var EmbedClient = class { + socketPath; + pidPath; + timeoutMs; + daemonEntry; + autoSpawn; + spawnWaitMs; + nextId = 0; + constructor(opts = {}) { + const uid = getUid(); + const dir = opts.socketDir ?? "/tmp"; + this.socketPath = socketPathFor(uid, dir); + this.pidPath = pidPathFor(uid, dir); + this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON; + this.autoSpawn = opts.autoSpawn ?? true; + this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; + } + /** + * Returns an embedding vector, or null on timeout/failure. Hooks MUST treat + * null as "skip embedding column" — never block the write path on us. + * + * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns + * null AND kicks off a background spawn. The next call finds a ready daemon. + */ + async embed(text, kind = "document") { + let sock; + try { + sock = await this.connectOnce(); + } catch { + if (this.autoSpawn) + this.trySpawnDaemon(); + return null; + } + try { + const id = String(++this.nextId); + const req = { op: "embed", id, kind, text }; + const resp = await this.sendAndWait(sock, req); + if (resp.error || !("embedding" in resp) || !resp.embedding) { + log3(`embed err: ${resp.error ?? "no embedding"}`); + return null; + } + return resp.embedding; + } catch (e6) { + const err = e6 instanceof Error ? e6.message : String(e6); + log3(`embed failed: ${err}`); + return null; + } finally { + try { + sock.end(); + } catch { + } + } + } + /** + * Wait up to spawnWaitMs for the daemon to accept connections, spawning if + * necessary. Meant for SessionStart / long-running batches — not the hot path. + */ + async warmup() { + try { + const s10 = await this.connectOnce(); + s10.end(); + return true; + } catch { + if (!this.autoSpawn) + return false; + this.trySpawnDaemon(); + try { + const s10 = await this.waitForSocket(); + s10.end(); + return true; + } catch { + return false; + } + } + } + connectOnce() { + return new Promise((resolve5, reject) => { + const sock = connect(this.socketPath); + const to3 = setTimeout(() => { + sock.destroy(); + reject(new Error("connect timeout")); + }, this.timeoutMs); + sock.once("connect", () => { + clearTimeout(to3); + resolve5(sock); + }); + sock.once("error", (e6) => { + clearTimeout(to3); + reject(e6); + }); + }); + } + trySpawnDaemon() { + let fd; + try { + fd = openSync(this.pidPath, "wx", 384); + writeSync(fd, String(process.pid)); + } catch (e6) { + if (this.isPidFileStale()) { + try { + unlinkSync(this.pidPath); + } catch { + } + try { + fd = openSync(this.pidPath, "wx", 384); + writeSync(fd, String(process.pid)); + } catch { + return; + } + } else { + return; + } + } + if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { + log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + try { + closeSync(fd); + unlinkSync(this.pidPath); + } catch { + } + return; + } + try { + const child = spawn(process.execPath, [this.daemonEntry], { + detached: true, + stdio: "ignore", + env: process.env + }); + child.unref(); + log3(`spawned daemon pid=${child.pid}`); + } finally { + closeSync(fd); + } + } + isPidFileStale() { + try { + const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const pid = Number(raw); + if (!pid || Number.isNaN(pid)) + return true; + try { + process.kill(pid, 0); + return false; + } catch { + return true; + } + } catch { + return true; + } + } + async waitForSocket() { + const deadline = Date.now() + this.spawnWaitMs; + let delay = 30; + while (Date.now() < deadline) { + await sleep2(delay); + delay = Math.min(delay * 1.5, 300); + if (!existsSync4(this.socketPath)) + continue; + try { + return await this.connectOnce(); + } catch { + } + } + throw new Error("daemon did not become ready within spawnWaitMs"); + } + sendAndWait(sock, req) { + return new Promise((resolve5, reject) => { + let buf = ""; + const to3 = setTimeout(() => { + sock.destroy(); + reject(new Error("request timeout")); + }, this.timeoutMs); + sock.setEncoding("utf-8"); + sock.on("data", (chunk) => { + buf += chunk; + const nl3 = buf.indexOf("\n"); + if (nl3 === -1) + return; + const line = buf.slice(0, nl3); + clearTimeout(to3); + try { + resolve5(JSON.parse(line)); + } catch (e6) { + reject(e6); + } + }); + sock.on("error", (e6) => { + clearTimeout(to3); + reject(e6); + }); + sock.write(JSON.stringify(req) + "\n"); + }); + } +}; +function sleep2(ms3) { + return new Promise((r10) => setTimeout(r10, ms3)); +} + +// dist/src/embeddings/sql.js +function embeddingSqlLiteral(vec) { + if (!vec || vec.length === 0) + return "NULL"; + const parts = []; + for (const v27 of vec) { + if (!Number.isFinite(v27)) + return "NULL"; + parts.push(String(v27)); + } + return `ARRAY[${parts.join(",")}]::float4[]`; +} + +// dist/src/embeddings/disable.js +function embeddingsDisabled() { + return process.env.HIVEMIND_EMBEDDINGS === "false"; +} + // dist/src/shell/deeplake-fs.js var BATCH_SIZE = 10; var PREFETCH_BATCH_SIZE = 50; @@ -67510,6 +67814,9 @@ function normalizeSessionMessage(path2, message) { const raw = typeof message === "string" ? message : JSON.stringify(message); return normalizeContent(path2, raw); } +function resolveEmbedDaemonPath() { + return join7(dirname4(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); +} function joinSessionMessages(path2, messages) { return messages.map((message) => normalizeSessionMessage(path2, message)).join("\n"); } @@ -67539,6 +67846,8 @@ var DeeplakeFs = class _DeeplakeFs { // Paths that live in the sessions table (multi-row, read by concatenation) sessionPaths = /* @__PURE__ */ new Set(); sessionsTable = null; + // Embedding client lazily created on first flush. Lives as long as the process. + embedClient = null; constructor(client, table, mountPoint) { this.client = client; this.table = table; @@ -67572,7 +67881,14 @@ var DeeplakeFs = class _DeeplakeFs { })(); const sessionsBootstrap = sessionsTable && sessionSyncOk ? (async () => { try { - const sessionRows = await client.query(`SELECT path, SUM(size_bytes) as total_size FROM "${sessionsTable}" GROUP BY path ORDER BY path`); + const sessionRows = await client.query( + // NOTE: SUM(size_bytes) returns NULL on the Deeplake backend when combined + // with GROUP BY path (confirmed against workspace `with_embedding`). MAX + // works and — for the single-row-per-file layout — is equal to SUM. For + // multi-row-per-turn layouts MAX under-reports total size but stays >0 + // so files don't look like empty placeholders in ls/stat. + `SELECT path, MAX(size_bytes) as total_size FROM "${sessionsTable}" GROUP BY path ORDER BY path` + ); for (const row of sessionRows) { const p22 = row["path"]; if (!fs3.files.has(p22)) { @@ -67632,7 +67948,8 @@ var DeeplakeFs = class _DeeplakeFs { } const rows = [...this.pending.values()]; this.pending.clear(); - const results = await Promise.allSettled(rows.map((r10) => this.upsertRow(r10))); + const embeddings = await this.computeEmbeddings(rows); + const results = await Promise.allSettled(rows.map((r10, i11) => this.upsertRow(r10, embeddings[i11]))); let failures = 0; for (let i11 = 0; i11 < results.length; i11++) { if (results[i11].status === "rejected") { @@ -67646,7 +67963,17 @@ var DeeplakeFs = class _DeeplakeFs { throw new Error(`flush: ${failures}/${rows.length} writes failed and were re-queued`); } } - async upsertRow(r10) { + async computeEmbeddings(rows) { + if (rows.length === 0) + return []; + if (embeddingsDisabled()) + return rows.map(() => null); + if (!this.embedClient) { + this.embedClient = new EmbedClient({ daemonEntry: resolveEmbedDaemonPath() }); + } + return Promise.all(rows.map((r10) => this.embedClient.embed(r10.contentText, "document"))); + } + async upsertRow(r10, embedding) { const text = sqlStr(r10.contentText); const p22 = sqlStr(r10.path); const fname = sqlStr(r10.filename); @@ -67654,8 +67981,9 @@ var DeeplakeFs = class _DeeplakeFs { const ts3 = (/* @__PURE__ */ new Date()).toISOString(); const cd = r10.creationDate ?? ts3; const lud = r10.lastUpdateDate ?? ts3; + const embSql = embeddingSqlLiteral(embedding); if (this.flushed.has(r10.path)) { - let setClauses = `filename = '${fname}', summary = E'${text}', mime_type = '${mime}', size_bytes = ${r10.sizeBytes}, last_update_date = '${sqlStr(lud)}'`; + let setClauses = `filename = '${fname}', summary = E'${text}', summary_embedding = ${embSql}, mime_type = '${mime}', size_bytes = ${r10.sizeBytes}, last_update_date = '${sqlStr(lud)}'`; if (r10.project !== void 0) setClauses += `, project = '${sqlStr(r10.project)}'`; if (r10.description !== void 0) @@ -67663,51 +67991,72 @@ var DeeplakeFs = class _DeeplakeFs { await this.client.query(`UPDATE "${this.table}" SET ${setClauses} WHERE path = '${p22}'`); } else { const id = randomUUID2(); - const cols = "id, path, filename, summary, mime_type, size_bytes, creation_date, last_update_date" + (r10.project !== void 0 ? ", project" : "") + (r10.description !== void 0 ? ", description" : ""); - const vals = `'${id}', '${p22}', '${fname}', E'${text}', '${mime}', ${r10.sizeBytes}, '${sqlStr(cd)}', '${sqlStr(lud)}'` + (r10.project !== void 0 ? `, '${sqlStr(r10.project)}'` : "") + (r10.description !== void 0 ? `, '${sqlStr(r10.description)}'` : ""); + const cols = "id, path, filename, summary, summary_embedding, mime_type, size_bytes, creation_date, last_update_date" + (r10.project !== void 0 ? ", project" : "") + (r10.description !== void 0 ? ", description" : ""); + const vals = `'${id}', '${p22}', '${fname}', E'${text}', ${embSql}, '${mime}', ${r10.sizeBytes}, '${sqlStr(cd)}', '${sqlStr(lud)}'` + (r10.project !== void 0 ? `, '${sqlStr(r10.project)}'` : "") + (r10.description !== void 0 ? `, '${sqlStr(r10.description)}'` : ""); await this.client.query(`INSERT INTO "${this.table}" (${cols}) VALUES (${vals})`); this.flushed.add(r10.path); } } // ── Virtual index.md generation ──────────────────────────────────────────── async generateVirtualIndex() { - const rows = await this.client.query(`SELECT path, project, description, creation_date, last_update_date FROM "${this.table}" WHERE path LIKE '${sqlStr("/summaries/")}%' ORDER BY last_update_date DESC`); - const sessionPathsByKey = /* @__PURE__ */ new Map(); - for (const sp of this.sessionPaths) { - const hivemind = sp.match(/\/sessions\/[^/]+\/[^/]+_([^.]+)\.jsonl$/); - if (hivemind) { - sessionPathsByKey.set(hivemind[1], sp.slice(1)); - } else { - const fname = sp.split("/").pop() ?? ""; - const stem = fname.replace(/\.[^.]+$/, ""); - if (stem) - sessionPathsByKey.set(stem, sp.slice(1)); + const summaryRows = await this.client.query(`SELECT path, project, description, creation_date, last_update_date FROM "${this.table}" WHERE path LIKE '${sqlStr("/summaries/")}%' ORDER BY last_update_date DESC`); + let sessionRows = []; + if (this.sessionsTable) { + try { + sessionRows = await this.client.query(`SELECT path, MAX(description) AS description, MIN(creation_date) AS creation_date, MAX(last_update_date) AS last_update_date FROM "${this.sessionsTable}" WHERE path LIKE '${sqlStr("/sessions/")}%' GROUP BY path ORDER BY path`); + } catch { + sessionRows = []; } } const lines = [ "# Session Index", "", - "List of all Claude Code sessions with summaries.", - "", - "| Session | Conversation | Created | Last Updated | Project | Description |", - "|---------|-------------|---------|--------------|---------|-------------|" + "Two sources are available. Consult the section relevant to the question.", + "" ]; - for (const row of rows) { - const p22 = row["path"]; - const match2 = p22.match(/\/summaries\/([^/]+)\/([^/]+)\.md$/); - if (!match2) - continue; - const summaryUser = match2[1]; - const sessionId = match2[2]; - const relPath = `summaries/${summaryUser}/${sessionId}.md`; - const baseName = sessionId.replace(/_summary$/, ""); - const convPath = sessionPathsByKey.get(sessionId) ?? sessionPathsByKey.get(baseName); - const convLink = convPath ? `[messages](${convPath})` : ""; - const project = row["project"] || ""; - const description = row["description"] || ""; - const creationDate = row["creation_date"] || ""; - const lastUpdateDate = row["last_update_date"] || ""; - lines.push(`| [${sessionId}](${relPath}) | ${convLink} | ${creationDate} | ${lastUpdateDate} | ${project} | ${description} |`); + lines.push("## memory"); + lines.push(""); + if (summaryRows.length === 0) { + lines.push("_(empty \u2014 no summaries ingested yet)_"); + } else { + lines.push("AI-generated summaries per session. Read these first for topic-level overviews."); + lines.push(""); + lines.push("| Session | Created | Last Updated | Project | Description |"); + lines.push("|---------|---------|--------------|---------|-------------|"); + for (const row of summaryRows) { + const p22 = row["path"]; + const match2 = p22.match(/\/summaries\/([^/]+)\/([^/]+)\.md$/); + if (!match2) + continue; + const summaryUser = match2[1]; + const sessionId = match2[2]; + const relPath = `summaries/${summaryUser}/${sessionId}.md`; + const project = row["project"] || ""; + const description = row["description"] || ""; + const creationDate = row["creation_date"] || ""; + const lastUpdateDate = row["last_update_date"] || ""; + lines.push(`| [${sessionId}](${relPath}) | ${creationDate} | ${lastUpdateDate} | ${project} | ${description} |`); + } + } + lines.push(""); + lines.push("## sessions"); + lines.push(""); + if (sessionRows.length === 0) { + lines.push("_(empty \u2014 no session records ingested yet)_"); + } else { + lines.push("Raw session records (dialogue, tool calls). Read for exact detail / quotes."); + lines.push(""); + lines.push("| Session | Created | Last Updated | Description |"); + lines.push("|---------|---------|--------------|-------------|"); + for (const row of sessionRows) { + const p22 = row["path"] || ""; + const rel = p22.startsWith("/") ? p22.slice(1) : p22; + const filename = p22.split("/").pop() ?? p22; + const description = row["description"] || ""; + const creationDate = row["creation_date"] || ""; + const lastUpdateDate = row["last_update_date"] || ""; + lines.push(`| [${filename}](${rel}) | ${creationDate} | ${lastUpdateDate} | ${description} |`); + } } lines.push(""); return lines.join("\n"); @@ -69018,7 +69367,7 @@ function stripQuotes(val) { } // node_modules/yargs-parser/build/lib/index.js -import { readFileSync as readFileSync3 } from "fs"; +import { readFileSync as readFileSync4 } from "fs"; import { createRequire } from "node:module"; var _a3; var _b; @@ -69045,7 +69394,7 @@ var parser = new YargsParser({ if (typeof require2 !== "undefined") { return require2(path2); } else if (path2.match(/\.json$/)) { - return JSON.parse(readFileSync3(path2, "utf8")); + return JSON.parse(readFileSync4(path2, "utf8")); } else { throw Error("only .json config files are supported in ESM"); } @@ -69064,6 +69413,33 @@ yargsParser.looksLikeNumber = looksLikeNumber; var lib_default = yargsParser; // dist/src/shell/grep-interceptor.js +import { fileURLToPath as fileURLToPath2 } from "node:url"; +import { dirname as dirname5, join as join8 } from "node:path"; +var SEMANTIC_SEARCH_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); +var SEMANTIC_EMBED_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); +function resolveGrepEmbedDaemonPath() { + return join8(dirname5(fileURLToPath2(import.meta.url)), "..", "embeddings", "embed-daemon.js"); +} +var sharedGrepEmbedClient = null; +function getGrepEmbedClient() { + if (!sharedGrepEmbedClient) { + sharedGrepEmbedClient = new EmbedClient({ + daemonEntry: resolveGrepEmbedDaemonPath(), + timeoutMs: SEMANTIC_EMBED_TIMEOUT_MS + }); + } + return sharedGrepEmbedClient; +} +function patternIsSemanticFriendly(pattern, fixedString) { + if (!pattern || pattern.length < 2) + return false; + if (fixedString) + return true; + const metaMatches = pattern.match(/[|()\[\]{}+?^$\\]/g); + if (!metaMatches) + return true; + return metaMatches.length <= 1; +} var MAX_FALLBACK_CANDIDATES = 500; function createGrepCommand(client, fs3, table, sessionsTable) { return Yi2("grep", async (args, ctx) => { @@ -69105,12 +69481,21 @@ function createGrepCommand(client, fs3, table, sessionsTable) { filesOnly: Boolean(parsed.l || parsed["files-with-matches"]), countOnly: Boolean(parsed.c || parsed["count"]) }; + let queryEmbedding = null; + if (SEMANTIC_SEARCH_ENABLED && patternIsSemanticFriendly(pattern, matchParams.fixedString)) { + try { + queryEmbedding = await getGrepEmbedClient().embed(pattern, "query"); + } catch { + queryEmbedding = null; + } + } let rows = []; try { const searchOptions = { ...buildGrepSearchOptions(matchParams, targets[0] ?? ctx.cwd), pathFilter: buildPathFilterForTargets(targets), - limit: 100 + limit: 100, + queryEmbedding }; const queryRows = await Promise.race([ searchDeeplakeTables(client, table, sessionsTable ?? "sessions", searchOptions), @@ -69120,6 +69505,21 @@ function createGrepCommand(client, fs3, table, sessionsTable) { } catch { rows = []; } + if (rows.length === 0 && queryEmbedding) { + try { + const lexicalOptions = { + ...buildGrepSearchOptions(matchParams, targets[0] ?? ctx.cwd), + pathFilter: buildPathFilterForTargets(targets), + limit: 100 + }; + const lexicalRows = await Promise.race([ + searchDeeplakeTables(client, table, sessionsTable ?? "sessions", lexicalOptions), + new Promise((_16, reject) => setTimeout(() => reject(new Error("timeout")), 3e3)) + ]); + rows.push(...lexicalRows); + } catch { + } + } const seen = /* @__PURE__ */ new Set(); rows = rows.filter((r10) => seen.has(r10.path) ? false : (seen.add(r10.path), true)); if (rows.length === 0) { @@ -69133,7 +69533,19 @@ function createGrepCommand(client, fs3, table, sessionsTable) { } } const normalized = rows.map((r10) => ({ path: r10.path, content: normalizeContent(r10.path, r10.content) })); - const output = refineGrepMatches(normalized, matchParams); + let output; + if (queryEmbedding && queryEmbedding.length > 0 && process.env.HIVEMIND_SEMANTIC_EMIT_ALL !== "false") { + output = []; + for (const r10 of normalized) { + for (const line of r10.content.split("\n")) { + const trimmed = line.trim(); + if (trimmed) + output.push(`${r10.path}:${line}`); + } + } + } else { + output = refineGrepMatches(normalized, matchParams); + } return { stdout: output.length > 0 ? output.join("\n") + "\n" : "", stderr: "", diff --git a/codex/bundle/stop.js b/codex/bundle/stop.js index 729c2c4..9e1563e 100755 --- a/codex/bundle/stop.js +++ b/codex/bundle/stop.js @@ -373,10 +373,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(tbl)) { log2(`table "${tbl}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${tbl}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', summary_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'text/plain', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${tbl}" created`); if (!tables.includes(tbl)) this._tablesCache = [...tables, tbl]; + } else { + try { + await this.query(`ALTER TABLE "${tbl}" ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add summary_embedding skipped: ${e.message}`); + } } } /** Create the sessions table (uses JSONB for message since every row is a JSON event). */ @@ -384,10 +390,16 @@ var DeeplakeApi = class { const tables = await this.listTables(); if (!tables.includes(name)) { log2(`table "${name}" not found, creating`); - await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); + await this.query(`CREATE TABLE IF NOT EXISTS "${name}" (id TEXT NOT NULL DEFAULT '', path TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', message JSONB, message_embedding FLOAT4[], author TEXT NOT NULL DEFAULT '', mime_type TEXT NOT NULL DEFAULT 'application/json', size_bytes BIGINT NOT NULL DEFAULT 0, project TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT '', creation_date TEXT NOT NULL DEFAULT '', last_update_date TEXT NOT NULL DEFAULT '') USING deeplake`); log2(`table "${name}" created`); if (!tables.includes(name)) this._tablesCache = [...tables, name]; + } else { + try { + await this.query(`ALTER TABLE "${name}" ADD COLUMN IF NOT EXISTS message_embedding FLOAT4[]`); + } catch (e) { + log2(`ALTER TABLE add message_embedding skipped: ${e.message}`); + } } await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`); } diff --git a/codex/bundle/wiki-worker.js b/codex/bundle/wiki-worker.js index bf134ed..5fffb2e 100755 --- a/codex/bundle/wiki-worker.js +++ b/codex/bundle/wiki-worker.js @@ -1,9 +1,10 @@ #!/usr/bin/env node // dist/src/hooks/codex/wiki-worker.js -import { readFileSync as readFileSync2, writeFileSync as writeFileSync2, existsSync as existsSync2, appendFileSync as appendFileSync2, mkdirSync as mkdirSync2, rmSync } from "node:fs"; +import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, existsSync as existsSync3, appendFileSync as appendFileSync2, mkdirSync as mkdirSync2, rmSync } from "node:fs"; import { execFileSync } from "node:child_process"; -import { join as join3 } from "node:path"; +import { dirname, join as join3 } from "node:path"; +import { fileURLToPath } from "node:url"; // dist/src/hooks/summary-state.js import { readFileSync, writeFileSync, writeSync, mkdirSync, renameSync, existsSync, unlinkSync, openSync, closeSync } from "node:fs"; @@ -106,6 +107,21 @@ function releaseLock(sessionId) { // dist/src/hooks/upload-summary.js import { randomUUID } from "node:crypto"; + +// dist/src/embeddings/sql.js +function embeddingSqlLiteral(vec) { + if (!vec || vec.length === 0) + return "NULL"; + const parts = []; + for (const v of vec) { + if (!Number.isFinite(v)) + return "NULL"; + parts.push(String(v)); + } + return `ARRAY[${parts.join(",")}]::float4[]`; +} + +// dist/src/hooks/upload-summary.js function esc(s) { return s.replace(/\\/g, "\\\\").replace(/'/g, "''").replace(/[\x01-\x08\x0b\x0c\x0e-\x1f\x7f]/g, ""); } @@ -118,20 +134,247 @@ async function uploadSummary(query2, params) { const ts = params.ts ?? (/* @__PURE__ */ new Date()).toISOString(); const desc = extractDescription(text); const sizeBytes = Buffer.byteLength(text); + const embSql = embeddingSqlLiteral(params.embedding ?? null); const existing = await query2(`SELECT path FROM "${tableName}" WHERE path = '${esc(vpath)}' LIMIT 1`); if (existing.length > 0) { - const sql2 = `UPDATE "${tableName}" SET summary = E'${esc(text)}', size_bytes = ${sizeBytes}, description = E'${esc(desc)}', last_update_date = '${ts}' WHERE path = '${esc(vpath)}'`; + const sql2 = `UPDATE "${tableName}" SET summary = E'${esc(text)}', summary_embedding = ${embSql}, size_bytes = ${sizeBytes}, description = E'${esc(desc)}', last_update_date = '${ts}' WHERE path = '${esc(vpath)}'`; await query2(sql2); return { path: "update", sql: sql2, descLength: desc.length, summaryLength: text.length }; } - const sql = `INSERT INTO "${tableName}" (id, path, filename, summary, author, mime_type, size_bytes, project, description, agent, creation_date, last_update_date) VALUES ('${randomUUID()}', '${esc(vpath)}', '${esc(fname)}', E'${esc(text)}', '${esc(userName)}', 'text/markdown', ${sizeBytes}, '${esc(project)}', E'${esc(desc)}', '${esc(agent)}', '${ts}', '${ts}')`; + const sql = `INSERT INTO "${tableName}" (id, path, filename, summary, summary_embedding, author, mime_type, size_bytes, project, description, agent, creation_date, last_update_date) VALUES ('${randomUUID()}', '${esc(vpath)}', '${esc(fname)}', E'${esc(text)}', ${embSql}, '${esc(userName)}', 'text/markdown', ${sizeBytes}, '${esc(project)}', E'${esc(desc)}', '${esc(agent)}', '${ts}', '${ts}')`; await query2(sql); return { path: "insert", sql, descLength: desc.length, summaryLength: text.length }; } +// dist/src/embeddings/client.js +import { connect } from "node:net"; +import { spawn } from "node:child_process"; +import { openSync as openSync2, closeSync as closeSync2, writeSync as writeSync2, unlinkSync as unlinkSync2, existsSync as existsSync2, readFileSync as readFileSync2 } from "node:fs"; + +// dist/src/embeddings/protocol.js +var DEFAULT_SOCKET_DIR = "/tmp"; +var DEFAULT_IDLE_TIMEOUT_MS = 15 * 60 * 1e3; +var DEFAULT_CLIENT_TIMEOUT_MS = 200; +function socketPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.sock`; +} +function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.pid`; +} + +// dist/src/embeddings/client.js +var log2 = (m) => log("embed-client", m); +function getUid() { + const uid = typeof process.getuid === "function" ? process.getuid() : void 0; + return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; +} +var EmbedClient = class { + socketPath; + pidPath; + timeoutMs; + daemonEntry; + autoSpawn; + spawnWaitMs; + nextId = 0; + constructor(opts = {}) { + const uid = getUid(); + const dir = opts.socketDir ?? "/tmp"; + this.socketPath = socketPathFor(uid, dir); + this.pidPath = pidPathFor(uid, dir); + this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON; + this.autoSpawn = opts.autoSpawn ?? true; + this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; + } + /** + * Returns an embedding vector, or null on timeout/failure. Hooks MUST treat + * null as "skip embedding column" — never block the write path on us. + * + * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns + * null AND kicks off a background spawn. The next call finds a ready daemon. + */ + async embed(text, kind = "document") { + let sock; + try { + sock = await this.connectOnce(); + } catch { + if (this.autoSpawn) + this.trySpawnDaemon(); + return null; + } + try { + const id = String(++this.nextId); + const req = { op: "embed", id, kind, text }; + const resp = await this.sendAndWait(sock, req); + if (resp.error || !("embedding" in resp) || !resp.embedding) { + log2(`embed err: ${resp.error ?? "no embedding"}`); + return null; + } + return resp.embedding; + } catch (e) { + const err = e instanceof Error ? e.message : String(e); + log2(`embed failed: ${err}`); + return null; + } finally { + try { + sock.end(); + } catch { + } + } + } + /** + * Wait up to spawnWaitMs for the daemon to accept connections, spawning if + * necessary. Meant for SessionStart / long-running batches — not the hot path. + */ + async warmup() { + try { + const s = await this.connectOnce(); + s.end(); + return true; + } catch { + if (!this.autoSpawn) + return false; + this.trySpawnDaemon(); + try { + const s = await this.waitForSocket(); + s.end(); + return true; + } catch { + return false; + } + } + } + connectOnce() { + return new Promise((resolve, reject) => { + const sock = connect(this.socketPath); + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("connect timeout")); + }, this.timeoutMs); + sock.once("connect", () => { + clearTimeout(to); + resolve(sock); + }); + sock.once("error", (e) => { + clearTimeout(to); + reject(e); + }); + }); + } + trySpawnDaemon() { + let fd; + try { + fd = openSync2(this.pidPath, "wx", 384); + writeSync2(fd, String(process.pid)); + } catch (e) { + if (this.isPidFileStale()) { + try { + unlinkSync2(this.pidPath); + } catch { + } + try { + fd = openSync2(this.pidPath, "wx", 384); + writeSync2(fd, String(process.pid)); + } catch { + return; + } + } else { + return; + } + } + if (!this.daemonEntry || !existsSync2(this.daemonEntry)) { + log2(`daemonEntry not configured or missing: ${this.daemonEntry}`); + try { + closeSync2(fd); + unlinkSync2(this.pidPath); + } catch { + } + return; + } + try { + const child = spawn(process.execPath, [this.daemonEntry], { + detached: true, + stdio: "ignore", + env: process.env + }); + child.unref(); + log2(`spawned daemon pid=${child.pid}`); + } finally { + closeSync2(fd); + } + } + isPidFileStale() { + try { + const raw = readFileSync2(this.pidPath, "utf-8").trim(); + const pid = Number(raw); + if (!pid || Number.isNaN(pid)) + return true; + try { + process.kill(pid, 0); + return false; + } catch { + return true; + } + } catch { + return true; + } + } + async waitForSocket() { + const deadline = Date.now() + this.spawnWaitMs; + let delay = 30; + while (Date.now() < deadline) { + await sleep(delay); + delay = Math.min(delay * 1.5, 300); + if (!existsSync2(this.socketPath)) + continue; + try { + return await this.connectOnce(); + } catch { + } + } + throw new Error("daemon did not become ready within spawnWaitMs"); + } + sendAndWait(sock, req) { + return new Promise((resolve, reject) => { + let buf = ""; + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("request timeout")); + }, this.timeoutMs); + sock.setEncoding("utf-8"); + sock.on("data", (chunk) => { + buf += chunk; + const nl = buf.indexOf("\n"); + if (nl === -1) + return; + const line = buf.slice(0, nl); + clearTimeout(to); + try { + resolve(JSON.parse(line)); + } catch (e) { + reject(e); + } + }); + sock.on("error", (e) => { + clearTimeout(to); + reject(e); + }); + sock.write(JSON.stringify(req) + "\n"); + }); + } +}; +function sleep(ms) { + return new Promise((r) => setTimeout(r, ms)); +} + +// dist/src/embeddings/disable.js +function embeddingsDisabled() { + return process.env.HIVEMIND_EMBEDDINGS === "false"; +} + // dist/src/hooks/codex/wiki-worker.js var dlog2 = (msg) => log("codex-wiki-worker", msg); -var cfg = JSON.parse(readFileSync2(process.argv[2], "utf-8")); +var cfg = JSON.parse(readFileSync3(process.argv[2], "utf-8")); var tmpDir = cfg.tmpDir; var tmpJsonl = join3(tmpDir, "session.jsonl"); var tmpSummary = join3(tmpDir, "summary.md"); @@ -225,11 +468,20 @@ async function main() { } catch (e) { wlog(`codex exec failed: ${e.status ?? e.message}`); } - if (existsSync2(tmpSummary)) { - const text = readFileSync2(tmpSummary, "utf-8"); + if (existsSync3(tmpSummary)) { + const text = readFileSync3(tmpSummary, "utf-8"); if (text.trim()) { const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; + let embedding = null; + if (!embeddingsDisabled()) { + try { + const daemonEntry = join3(dirname(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); + embedding = await new EmbedClient({ daemonEntry }).embed(text, "document"); + } catch (e) { + wlog(`summary embedding failed, writing NULL: ${e.message}`); + } + } const result = await uploadSummary(query, { tableName: cfg.memoryTable, vpath, @@ -238,7 +490,8 @@ async function main() { project: cfg.project, agent: "codex", sessionId: cfg.sessionId, - text + text, + embedding }); wlog(`uploaded ${vpath} (summary=${result.summaryLength}, desc=${result.descLength})`); try { diff --git a/codex/package.json b/codex/package.json index 2712d4c..9f305d8 100644 --- a/codex/package.json +++ b/codex/package.json @@ -1,6 +1,6 @@ { "name": "hivemind-codex", - "version": "0.6.46", + "version": "0.7.0", "description": "Cloud-backed persistent shared memory for OpenAI Codex CLI powered by Deeplake", "type": "module" } diff --git a/esbuild.config.mjs b/esbuild.config.mjs index 7b05a62..7556c43 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -23,7 +23,11 @@ const ccCommands = [ { entry: "dist/src/commands/auth-login.js", out: "commands/auth-login" }, ]; -const ccAll = [...ccHooks, ...ccShell, ...ccCommands]; +const ccEmbed = [ + { entry: "dist/src/embeddings/daemon.js", out: "embeddings/embed-daemon" }, +]; + +const ccAll = [...ccHooks, ...ccShell, ...ccCommands, ...ccEmbed]; await build({ entryPoints: Object.fromEntries(ccAll.map(h => [h.out, h.entry])), @@ -31,7 +35,15 @@ await build({ platform: "node", format: "esm", outdir: "claude-code/bundle", - external: ["node:*", "node-liblzma", "@mongodb-js/zstd"], + external: [ + "node:*", + "node-liblzma", + "@mongodb-js/zstd", + "@huggingface/transformers", + "onnxruntime-node", + "onnxruntime-common", + "sharp", + ], }); for (const h of ccAll) { @@ -57,7 +69,11 @@ const codexCommands = [ { entry: "dist/src/commands/auth-login.js", out: "commands/auth-login" }, ]; -const codexAll = [...codexHooks, ...codexShell, ...codexCommands]; +const codexEmbed = [ + { entry: "dist/src/embeddings/daemon.js", out: "embeddings/embed-daemon" }, +]; + +const codexAll = [...codexHooks, ...codexShell, ...codexCommands, ...codexEmbed]; await build({ entryPoints: Object.fromEntries(codexAll.map(h => [h.out, h.entry])), @@ -65,7 +81,15 @@ await build({ platform: "node", format: "esm", outdir: "codex/bundle", - external: ["node:*", "node-liblzma", "@mongodb-js/zstd"], + external: [ + "node:*", + "node-liblzma", + "@mongodb-js/zstd", + "@huggingface/transformers", + "onnxruntime-node", + "onnxruntime-common", + "sharp", + ], }); for (const h of codexAll) { diff --git a/package-lock.json b/package-lock.json index c897ae3..cffa848 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,13 +1,14 @@ { "name": "hivemind", - "version": "0.6.46", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "hivemind", - "version": "0.6.46", + "version": "0.7.0", "dependencies": { + "@huggingface/transformers": "^3.0.0", "deeplake": "^0.3.30", "just-bash": "^2.14.0", "yargs-parser": "^22.0.0" @@ -1544,12 +1545,32 @@ "node": ">=18" } }, + "node_modules/@huggingface/jinja": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.7.tgz", + "integrity": "sha512-OosMEbF/R6zkKNNzqhI7kvKYCpo1F0UeIv46/h4D4UjVEKKd6k3TiV8sgu6fkreX4lbBiRI+lZG8UnXnqVQmEQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/transformers": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.8.1.tgz", + "integrity": "sha512-tsTk4zVjImqdqjS8/AOZg2yNLd1z9S5v+7oUPpXaasDRwEDhB+xnglK1k5cad26lL5/ZIaeREgWWy0bs9y9pPA==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.5.3", + "onnxruntime-node": "1.21.0", + "onnxruntime-web": "1.22.0-dev.20250409-89f8206ba4", + "sharp": "^0.34.1" + } + }, "node_modules/@img/colour": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", "license": "MIT", - "optional": true, "engines": { "node": ">=18" } @@ -2010,6 +2031,18 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@jitl/quickjs-ffi-types": { "version": "0.32.0", "resolved": "https://registry.npmjs.org/@jitl/quickjs-ffi-types/-/quickjs-ffi-types-0.32.0.tgz", @@ -2233,6 +2266,70 @@ "url": "https://github.com/sponsors/Boshen" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.0.0-rc.13", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.13.tgz", @@ -3346,7 +3443,6 @@ "version": "25.5.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.2.tgz", "integrity": "sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~7.18.0" @@ -3686,6 +3782,13 @@ "node": ">=8.9" } }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT" + }, "node_modules/bowser": { "version": "2.14.1", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", @@ -4048,16 +4151,55 @@ "sharp": "^0.34.5" } }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=8" } }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "license": "MIT" + }, "node_modules/diff": { "version": "8.0.4", "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", @@ -4123,7 +4265,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -4133,7 +4274,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -4159,6 +4299,12 @@ "node": ">= 0.4" } }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.28.0", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.0.tgz", @@ -4201,6 +4347,18 @@ "@esbuild/win32-x64": "0.28.0" } }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/estree-walker": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", @@ -4396,6 +4554,12 @@ "node": ">=8" } }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -4544,11 +4708,43 @@ "node": ">= 6" } }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/globalthis": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", + "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", + "license": "MIT", + "dependencies": { + "define-properties": "^1.2.1", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -4570,6 +4766,12 @@ "integrity": "sha512-8tLu60LgxF6XpdbK8OW3FA+IfTNBn1ZHGHKF4KQbEeSkajYw5PlYJcKluntgegDPTg8UkHjpet1T82vk6TQ68w==", "license": "MIT" }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -4580,6 +4782,18 @@ "node": ">=8" } }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -4909,6 +5123,12 @@ "node": ">= 6" } }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "license": "ISC" + }, "node_modules/jsonfile": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", @@ -5314,6 +5534,12 @@ "url": "https://github.com/chalk/slice-ansi?sponsor=1" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -5366,6 +5592,18 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -5481,6 +5719,27 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", @@ -5623,6 +5882,15 @@ "node": ">=0.10.0" } }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/obug": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", @@ -5660,6 +5928,49 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/onnxruntime-common": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.21.0.tgz", + "integrity": "sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.21.0.tgz", + "integrity": "sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "global-agent": "^3.0.0", + "onnxruntime-common": "1.21.0", + "tar": "^7.0.1" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.22.0-dev.20250409-89f8206ba4", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.22.0-dev.20250409-89f8206ba4.tgz", + "integrity": "sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.22.0-dev.20250409-89f8206ba4", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.22.0-dev.20250409-89f8206ba4.tgz", + "integrity": "sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ==", + "license": "MIT" + }, "node_modules/papaparse": { "version": "5.5.3", "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.5.3.tgz", @@ -5821,6 +6132,12 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, "node_modules/postcss": { "version": "8.5.8", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", @@ -5931,6 +6248,30 @@ "asap": "~2.0.3" } }, + "node_modules/protobufjs": { + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/pug": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/pug/-/pug-3.0.4.tgz", @@ -6252,6 +6593,23 @@ "dev": true, "license": "MIT" }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, "node_modules/rolldown": { "version": "1.0.0-rc.13", "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.13.tgz", @@ -6335,7 +6693,6 @@ "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "devOptional": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -6344,13 +6701,33 @@ "node": ">=10" } }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "license": "MIT" + }, + "node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/sharp": { "version": "0.34.5", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", "hasInstallScript": true, "license": "Apache-2.0", - "optional": true, "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", @@ -6688,6 +7065,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/tar": { + "version": "7.5.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.13.tgz", + "integrity": "sha512-tOG/7GyXpFevhXVh8jOPJrmtRpOTsYqUIkVdVooZYJS/z8WhfQUX8RJILmeuJNinGAMSu1veBr4asSHFt5/hng==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/tar-fs": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", @@ -6718,6 +7111,15 @@ "node": ">=6" } }, + "node_modules/tar/node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -7337,6 +7739,18 @@ "npm": ">=9" } }, + "node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/typescript": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.2.tgz", @@ -7367,7 +7781,6 @@ "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "dev": true, "license": "MIT" }, "node_modules/universalify": { @@ -7667,6 +8080,15 @@ "node": ">=0.4" } }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/yaml": { "version": "2.8.3", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", diff --git a/package.json b/package.json index 14f6ee4..788b312 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hivemind", - "version": "0.6.46", + "version": "0.7.0", "description": "Cloud-backed persistent shared memory for AI agents powered by Deeplake", "type": "module", "bin": { @@ -24,6 +24,7 @@ "*.md": [] }, "dependencies": { + "@huggingface/transformers": "^3.0.0", "deeplake": "^0.3.30", "just-bash": "^2.14.0", "yargs-parser": "^22.0.0" diff --git a/src/deeplake-api.ts b/src/deeplake-api.ts index 5e763b2..2933384 100644 --- a/src/deeplake-api.ts +++ b/src/deeplake-api.ts @@ -356,6 +356,7 @@ export class DeeplakeApi { `path TEXT NOT NULL DEFAULT '', ` + `filename TEXT NOT NULL DEFAULT '', ` + `summary TEXT NOT NULL DEFAULT '', ` + + `summary_embedding FLOAT4[], ` + `author TEXT NOT NULL DEFAULT '', ` + `mime_type TEXT NOT NULL DEFAULT 'text/plain', ` + `size_bytes BIGINT NOT NULL DEFAULT 0, ` + @@ -368,6 +369,19 @@ export class DeeplakeApi { ); log(`table "${tbl}" created`); if (!tables.includes(tbl)) this._tablesCache = [...tables, tbl]; + } else { + // Migrate older memory tables that were created before the embeddings + // feature landed. ADD COLUMN IF NOT EXISTS is idempotent on Postgres + // and Deeplake; we swallow errors because on backends that don't + // support it the column just stays absent and the embedding-column + // writes fall through to NULL (capture / flush tolerate that). + try { + await this.query( + `ALTER TABLE "${tbl}" ADD COLUMN IF NOT EXISTS summary_embedding FLOAT4[]`, + ); + } catch (e: any) { + log(`ALTER TABLE add summary_embedding skipped: ${e.message}`); + } } // BM25 index disabled — CREATE INDEX causes intermittent oid errors on fresh tables. // See bm25-oid-bug.sh for reproduction. Re-enable once Deeplake fixes the oid invalidation. @@ -389,6 +403,7 @@ export class DeeplakeApi { `path TEXT NOT NULL DEFAULT '', ` + `filename TEXT NOT NULL DEFAULT '', ` + `message JSONB, ` + + `message_embedding FLOAT4[], ` + `author TEXT NOT NULL DEFAULT '', ` + `mime_type TEXT NOT NULL DEFAULT 'application/json', ` + `size_bytes BIGINT NOT NULL DEFAULT 0, ` + @@ -401,6 +416,15 @@ export class DeeplakeApi { ); log(`table "${name}" created`); if (!tables.includes(name)) this._tablesCache = [...tables, name]; + } else { + // Same rationale as ensureTable: migrate pre-embeddings sessions tables. + try { + await this.query( + `ALTER TABLE "${name}" ADD COLUMN IF NOT EXISTS message_embedding FLOAT4[]`, + ); + } catch (e: any) { + log(`ALTER TABLE add message_embedding skipped: ${e.message}`); + } } await this.ensureLookupIndex(name, "path_creation_date", `("path", "creation_date")`); } diff --git a/src/embeddings/client.ts b/src/embeddings/client.ts new file mode 100644 index 0000000..9798829 --- /dev/null +++ b/src/embeddings/client.ts @@ -0,0 +1,242 @@ +// Thin client used by hooks to request embeddings from the daemon. +// Self-heals: if the socket is missing, the first caller spawns the daemon +// under an O_EXCL pidfile lock so concurrent callers don't spawn duplicates. + +import { connect, type Socket } from "node:net"; +import { spawn } from "node:child_process"; +import { openSync, closeSync, writeSync, unlinkSync, existsSync, readFileSync } from "node:fs"; +import { + DEFAULT_CLIENT_TIMEOUT_MS, + pidPathFor, + socketPathFor, + type DaemonResponse, + type EmbedKind, + type EmbedRequest, +} from "./protocol.js"; +import { log as _log } from "../utils/debug.js"; + +const log = (m: string) => _log("embed-client", m); + +function getUid(): string { + const uid = typeof process.getuid === "function" ? process.getuid() : undefined; + return uid !== undefined ? String(uid) : (process.env.USER ?? "default"); +} + +export interface ClientOptions { + socketDir?: string; + timeoutMs?: number; + daemonEntry?: string; // path to bundled embed-daemon.js + autoSpawn?: boolean; + spawnWaitMs?: number; +} + +export class EmbedClient { + private socketPath: string; + private pidPath: string; + private timeoutMs: number; + private daemonEntry: string | undefined; + private autoSpawn: boolean; + private spawnWaitMs: number; + private nextId = 0; + + constructor(opts: ClientOptions = {}) { + const uid = getUid(); + const dir = opts.socketDir ?? "/tmp"; + this.socketPath = socketPathFor(uid, dir); + this.pidPath = pidPathFor(uid, dir); + this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON; + this.autoSpawn = opts.autoSpawn ?? true; + this.spawnWaitMs = opts.spawnWaitMs ?? 5000; + } + + /** + * Returns an embedding vector, or null on timeout/failure. Hooks MUST treat + * null as "skip embedding column" — never block the write path on us. + * + * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns + * null AND kicks off a background spawn. The next call finds a ready daemon. + */ + async embed(text: string, kind: EmbedKind = "document"): Promise { + let sock: Socket; + try { + sock = await this.connectOnce(); + } catch { + if (this.autoSpawn) this.trySpawnDaemon(); + return null; + } + try { + const id = String(++this.nextId); + const req: EmbedRequest = { op: "embed", id, kind, text }; + const resp = await this.sendAndWait(sock, req); + if (resp.error || !("embedding" in resp) || !resp.embedding) { + log(`embed err: ${resp.error ?? "no embedding"}`); + return null; + } + return resp.embedding; + } catch (e: unknown) { + const err = e instanceof Error ? e.message : String(e); + log(`embed failed: ${err}`); + return null; + } finally { + try { sock.end(); } catch { /* best-effort */ } + } + } + + /** + * Wait up to spawnWaitMs for the daemon to accept connections, spawning if + * necessary. Meant for SessionStart / long-running batches — not the hot path. + */ + async warmup(): Promise { + try { + const s = await this.connectOnce(); + s.end(); + return true; + } catch { + if (!this.autoSpawn) return false; + this.trySpawnDaemon(); + try { + const s = await this.waitForSocket(); + s.end(); + return true; + } catch { + return false; + } + } + } + + private connectOnce(): Promise { + return new Promise((resolve, reject) => { + const sock = connect(this.socketPath); + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("connect timeout")); + }, this.timeoutMs); + sock.once("connect", () => { + clearTimeout(to); + resolve(sock); + }); + sock.once("error", (e) => { + clearTimeout(to); + reject(e); + }); + }); + } + + private trySpawnDaemon(): void { + // O_EXCL pidfile — only the first caller wins. Others find the pid file + // and wait for the socket to appear. + // + // Race subtlety: we IMMEDIATELY write our own PID into the file to close + // the window where another worker could see an empty pidfile and interpret + // it as "stale". The daemon itself overwrites the file with its own PID + // during startup (see daemon.ts start()). + let fd: number; + try { + fd = openSync(this.pidPath, "wx", 0o600); + writeSync(fd, String(process.pid)); + } catch (e: unknown) { + // Someone else is spawning (EEXIST) — or pidfile is stale. If stale, clean up and retry. + if (this.isPidFileStale()) { + try { unlinkSync(this.pidPath); } catch { /* best-effort */ } + try { + fd = openSync(this.pidPath, "wx", 0o600); + writeSync(fd, String(process.pid)); + } catch { + return; // someone else just claimed it; let waitForSocket handle it + } + } else { + return; + } + } + + if (!this.daemonEntry || !existsSync(this.daemonEntry)) { + log(`daemonEntry not configured or missing: ${this.daemonEntry}`); + try { closeSync(fd); unlinkSync(this.pidPath); } catch { /* best-effort */ } + return; + } + + try { + const child = spawn(process.execPath, [this.daemonEntry], { + detached: true, + stdio: "ignore", + env: process.env, + }); + child.unref(); + log(`spawned daemon pid=${child.pid}`); + } finally { + closeSync(fd); + } + } + + private isPidFileStale(): boolean { + try { + const raw = readFileSync(this.pidPath, "utf-8").trim(); + const pid = Number(raw); + if (!pid || Number.isNaN(pid)) return true; + // kill(pid, 0) throws if process is gone. + try { + process.kill(pid, 0); + // Process is alive — the daemon might just be loading the model and + // hasn't bound the socket yet. DON'T treat as stale; let waitForSocket + // poll. A hung daemon will eventually time out at the caller. + return false; + } catch { + return true; + } + } catch { + return true; + } + } + + private async waitForSocket(): Promise { + const deadline = Date.now() + this.spawnWaitMs; + let delay = 30; + while (Date.now() < deadline) { + await sleep(delay); + delay = Math.min(delay * 1.5, 300); + if (!existsSync(this.socketPath)) continue; + try { + return await this.connectOnce(); + } catch { + // socket appeared but daemon not ready yet — keep waiting + } + } + throw new Error("daemon did not become ready within spawnWaitMs"); + } + + private sendAndWait(sock: Socket, req: EmbedRequest): Promise { + return new Promise((resolve, reject) => { + let buf = ""; + const to = setTimeout(() => { + sock.destroy(); + reject(new Error("request timeout")); + }, this.timeoutMs); + sock.setEncoding("utf-8"); + sock.on("data", (chunk: string) => { + buf += chunk; + const nl = buf.indexOf("\n"); + if (nl === -1) return; + const line = buf.slice(0, nl); + clearTimeout(to); + try { + resolve(JSON.parse(line) as DaemonResponse); + } catch (e) { + reject(e as Error); + } + }); + sock.on("error", (e) => { clearTimeout(to); reject(e); }); + sock.write(JSON.stringify(req) + "\n"); + }); + } +} + +function sleep(ms: number): Promise { + return new Promise(r => setTimeout(r, ms)); +} + +let singleton: EmbedClient | null = null; +export function getEmbedClient(): EmbedClient { + if (!singleton) singleton = new EmbedClient(); + return singleton; +} diff --git a/src/embeddings/daemon.ts b/src/embeddings/daemon.ts new file mode 100644 index 0000000..32951b0 --- /dev/null +++ b/src/embeddings/daemon.ts @@ -0,0 +1,159 @@ +#!/usr/bin/env node + +// Long-lived embedding daemon. Holds the nomic model in RAM and serves +// embed requests over a per-user Unix socket. Exits after an idle window +// so it doesn't sit around consuming ~200 MB of RAM forever. + +import { createServer, type Server, type Socket } from "node:net"; +import { unlinkSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs"; +import { NomicEmbedder } from "./nomic.js"; +import { + DEFAULT_IDLE_TIMEOUT_MS, + pidPathFor, + socketPathFor, + type DaemonRequest, + type DaemonResponse, + type EmbedRequest, + type PingRequest, +} from "./protocol.js"; +import { log as _log } from "../utils/debug.js"; + +const log = (m: string) => _log("embed-daemon", m); + +function getUid(): string { + const uid = typeof process.getuid === "function" ? process.getuid() : undefined; + return uid !== undefined ? String(uid) : (process.env.USER ?? "default"); +} + +export interface DaemonOptions { + socketDir?: string; + idleTimeoutMs?: number; + dims?: number; + dtype?: string; + repo?: string; +} + +export class EmbedDaemon { + private server: Server | null = null; + private embedder: NomicEmbedder; + private socketPath: string; + private pidPath: string; + private idleTimeoutMs: number; + private idleTimer: NodeJS.Timeout | null = null; + + constructor(opts: DaemonOptions = {}) { + const uid = getUid(); + const dir = opts.socketDir ?? "/tmp"; + this.socketPath = socketPathFor(uid, dir); + this.pidPath = pidPathFor(uid, dir); + this.idleTimeoutMs = opts.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS; + this.embedder = new NomicEmbedder({ repo: opts.repo, dtype: opts.dtype, dims: opts.dims }); + } + + async start(): Promise { + mkdirSync(this.socketPath.replace(/\/[^/]+$/, ""), { recursive: true }); + // Overwrite pidfile FIRST — the client wrote its own (transient) pid as a + // placeholder during spawn to avoid a race; now that the daemon is live, + // replace it with ours so subsequent clients see the long-lived pid. + writeFileSync(this.pidPath, String(process.pid), { mode: 0o600 }); + if (existsSync(this.socketPath)) { + // Stale from a previous crash. unlink so bind() can succeed. + try { unlinkSync(this.socketPath); } catch { /* best-effort */ } + } + + // Warmup the model in the background so the first real request is fast. + this.embedder.load().then(() => log("model ready")).catch(e => log(`load err: ${e.message}`)); + + this.server = createServer((sock) => this.handleConnection(sock)); + await new Promise((resolve, reject) => { + this.server!.once("error", reject); + this.server!.listen(this.socketPath, () => { + try { chmodSync(this.socketPath, 0o600); } catch { /* best-effort */ } + log(`listening on ${this.socketPath}`); + resolve(); + }); + }); + + this.resetIdleTimer(); + process.on("SIGINT", () => this.shutdown()); + process.on("SIGTERM", () => this.shutdown()); + } + + private resetIdleTimer(): void { + if (this.idleTimer) clearTimeout(this.idleTimer); + this.idleTimer = setTimeout(() => { + log(`idle timeout ${this.idleTimeoutMs}ms reached, shutting down`); + this.shutdown(); + }, this.idleTimeoutMs); + this.idleTimer.unref(); + } + + shutdown(): void { + try { this.server?.close(); } catch { /* best-effort */ } + try { if (existsSync(this.socketPath)) unlinkSync(this.socketPath); } catch { /* best-effort */ } + try { if (existsSync(this.pidPath)) unlinkSync(this.pidPath); } catch { /* best-effort */ } + process.exit(0); + } + + private handleConnection(sock: Socket): void { + let buf = ""; + sock.setEncoding("utf-8"); + sock.on("data", (chunk: string) => { + buf += chunk; + let nl: number; + while ((nl = buf.indexOf("\n")) !== -1) { + const line = buf.slice(0, nl); + buf = buf.slice(nl + 1); + if (line.length === 0) continue; + this.handleLine(sock, line); + } + }); + sock.on("error", () => { /* client disconnect is normal */ }); + } + + private async handleLine(sock: Socket, line: string): Promise { + this.resetIdleTimer(); + let req: DaemonRequest; + try { + req = JSON.parse(line); + } catch { + return; + } + try { + const resp = await this.dispatch(req); + sock.write(JSON.stringify(resp) + "\n"); + } catch (e: unknown) { + const err = e instanceof Error ? e.message : String(e); + const resp: DaemonResponse = { id: req.id, error: err }; + sock.write(JSON.stringify(resp) + "\n"); + } + } + + private async dispatch(req: DaemonRequest): Promise { + if (req.op === "ping") { + const p = req as PingRequest; + return { id: p.id, ready: true, model: this.embedder.repo, dims: this.embedder.dims }; + } + if (req.op === "embed") { + const e = req as EmbedRequest; + const vec = await this.embedder.embed(e.text, e.kind); + return { id: e.id, embedding: vec }; + } + return { id: (req as { id: string }).id, error: "unknown op" }; + } +} + +/* v8 ignore start — CLI entrypoint, only runs when file is node's argv[1] */ +const invokedDirectly = import.meta.url === `file://${process.argv[1]}` + || (process.argv[1] && import.meta.url.endsWith(process.argv[1].split("/").pop() ?? "")); + +if (invokedDirectly) { + const dims = process.env.HIVEMIND_EMBED_DIMS ? Number(process.env.HIVEMIND_EMBED_DIMS) : undefined; + const idleTimeoutMs = process.env.HIVEMIND_EMBED_IDLE_MS ? Number(process.env.HIVEMIND_EMBED_IDLE_MS) : undefined; + const d = new EmbedDaemon({ dims, idleTimeoutMs }); + d.start().catch((e) => { + log(`fatal: ${e.message}`); + process.exit(1); + }); +} +/* v8 ignore stop */ diff --git a/src/embeddings/disable.ts b/src/embeddings/disable.ts new file mode 100644 index 0000000..8c0db83 --- /dev/null +++ b/src/embeddings/disable.ts @@ -0,0 +1,22 @@ +/** + * Master opt-out for the embedding feature. + * + * `HIVEMIND_EMBEDDINGS=false` short-circuits every call site that would + * otherwise talk to the nomic daemon: the SessionStart warmup, the + * capture-side write embed, the batched flush embed in DeeplakeFs, and + * both grep query-time embed paths (direct + interceptor). The SQL + * schema still has the embedding columns and existing rows' embeddings + * remain readable, but no new embedding is computed and no daemon is + * spawned. + * + * Intended for: air-gapped / no-network installs, CI / benchmarks that + * want pure-lexical retrieval, and users who want the plugin's capture + * + grep without paying the ~110 MB nomic download. + * + * Read-once: honours mutations during the process lifetime; the hooks + * are short-lived subprocesses so a live toggle via `export` takes + * effect on the next session. + */ +export function embeddingsDisabled(): boolean { + return process.env.HIVEMIND_EMBEDDINGS === "false"; +} diff --git a/src/embeddings/nomic.ts b/src/embeddings/nomic.ts new file mode 100644 index 0000000..d9dd77d --- /dev/null +++ b/src/embeddings/nomic.ts @@ -0,0 +1,90 @@ +// Thin wrapper around @huggingface/transformers. Only loaded inside the daemon +// process — hooks never import this. Kept isolated so the heavyweight transformer +// dependency is not pulled into every bundled hook. + +import { + DEFAULT_DIMS, + DEFAULT_DTYPE, + DEFAULT_MODEL_REPO, + DOC_PREFIX, + QUERY_PREFIX, + type EmbedKind, +} from "./protocol.js"; + +type Embedder = (input: string | string[], opts: Record) => Promise<{ data: Float32Array | number[] }>; + +export interface NomicOptions { + repo?: string; + dtype?: string; + dims?: number; +} + +export class NomicEmbedder { + private pipeline: Embedder | null = null; + private loading: Promise | null = null; + readonly repo: string; + readonly dtype: string; + readonly dims: number; + + constructor(opts: NomicOptions = {}) { + this.repo = opts.repo ?? DEFAULT_MODEL_REPO; + this.dtype = opts.dtype ?? DEFAULT_DTYPE; + this.dims = opts.dims ?? DEFAULT_DIMS; + } + + async load(): Promise { + if (this.pipeline) return; + if (this.loading) return this.loading; + this.loading = (async () => { + const mod = await import("@huggingface/transformers"); + mod.env.allowLocalModels = false; + mod.env.useFSCache = true; + this.pipeline = (await mod.pipeline("feature-extraction", this.repo, { dtype: this.dtype as "fp32" | "q8" })) as unknown as Embedder; + })(); + try { + await this.loading; + } finally { + this.loading = null; + } + } + + private addPrefix(text: string, kind: EmbedKind): string { + return (kind === "query" ? QUERY_PREFIX : DOC_PREFIX) + text; + } + + async embed(text: string, kind: EmbedKind = "document"): Promise { + await this.load(); + if (!this.pipeline) throw new Error("embedder not loaded"); + const out = await this.pipeline(this.addPrefix(text, kind), { pooling: "mean", normalize: true }); + const full = Array.from(out.data as ArrayLike); + return this.truncate(full); + } + + async embedBatch(texts: string[], kind: EmbedKind = "document"): Promise { + if (texts.length === 0) return []; + await this.load(); + if (!this.pipeline) throw new Error("embedder not loaded"); + const prefixed = texts.map(t => this.addPrefix(t, kind)); + const out = await this.pipeline(prefixed, { pooling: "mean", normalize: true }); + const flat = Array.from(out.data as ArrayLike); + const total = flat.length; + const full = total / texts.length; + const batches: number[][] = []; + for (let i = 0; i < texts.length; i++) { + batches.push(this.truncate(flat.slice(i * full, (i + 1) * full))); + } + return batches; + } + + private truncate(vec: number[]): number[] { + if (this.dims >= vec.length) return vec; + // Matryoshka: truncate then re-normalize. + const head = vec.slice(0, this.dims); + let norm = 0; + for (const v of head) norm += v * v; + norm = Math.sqrt(norm); + if (norm === 0) return head; + for (let i = 0; i < head.length; i++) head[i] /= norm; + return head; + } +} diff --git a/src/embeddings/protocol.ts b/src/embeddings/protocol.ts new file mode 100644 index 0000000..9959ced --- /dev/null +++ b/src/embeddings/protocol.ts @@ -0,0 +1,50 @@ +// Shared types for the embedding daemon <-> client IPC. +// Newline-delimited JSON over Unix socket. + +export type EmbedKind = "document" | "query"; + +export interface EmbedRequest { + op: "embed"; + id: string; + kind: EmbedKind; + text: string; +} + +export interface EmbedResponse { + id: string; + embedding?: number[]; + error?: string; +} + +export interface PingRequest { + op: "ping"; + id: string; +} + +export interface PingResponse { + id: string; + ready: boolean; + model?: string; + dims?: number; + error?: string; +} + +export type DaemonRequest = EmbedRequest | PingRequest; +export type DaemonResponse = EmbedResponse | PingResponse; + +export const DEFAULT_SOCKET_DIR = "/tmp"; +export const DEFAULT_MODEL_REPO = "nomic-ai/nomic-embed-text-v1.5"; +export const DEFAULT_DTYPE = "q8"; +export const DEFAULT_DIMS = 768; +export const DEFAULT_IDLE_TIMEOUT_MS = 15 * 60 * 1000; +export const DEFAULT_CLIENT_TIMEOUT_MS = 200; +export const DOC_PREFIX = "search_document: "; +export const QUERY_PREFIX = "search_query: "; + +export function socketPathFor(uid: number | string, dir = DEFAULT_SOCKET_DIR): string { + return `${dir}/hivemind-embed-${uid}.sock`; +} + +export function pidPathFor(uid: number | string, dir = DEFAULT_SOCKET_DIR): string { + return `${dir}/hivemind-embed-${uid}.pid`; +} diff --git a/src/embeddings/sql.ts b/src/embeddings/sql.ts new file mode 100644 index 0000000..4387f7e --- /dev/null +++ b/src/embeddings/sql.ts @@ -0,0 +1,16 @@ +// Helpers for embedding values in SQL. Deeplake stores vectors as `FLOAT4[]`; +// the literal form is `ARRAY[f1, f2, ...]::float4[]`. When the embedding is +// missing (daemon unavailable, timeout, etc.) we emit `NULL`. + +export function embeddingSqlLiteral(vec: number[] | null | undefined): string { + if (!vec || vec.length === 0) return "NULL"; + // FLOAT4 is IEEE-754 single-precision. `toFixed` would lose precision; use + // the raw JS Number → string conversion which yields the shortest round-trip. + // Safety: only allow finite numbers; otherwise NULL. + const parts: string[] = []; + for (const v of vec) { + if (!Number.isFinite(v)) return "NULL"; + parts.push(String(v)); + } + return `ARRAY[${parts.join(",")}]::float4[]`; +} diff --git a/src/hooks/capture.ts b/src/hooks/capture.ts index 81c8385..1c5b3a3 100644 --- a/src/hooks/capture.ts +++ b/src/hooks/capture.ts @@ -21,8 +21,17 @@ import { releaseLock, } from "./summary-state.js"; import { bundleDirFromImportMeta, spawnWikiWorker, wikiLog } from "./spawn-wiki-worker.js"; +import { EmbedClient } from "../embeddings/client.js"; +import { embeddingSqlLiteral } from "../embeddings/sql.js"; +import { embeddingsDisabled } from "../embeddings/disable.js"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; const log = (msg: string) => _log("capture", msg); +function resolveEmbedDaemonPath(): string { + return join(dirname(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); +} + interface HookInput { session_id: string; transcript_path?: string; @@ -115,9 +124,16 @@ async function main(): Promise { // sqlStr() would also escape backslashes and strip control chars, corrupting the JSON. const jsonForSql = line.replace(/'/g, "''"); + // Skip the daemon round-trip entirely when embeddings are globally disabled — + // the column stays NULL, schema-compatible with future re-enabling. + const embedding = embeddingsDisabled() + ? null + : await new EmbedClient({ daemonEntry: resolveEmbedDaemonPath() }).embed(line, "document"); + const embeddingSql = embeddingSqlLiteral(embedding); + const insertSql = - `INSERT INTO "${sessionsTable}" (id, path, filename, message, author, size_bytes, project, description, agent, creation_date, last_update_date) ` + - `VALUES ('${crypto.randomUUID()}', '${sqlStr(sessionPath)}', '${sqlStr(filename)}', '${jsonForSql}'::jsonb, '${sqlStr(config.userName)}', ` + + `INSERT INTO "${sessionsTable}" (id, path, filename, message, message_embedding, author, size_bytes, project, description, agent, creation_date, last_update_date) ` + + `VALUES ('${crypto.randomUUID()}', '${sqlStr(sessionPath)}', '${sqlStr(filename)}', '${jsonForSql}'::jsonb, ${embeddingSql}, '${sqlStr(config.userName)}', ` + `${Buffer.byteLength(line, "utf-8")}, '${sqlStr(projectName)}', '${sqlStr(input.hook_event_name ?? "")}', 'claude_code', '${ts}', '${ts}')`; try { diff --git a/src/hooks/codex/wiki-worker.ts b/src/hooks/codex/wiki-worker.ts index 7d74f75..ce016e1 100644 --- a/src/hooks/codex/wiki-worker.ts +++ b/src/hooks/codex/wiki-worker.ts @@ -9,10 +9,13 @@ import { readFileSync, writeFileSync, existsSync, appendFileSync, mkdirSync, rmSync } from "node:fs"; import { execFileSync } from "node:child_process"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; import { finalizeSummary, releaseLock } from "../summary-state.js"; import { uploadSummary } from "../upload-summary.js"; import { log as _log } from "../../utils/debug.js"; +import { EmbedClient } from "../../embeddings/client.js"; +import { embeddingsDisabled } from "../../embeddings/disable.js"; const dlog = (msg: string) => _log("codex-wiki-worker", msg); @@ -176,6 +179,18 @@ async function main(): Promise { if (text.trim()) { const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; + // Embed the summary so it ranks in the semantic retrieval branch. + // Skipped when globally disabled or the daemon is unreachable — + // uploadSummary() writes SQL NULL in that case. + let embedding: number[] | null = null; + if (!embeddingsDisabled()) { + try { + const daemonEntry = join(dirname(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); + embedding = await new EmbedClient({ daemonEntry }).embed(text, "document"); + } catch (e: any) { + wlog(`summary embedding failed, writing NULL: ${e.message}`); + } + } const result = await uploadSummary(query, { tableName: cfg.memoryTable, vpath, fname, @@ -184,6 +199,7 @@ async function main(): Promise { agent: "codex", sessionId: cfg.sessionId, text, + embedding, }); wlog(`uploaded ${vpath} (summary=${result.summaryLength}, desc=${result.descLength})`); diff --git a/src/hooks/grep-direct.ts b/src/hooks/grep-direct.ts index 95e15d9..789a1bc 100644 --- a/src/hooks/grep-direct.ts +++ b/src/hooks/grep-direct.ts @@ -8,6 +8,38 @@ import type { DeeplakeApi } from "../deeplake-api.js"; import { grepBothTables, type GrepMatchParams } from "../shell/grep-core.js"; import { capOutputForClaude } from "../utils/output-cap.js"; +import { EmbedClient } from "../embeddings/client.js"; +import { embeddingsDisabled } from "../embeddings/disable.js"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +const SEMANTIC_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); +const SEMANTIC_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); + +function resolveDaemonPath(): string { + // When bundled as bundle/pre-tool-use.js, the daemon sits at + // bundle/embeddings/embed-daemon.js — one level up from src/hooks/. + return join(dirname(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); +} + +let sharedEmbedClient: EmbedClient | null = null; +function getEmbedClient(): EmbedClient { + if (!sharedEmbedClient) { + sharedEmbedClient = new EmbedClient({ + daemonEntry: resolveDaemonPath(), + timeoutMs: SEMANTIC_TIMEOUT_MS, + }); + } + return sharedEmbedClient; +} + +function patternIsSemanticFriendly(pattern: string, fixedString: boolean): boolean { + if (!pattern || pattern.length < 2) return false; + if (fixedString) return true; + const meta = pattern.match(/[|()\[\]{}+?^$\\]/g); + if (!meta) return true; + return meta.length <= 1; +} export interface GrepParams { pattern: string; @@ -229,7 +261,21 @@ export async function handleGrepDirect( fixedString: params.fixedString, }; - const output = await grepBothTables(api, table, sessionsTable, matchParams, params.targetPath); + // Attempt semantic search. If the daemon is unavailable or the pattern is + // regex-heavy, embed returns null and searchDeeplakeTables falls back to + // lexical LIKE. + let queryEmbedding: number[] | null = null; + if (SEMANTIC_ENABLED && patternIsSemanticFriendly(params.pattern, params.fixedString)) { + try { + queryEmbedding = await getEmbedClient().embed(params.pattern, "query"); + } catch { + queryEmbedding = null; + } + } + + const output = await grepBothTables( + api, table, sessionsTable, matchParams, params.targetPath, queryEmbedding, + ); const joined = output.join("\n") || "(no matches)"; return capOutputForClaude(joined, { kind: "grep" }); } diff --git a/src/hooks/session-start-setup.ts b/src/hooks/session-start-setup.ts index d55ef93..6c62d9a 100644 --- a/src/hooks/session-start-setup.ts +++ b/src/hooks/session-start-setup.ts @@ -18,6 +18,8 @@ import { log as _log } from "../utils/debug.js"; import { getInstalledVersion, getLatestVersion, isNewer } from "../utils/version-check.js"; import { makeWikiLogger } from "../utils/wiki-log.js"; import { resolveVersionedPluginDir, snapshotPluginDir, restoreOrCleanup } from "../utils/plugin-cache.js"; +import { EmbedClient } from "../embeddings/client.js"; +import { embeddingsDisabled } from "../embeddings/disable.js"; const log = (msg: string) => _log("session-setup", msg); const __bundleDir = dirname(fileURLToPath(import.meta.url)); @@ -103,6 +105,31 @@ async function main(): Promise { } catch (e: any) { log(`version check failed: ${e.message}`); } + + // Warm up the embedding daemon so the nomic-embed-text-v1.5 model is + // cached and loaded before the first Grep call. The daemon eagerly + // calls `embedder.load()` on startup (fire-and-forget), which downloads + // the model to ~/.cache/huggingface/hub/ on first run (~130 MB q8 / + // ~500 MB fp32) and keeps it resident for the lifetime of the process. + // `warmup()` itself just ensures the socket is accepting connections; + // the actual model download runs in the daemon's background — so this + // hook stays quick even on a cold install. Opt-out via + // HIVEMIND_EMBED_WARMUP=false for sessions that will never touch the + // memory path (lightweight CC runs, no-network CI). + if (embeddingsDisabled()) { + log("embed daemon warmup skipped via HIVEMIND_EMBEDDINGS=false"); + } else if (process.env.HIVEMIND_EMBED_WARMUP !== "false") { + try { + const daemonEntry = join(__bundleDir, "embeddings", "embed-daemon.js"); + const client = new EmbedClient({ daemonEntry, timeoutMs: 300, spawnWaitMs: 5000 }); + const ok = await client.warmup(); + log(`embed daemon warmup: ${ok ? "ok" : "failed"}`); + } catch (e: any) { + log(`embed daemon warmup threw: ${e.message}`); + } + } else { + log("embed daemon warmup skipped via HIVEMIND_EMBED_WARMUP=false"); + } } main().catch((e) => { log(`fatal: ${e.message}`); process.exit(0); }); diff --git a/src/hooks/session-start.ts b/src/hooks/session-start.ts index 9bcc4e8..727b991 100644 --- a/src/hooks/session-start.ts +++ b/src/hooks/session-start.ts @@ -49,7 +49,7 @@ Organization management — each argument is SEPARATE (do NOT quote subcommands - node "HIVEMIND_AUTH_CMD" members — list members - node "HIVEMIND_AUTH_CMD" remove — remove member -IMPORTANT: Only use bash commands (cat, ls, grep, echo, jq, head, tail, etc.) to interact with ~/.deeplake/memory/. Do NOT use python, python3, node, curl, or other interpreters — they are not available in the memory filesystem. If a task seems to require Python, rewrite it using bash commands and standard text-processing tools (awk, sed, jq, grep, etc.). +IMPORTANT: Only use bash commands (cat, ls, grep, echo, jq, head, tail, etc.) to interact with ~/.deeplake/memory/. Do NOT use python, python3, node, curl, or other interpreters — they are not available in the memory filesystem. Avoid bash brace expansions like \`{1..10}\` (not fully supported); spell out paths explicitly. Bash output is capped at 10MB total — avoid \`for f in *.json; do cat $f\` style loops on the whole sessions dir. LIMITS: Do NOT spawn subagents to read deeplake memory. If a file returns empty after 2 attempts, skip it and move on. Report what you found rather than exhaustively retrying. diff --git a/src/hooks/upload-summary.ts b/src/hooks/upload-summary.ts index f6c96a0..7d9e0cf 100644 --- a/src/hooks/upload-summary.ts +++ b/src/hooks/upload-summary.ts @@ -9,6 +9,7 @@ */ import { randomUUID } from "node:crypto"; +import { embeddingSqlLiteral } from "../embeddings/sql.js"; export type QueryFn = (sql: string) => Promise>>; @@ -22,6 +23,13 @@ export interface UploadParams { sessionId: string; text: string; ts?: string; + /** + * Pre-computed nomic embedding of `text` to store alongside the summary. + * Passing `null` or `undefined` writes SQL NULL — the column stays + * schema-compatible and the row is still reachable via the lexical + * retrieval branch, it just won't show up in the semantic branch. + */ + embedding?: number[] | null; } export interface UploadResult { @@ -56,6 +64,7 @@ export async function uploadSummary(query: QueryFn, params: UploadParams): Promi const ts = params.ts ?? new Date().toISOString(); const desc = extractDescription(text); const sizeBytes = Buffer.byteLength(text); + const embSql = embeddingSqlLiteral(params.embedding ?? null); const existing = await query( `SELECT path FROM "${tableName}" WHERE path = '${esc(vpath)}' LIMIT 1` @@ -65,6 +74,7 @@ export async function uploadSummary(query: QueryFn, params: UploadParams): Promi const sql = `UPDATE "${tableName}" SET ` + `summary = E'${esc(text)}', ` + + `summary_embedding = ${embSql}, ` + `size_bytes = ${sizeBytes}, ` + `description = E'${esc(desc)}', ` + `last_update_date = '${ts}' ` + @@ -74,8 +84,8 @@ export async function uploadSummary(query: QueryFn, params: UploadParams): Promi } const sql = - `INSERT INTO "${tableName}" (id, path, filename, summary, author, mime_type, size_bytes, project, description, agent, creation_date, last_update_date) ` + - `VALUES ('${randomUUID()}', '${esc(vpath)}', '${esc(fname)}', E'${esc(text)}', '${esc(userName)}', 'text/markdown', ` + + `INSERT INTO "${tableName}" (id, path, filename, summary, summary_embedding, author, mime_type, size_bytes, project, description, agent, creation_date, last_update_date) ` + + `VALUES ('${randomUUID()}', '${esc(vpath)}', '${esc(fname)}', E'${esc(text)}', ${embSql}, '${esc(userName)}', 'text/markdown', ` + `${sizeBytes}, '${esc(project)}', E'${esc(desc)}', '${esc(agent)}', '${ts}', '${ts}')`; await query(sql); return { path: "insert", sql, descLength: desc.length, summaryLength: text.length }; diff --git a/src/hooks/wiki-worker.ts b/src/hooks/wiki-worker.ts index 2359ea0..990d054 100644 --- a/src/hooks/wiki-worker.ts +++ b/src/hooks/wiki-worker.ts @@ -9,12 +9,15 @@ import { readFileSync, writeFileSync, existsSync, appendFileSync, mkdirSync, rmSync } from "node:fs"; import { execFileSync } from "node:child_process"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; import { utcTimestamp, log as _log } from "../utils/debug.js"; const dlog = (msg: string) => _log("wiki-worker", msg); import { finalizeSummary, releaseLock } from "./summary-state.js"; import { uploadSummary } from "./upload-summary.js"; +import { EmbedClient } from "../embeddings/client.js"; +import { embeddingsDisabled } from "../embeddings/disable.js"; interface WorkerConfig { apiUrl: string; @@ -181,6 +184,18 @@ async function main(): Promise { if (text.trim()) { const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; + // Embed the summary so it ranks in the semantic retrieval branch. + // Skipped when globally disabled or the daemon is unreachable — + // uploadSummary() writes SQL NULL in that case. + let embedding: number[] | null = null; + if (!embeddingsDisabled()) { + try { + const daemonEntry = join(dirname(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); + embedding = await new EmbedClient({ daemonEntry }).embed(text, "document"); + } catch (e: any) { + wlog(`summary embedding failed, writing NULL: ${e.message}`); + } + } const result = await uploadSummary(query, { tableName: cfg.memoryTable, vpath, fname, @@ -189,6 +204,7 @@ async function main(): Promise { agent: "claude_code", sessionId: cfg.sessionId, text, + embedding, }); wlog(`uploaded ${vpath} (summary=${result.summaryLength}, desc=${result.descLength})`); diff --git a/src/shell/deeplake-fs.ts b/src/shell/deeplake-fs.ts index 8db0716..24c3ba7 100644 --- a/src/shell/deeplake-fs.ts +++ b/src/shell/deeplake-fs.ts @@ -1,11 +1,16 @@ import { basename, posix } from "node:path"; import { randomUUID } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; import type { DeeplakeApi } from "../deeplake-api.js"; import type { IFileSystem, FsStat, MkdirOptions, RmOptions, CpOptions, FileContent, BufferEncoding, } from "just-bash"; import { normalizeContent } from "./grep-core.js"; +import { EmbedClient } from "../embeddings/client.js"; +import { embeddingSqlLiteral } from "../embeddings/sql.js"; +import { embeddingsDisabled } from "../embeddings/disable.js"; interface ReadFileOptions { encoding?: BufferEncoding } interface WriteFileOptions { encoding?: BufferEncoding } @@ -45,6 +50,10 @@ function normalizeSessionMessage(path: string, message: unknown): string { return normalizeContent(path, raw); } +function resolveEmbedDaemonPath(): string { + return join(dirname(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); +} + function joinSessionMessages(path: string, messages: unknown[]): string { return messages.map((message) => normalizeSessionMessage(path, message)).join("\n"); } @@ -85,6 +94,9 @@ export class DeeplakeFs implements IFileSystem { private sessionPaths = new Set(); private sessionsTable: string | null = null; + // Embedding client lazily created on first flush. Lives as long as the process. + private embedClient: EmbedClient | null = null; + private constructor( private readonly client: DeeplakeApi, private readonly table: string, @@ -130,7 +142,12 @@ export class DeeplakeFs implements IFileSystem { const sessionsBootstrap = (sessionsTable && sessionSyncOk) ? (async () => { try { const sessionRows = await client.query( - `SELECT path, SUM(size_bytes) as total_size FROM "${sessionsTable}" GROUP BY path ORDER BY path` + // NOTE: SUM(size_bytes) returns NULL on the Deeplake backend when combined + // with GROUP BY path (confirmed against workspace `with_embedding`). MAX + // works and — for the single-row-per-file layout — is equal to SUM. For + // multi-row-per-turn layouts MAX under-reports total size but stays >0 + // so files don't look like empty placeholders in ls/stat. + `SELECT path, MAX(size_bytes) as total_size FROM "${sessionsTable}" GROUP BY path ORDER BY path` ); for (const row of sessionRows) { const p = row["path"] as string; @@ -194,9 +211,11 @@ export class DeeplakeFs implements IFileSystem { const rows = [...this.pending.values()]; this.pending.clear(); + const embeddings = await this.computeEmbeddings(rows); + // Upsert in parallel — the semaphore in DeeplakeApi.query() handles concurrency. // Re-queue any rows that failed so they are retried on the next flush. - const results = await Promise.allSettled(rows.map(r => this.upsertRow(r))); + const results = await Promise.allSettled(rows.map((r, i) => this.upsertRow(r, embeddings[i]))); let failures = 0; for (let i = 0; i < results.length; i++) { if (results[i].status === "rejected") { @@ -212,7 +231,22 @@ export class DeeplakeFs implements IFileSystem { } } - private async upsertRow(r: PendingRow): Promise { + private async computeEmbeddings(rows: PendingRow[]): Promise<(number[] | null)[]> { + if (rows.length === 0) return []; + // Skip the daemon hop entirely when embeddings are globally disabled. + // upsertRow writes NULL for embedding columns when the value is null, + // so the INSERT / UPDATE shape stays identical. + if (embeddingsDisabled()) return rows.map(() => null); + if (!this.embedClient) { + this.embedClient = new EmbedClient({ daemonEntry: resolveEmbedDaemonPath() }); + } + // One request per row over the same daemon — daemon batches internally if + // ONNX is configured to do so. We fire in parallel; the Unix socket + daemon + // queue handles ordering. null entries are silently stored as empty. + return Promise.all(rows.map(r => this.embedClient!.embed(r.contentText, "document"))); + } + + private async upsertRow(r: PendingRow, embedding: number[] | null): Promise { const text = esc(r.contentText); const p = esc(r.path); const fname = esc(r.filename); @@ -220,8 +254,9 @@ export class DeeplakeFs implements IFileSystem { const ts = new Date().toISOString(); const cd = r.creationDate ?? ts; const lud = r.lastUpdateDate ?? ts; + const embSql = embeddingSqlLiteral(embedding); if (this.flushed.has(r.path)) { - let setClauses = `filename = '${fname}', summary = E'${text}', ` + + let setClauses = `filename = '${fname}', summary = E'${text}', summary_embedding = ${embSql}, ` + `mime_type = '${mime}', size_bytes = ${r.sizeBytes}, last_update_date = '${esc(lud)}'`; if (r.project !== undefined) setClauses += `, project = '${esc(r.project)}'`; if (r.description !== undefined) setClauses += `, description = '${esc(r.description)}'`; @@ -230,10 +265,10 @@ export class DeeplakeFs implements IFileSystem { ); } else { const id = randomUUID(); - const cols = "id, path, filename, summary, mime_type, size_bytes, creation_date, last_update_date" + + const cols = "id, path, filename, summary, summary_embedding, mime_type, size_bytes, creation_date, last_update_date" + (r.project !== undefined ? ", project" : "") + (r.description !== undefined ? ", description" : ""); - const vals = `'${id}', '${p}', '${fname}', E'${text}', '${mime}', ${r.sizeBytes}, '${esc(cd)}', '${esc(lud)}'` + + const vals = `'${id}', '${p}', '${fname}', E'${text}', ${embSql}, '${mime}', ${r.sizeBytes}, '${esc(cd)}', '${esc(lud)}'` + (r.project !== undefined ? `, '${esc(r.project)}'` : "") + (r.description !== undefined ? `, '${esc(r.description)}'` : ""); await this.client.query( @@ -246,55 +281,85 @@ export class DeeplakeFs implements IFileSystem { // ── Virtual index.md generation ──────────────────────────────────────────── private async generateVirtualIndex(): Promise { - const rows = await this.client.query( + // Memory (summaries) section — high-level wikipage per session. + const summaryRows = await this.client.query( `SELECT path, project, description, creation_date, last_update_date FROM "${this.table}" ` + `WHERE path LIKE '${esc("/summaries/")}%' ORDER BY last_update_date DESC` ); - // Build a lookup: key → session path from sessionPaths - // Supports two formats: - // 1. /sessions//___.jsonl → key = sessionId - // 2. /sessions//.json or .jsonl → key = filename stem - const sessionPathsByKey = new Map(); - for (const sp of this.sessionPaths) { - const hivemind = sp.match(/\/sessions\/[^/]+\/[^/]+_([^.]+)\.jsonl$/); - if (hivemind) { - sessionPathsByKey.set(hivemind[1], sp.slice(1)); - } else { - // Generic: extract filename without extension - const fname = sp.split("/").pop() ?? ""; - const stem = fname.replace(/\.[^.]+$/, ""); - if (stem) sessionPathsByKey.set(stem, sp.slice(1)); + // Sessions section — raw session records (dialogue / events). Pulled + // directly from the sessions table so the index is never empty just + // because memory has no summaries yet. + let sessionRows: Record[] = []; + if (this.sessionsTable) { + try { + sessionRows = await this.client.query( + `SELECT path, MAX(description) AS description, MIN(creation_date) AS creation_date, MAX(last_update_date) AS last_update_date ` + + `FROM "${this.sessionsTable}" WHERE path LIKE '${esc("/sessions/")}%' ` + + `GROUP BY path ORDER BY path` + ); + } catch { + // sessions table absent or schema mismatch — leave empty, emit memory-only index. + sessionRows = []; } } const lines: string[] = [ "# Session Index", "", - "List of all Claude Code sessions with summaries.", + "Two sources are available. Consult the section relevant to the question.", "", - "| Session | Conversation | Created | Last Updated | Project | Description |", - "|---------|-------------|---------|--------------|---------|-------------|", ]; - for (const row of rows) { - const p = row["path"] as string; - // Extract session ID from path: /summaries//.md - const match = p.match(/\/summaries\/([^/]+)\/([^/]+)\.md$/); - if (!match) continue; - const summaryUser = match[1]; - const sessionId = match[2]; - const relPath = `summaries/${summaryUser}/${sessionId}.md`; - // Try matching session: first exact sessionId, then strip _summary suffix - const baseName = sessionId.replace(/_summary$/, ""); - const convPath = sessionPathsByKey.get(sessionId) ?? sessionPathsByKey.get(baseName); - const convLink = convPath ? `[messages](${convPath})` : ""; - const project = (row["project"] as string) || ""; - const description = (row["description"] as string) || ""; - const creationDate = (row["creation_date"] as string) || ""; - const lastUpdateDate = (row["last_update_date"] as string) || ""; - lines.push(`| [${sessionId}](${relPath}) | ${convLink} | ${creationDate} | ${lastUpdateDate} | ${project} | ${description} |`); + + // ── ## memory ──────────────────────────────────────────────────────────── + lines.push("## memory"); + lines.push(""); + if (summaryRows.length === 0) { + lines.push("_(empty — no summaries ingested yet)_"); + } else { + lines.push("AI-generated summaries per session. Read these first for topic-level overviews."); + lines.push(""); + lines.push("| Session | Created | Last Updated | Project | Description |"); + lines.push("|---------|---------|--------------|---------|-------------|"); + for (const row of summaryRows) { + const p = row["path"] as string; + const match = p.match(/\/summaries\/([^/]+)\/([^/]+)\.md$/); + if (!match) continue; + const summaryUser = match[1]; + const sessionId = match[2]; + const relPath = `summaries/${summaryUser}/${sessionId}.md`; + const project = (row["project"] as string) || ""; + const description = (row["description"] as string) || ""; + const creationDate = (row["creation_date"] as string) || ""; + const lastUpdateDate = (row["last_update_date"] as string) || ""; + lines.push(`| [${sessionId}](${relPath}) | ${creationDate} | ${lastUpdateDate} | ${project} | ${description} |`); + } + } + lines.push(""); + + // ── ## sessions ───────────────────────────────────────────────────────── + lines.push("## sessions"); + lines.push(""); + if (sessionRows.length === 0) { + lines.push("_(empty — no session records ingested yet)_"); + } else { + lines.push("Raw session records (dialogue, tool calls). Read for exact detail / quotes."); + lines.push(""); + lines.push("| Session | Created | Last Updated | Description |"); + lines.push("|---------|---------|--------------|-------------|"); + for (const row of sessionRows) { + const p = (row["path"] as string) || ""; + // Show the path relative to /sessions/ so the table stays compact. + const rel = p.startsWith("/") ? p.slice(1) : p; + const filename = p.split("/").pop() ?? p; + const description = (row["description"] as string) || ""; + const creationDate = (row["creation_date"] as string) || ""; + const lastUpdateDate = (row["last_update_date"] as string) || ""; + lines.push(`| [${filename}](${rel}) | ${creationDate} | ${lastUpdateDate} | ${description} |`); + } } lines.push(""); + return lines.join("\n"); } diff --git a/src/shell/grep-core.ts b/src/shell/grep-core.ts index 9972594..ae65912 100644 --- a/src/shell/grep-core.ts +++ b/src/shell/grep-core.ts @@ -52,6 +52,23 @@ export interface SearchOptions { multiWordPatterns?: string[]; /** Per-table row cap. */ limit?: number; + /** + * If set, switches to semantic (cosine) search via Deeplake's `<#>` operator + * against `summary_embedding` / `message_embedding` FLOAT4[] columns. When + * absent, the BM25/LIKE path runs. Callers compute this vector via the + * EmbedClient; null means the daemon was unreachable and we should stick + * with lexical search. + */ + queryEmbedding?: number[] | null; + /** + * Plain-text phrase used as the BM25 search term via Deeplake's `<#>` + * operator on TEXT columns (and on `message::text` for JSONB). Replaces + * the old LIKE/ILIKE substring scan: BM25 ranks by term frequency, handles + * multi-word queries natively, and respects a real token index (no full + * table scan). When the pattern is pure regex (no usable literal) this is + * undefined and the caller falls back to the semantic branch alone. + */ + bm25Term?: string; } // ── Content normalization ─────────────────────────────────────────────────── @@ -176,24 +193,43 @@ export function normalizeContent(path: string, raw: string): string { try { obj = JSON.parse(raw); } catch { return raw; } // ── Turn-array session shape: { turns: [...] } ─────────────────────────── + // + // Emit the session date as a prefix on EVERY turn line rather than a + // standalone header row. The downstream `refineGrepMatches` regex filter + // drops non-matching lines, so a header-only date gets stripped before + // Claude sees any grep hit — temporal questions ("When did X?") then + // answer with relative phrases like "Last Friday" because the absolute + // date was in the discarded header. Inlining the date keeps it attached + // to every line that survives the regex. if (Array.isArray(obj.turns)) { - const header: string[] = []; - if (obj.date_time) header.push(`date: ${obj.date_time}`); - if (obj.speakers) { - const s = obj.speakers; - const names = [s.speaker_a, s.speaker_b].filter(Boolean).join(", "); - if (names) header.push(`speakers: ${names}`); - } + const dateHeader = obj.date_time ? `(${String(obj.date_time)}) ` : ""; const lines = obj.turns.map((t: any) => { const sp = String(t?.speaker ?? t?.name ?? "?").trim(); const tx = String(t?.text ?? t?.content ?? "").replace(/\s+/g, " ").trim(); const tag = t?.dia_id ? `[${t.dia_id}] ` : ""; - return `${tag}${sp}: ${tx}`; + return `${dateHeader}${tag}${sp}: ${tx}`; }); - const out = [...header, ...lines].join("\n"); + const out = lines.join("\n"); return out.trim() ? out : raw; } + // ── Single-turn shape: { turn: { dia_id, speaker, text }, ... } ────────── + // Per-row per-turn ingestion (see workspace `with_embedding_multi_rows`) + // stores each row as one turn with enclosing session metadata. Emit the + // session date inline on every turn line so Claude can resolve relative + // times ("last Friday", "last month") against a real reference point — + // without the prefix, temporal-category questions degrade sharply + // because the turn text on its own lacks absolute dating. + if (obj.turn && typeof obj.turn === "object" && !Array.isArray(obj.turn)) { + const t = obj.turn as { dia_id?: unknown; speaker?: unknown; name?: unknown; text?: unknown; content?: unknown }; + const sp = String(t.speaker ?? t.name ?? "?").trim(); + const tx = String(t.text ?? t.content ?? "").replace(/\s+/g, " ").trim(); + const tag = t.dia_id ? `[${String(t.dia_id)}] ` : ""; + const dateHeader = obj.date_time ? `(${String(obj.date_time)}) ` : ""; + const line = `${dateHeader}${tag}${sp}: ${tx}`; + return line.trim() ? line : raw; + } + // ── Production shape: single hook-event row (capture.ts output) ───────── // // `` blocks are injected by OpenClaw as extra context @@ -246,9 +282,15 @@ function buildPathCondition(targetPath: string): string { } /** - * Dual-table LIKE/ILIKE search. Casts `summary` (TEXT) and `message` (JSONB) - * to ::text so the same predicate works across both. The lookup always goes - * through a single UNION ALL query so one grep maps to one SQL search. + * Dual-table search. Two branches: + * • semantic — when `opts.queryEmbedding` is a non-empty vector, cosine + * similarity (`<#>`) against the FLOAT4[] embedding columns. Rows are + * ordered by score DESC and the top-N from both tables are merged. + * • lexical — otherwise, LIKE/ILIKE against ::text of `summary` and + * `message`. Same UNION ALL shape as before for backwards compat. + * + * The lookup always goes through a single top-level SQL query so one grep + * maps to one round-trip. */ export async function searchDeeplakeTables( api: DeeplakeApi, @@ -256,8 +298,92 @@ export async function searchDeeplakeTables( sessionsTable: string, opts: SearchOptions, ): Promise { - const { pathFilter, contentScanOnly, likeOp, escapedPattern, prefilterPattern, prefilterPatterns, multiWordPatterns } = opts; + const { pathFilter, contentScanOnly, likeOp, escapedPattern, prefilterPattern, prefilterPatterns, queryEmbedding, bm25Term, multiWordPatterns } = opts; const limit = opts.limit ?? 100; + + // ── Hybrid (lexical + semantic) branch ─────────────────────────────────── + // Runs both halves in a single UNION ALL query so each grep = one round- + // trip. Lexical catches literal-keyword matches that semantic misses + // (single-word queries diluted by document-level embedding — see + // PR-NOTES.md P2/P3). Semantic catches conceptual matches that lexical + // can't express. De-duplicate by path in the outer layer; when a path + // appears in both halves, the semantic score wins (real cosine signal vs + // the lexical branch's constant 1.0 sentinel). + if (queryEmbedding && queryEmbedding.length > 0) { + const vecLit = serializeFloat4Array(queryEmbedding); + const semanticLimit = Math.min( + limit, + Number(process.env.HIVEMIND_SEMANTIC_LIMIT ?? "20"), + ); + const lexicalLimit = Math.min( + limit, + Number(process.env.HIVEMIND_HYBRID_LEXICAL_LIMIT ?? "20"), + ); + + // Single UNION ALL of lexical (LIKE/ILIKE substring) + semantic (cosine). + // Lexical rows emit a score=1.0 sentinel, semantic rows emit their real + // cosine (0..1). ORDER BY score DESC then LIMIT top-K: + // • exact-substring matches (lexical) dominate the top of the list + // regardless of cosine score — desirable because they're likely to + // contain the literal keyword Claude asked for + // • semantic hits fill in below, covering concept matches where the + // literal keyword doesn't appear + // BM25 tried and dropped (PR-NOTES F4c): score scale (~1..3) overpowered + // cosine in UNION, semantic hits were pushed out of top-K. LIKE is a + // better fit for "find any session mentioning X" which is the actual + // plugin use case. + const filterPatternsForLex = contentScanOnly + ? (prefilterPatterns && prefilterPatterns.length > 0 + ? prefilterPatterns + : (prefilterPattern ? [prefilterPattern] : [])) + : [escapedPattern]; + const memLexFilter = buildContentFilter("summary::text", likeOp, filterPatternsForLex); + const sessLexFilter = buildContentFilter("message::text", likeOp, filterPatternsForLex); + + const memLexQuery = memLexFilter + ? `SELECT path, summary::text AS content, 0 AS source_order, '' AS creation_date, 1.0 AS score ` + + `FROM "${memoryTable}" WHERE 1=1${pathFilter}${memLexFilter} LIMIT ${lexicalLimit}` + : null; + const sessLexQuery = sessLexFilter + ? `SELECT path, message::text AS content, 1 AS source_order, COALESCE(creation_date::text, '') AS creation_date, 1.0 AS score ` + + `FROM "${sessionsTable}" WHERE 1=1${pathFilter}${sessLexFilter} LIMIT ${lexicalLimit}` + : null; + + const memSemQuery = + `SELECT path, summary::text AS content, 0 AS source_order, '' AS creation_date, ` + + `(summary_embedding <#> ${vecLit}) AS score ` + + `FROM "${memoryTable}" WHERE summary_embedding IS NOT NULL${pathFilter} ` + + `ORDER BY score DESC LIMIT ${semanticLimit}`; + const sessSemQuery = + `SELECT path, message::text AS content, 1 AS source_order, COALESCE(creation_date::text, '') AS creation_date, ` + + `(message_embedding <#> ${vecLit}) AS score ` + + `FROM "${sessionsTable}" WHERE message_embedding IS NOT NULL${pathFilter} ` + + `ORDER BY score DESC LIMIT ${semanticLimit}`; + + const parts = [memSemQuery, sessSemQuery]; + if (memLexQuery) parts.push(memLexQuery); + if (sessLexQuery) parts.push(sessLexQuery); + const unionSql = parts.map(q => `(${q})`).join(" UNION ALL "); + + const outerLimit = semanticLimit + lexicalLimit; + const rows = await api.query( + `SELECT path, content, source_order, creation_date, score FROM (` + + unionSql + + `) AS combined ORDER BY score DESC LIMIT ${outerLimit}` + ); + + const seen = new Set(); + const unique: ContentRow[] = []; + for (const row of rows) { + const p = String(row["path"]); + if (seen.has(p)) continue; + seen.add(p); + unique.push({ path: p, content: String(row["content"] ?? "") }); + } + return unique; + } + + // ── Lexical branch ─────────────────────────────────────────────────────── const filterPatterns = contentScanOnly ? (prefilterPatterns && prefilterPatterns.length > 0 ? prefilterPatterns : (prefilterPattern ? [prefilterPattern] : [])) : (multiWordPatterns && multiWordPatterns.length > 1 ? multiWordPatterns : [escapedPattern]); @@ -279,6 +405,15 @@ export async function searchDeeplakeTables( })); } +function serializeFloat4Array(vec: number[]): string { + const parts: string[] = []; + for (const v of vec) { + if (!Number.isFinite(v)) return "NULL"; + parts.push(String(v)); + } + return `ARRAY[${parts.join(",")}]::float4[]`; +} + /** Build a LIKE pathFilter clause for a `path` column. Returns "" if targetPath is root or empty. */ export function buildPathFilter(targetPath: string): string { const condition = buildPathCondition(targetPath); @@ -379,6 +514,20 @@ export function buildGrepSearchOptions(params: GrepMatchParams, targetPath: stri const hasRegexMeta = !params.fixedString && /[.*+?^${}()|[\]\\]/.test(params.pattern); const literalPrefilter = hasRegexMeta ? extractRegexLiteralPrefilter(params.pattern) : null; const alternationPrefilters = hasRegexMeta ? extractRegexAlternationPrefilters(params.pattern) : null; + // bm25Term: the raw phrase we hand to Deeplake's `<#>` operator on TEXT / + // JSONB. Non-regex patterns go through verbatim; regex patterns collapse + // to their extracted literal prefilter (longest literal for a single + // prefilter, space-joined alternations otherwise). If nothing literal can + // be extracted, bm25Term is undefined and callers fall back to semantic. + let bm25Term: string | undefined; + if (!hasRegexMeta) { + bm25Term = params.pattern; + } else if (alternationPrefilters && alternationPrefilters.length > 0) { + bm25Term = alternationPrefilters.join(" "); + } else if (literalPrefilter) { + bm25Term = literalPrefilter; + } + // For non-regex multi-word patterns, split into per-word OR filters so // natural-language queries match any token, not only the full phrase. const multiWordPatterns = (!hasRegexMeta) @@ -388,10 +537,13 @@ export function buildGrepSearchOptions(params: GrepMatchParams, targetPath: stri return { pathFilter: buildPathFilter(targetPath), contentScanOnly: hasRegexMeta, - likeOp: params.ignoreCase ? "ILIKE" : "LIKE", + // Kept for the pure-lexical fallback path and existing tests; BM25 is now + // the preferred lexical ranker when a term can be extracted. + likeOp: process.env.HIVEMIND_GREP_LIKE === "case-sensitive" ? "LIKE" : "ILIKE", escapedPattern: sqlLike(params.pattern), prefilterPattern: literalPrefilter ? sqlLike(literalPrefilter) : undefined, prefilterPatterns: alternationPrefilters?.map((literal) => sqlLike(literal)), + bm25Term, multiWordPatterns: multiWordPatterns.length > 1 ? multiWordPatterns.map((w) => sqlLike(w)) : undefined, @@ -473,8 +625,12 @@ export async function grepBothTables( sessionsTable: string, params: GrepMatchParams, targetPath: string, + queryEmbedding?: number[] | null, ): Promise { - const rows = await searchDeeplakeTables(api, memoryTable, sessionsTable, buildGrepSearchOptions(params, targetPath)); + const rows = await searchDeeplakeTables(api, memoryTable, sessionsTable, { + ...buildGrepSearchOptions(params, targetPath), + queryEmbedding, + }); // Defensive path dedup — memory and sessions tables use disjoint path // prefixes in every schema we ship (/summaries/… vs /sessions/…), so the // overlap is theoretical, but we dedupe to match grep-interceptor.ts and @@ -483,5 +639,25 @@ export async function grepBothTables( const seen = new Set(); const unique = rows.filter(r => seen.has(r.path) ? false : (seen.add(r.path), true)); const normalized = unique.map(r => ({ path: r.path, content: normalizeContent(r.path, r.content) })); + + // Semantic mode: the ranking IS the retrieval. Emitting only regex-matched + // lines would drop relevant turns whose literal text doesn't contain the + // pattern (the whole point of semantic). Return every non-empty normalized + // line from the top-K rows, prefixed with the path so Claude can follow up + // with Read. The downstream output-cap keeps the response bounded. + if (queryEmbedding && queryEmbedding.length > 0) { + const emitAllLines = process.env.HIVEMIND_SEMANTIC_EMIT_ALL !== "false"; + if (emitAllLines) { + const lines: string[] = []; + for (const r of normalized) { + for (const line of r.content.split("\n")) { + const trimmed = line.trim(); + if (trimmed) lines.push(`${r.path}:${line}`); + } + } + return lines; + } + } + return refineGrepMatches(normalized, params); } diff --git a/src/shell/grep-interceptor.ts b/src/shell/grep-interceptor.ts index debd0cd..46f9613 100644 --- a/src/shell/grep-interceptor.ts +++ b/src/shell/grep-interceptor.ts @@ -2,6 +2,10 @@ import type { DeeplakeApi } from "../deeplake-api.js"; import { defineCommand } from "just-bash"; import yargsParser from "yargs-parser"; import type { DeeplakeFs } from "./deeplake-fs.js"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { EmbedClient } from "../embeddings/client.js"; +import { embeddingsDisabled } from "../embeddings/disable.js"; import { buildGrepSearchOptions, @@ -13,6 +17,38 @@ import { type ContentRow, } from "./grep-core.js"; +const SEMANTIC_SEARCH_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); +const SEMANTIC_EMBED_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); + +function resolveGrepEmbedDaemonPath(): string { + return join(dirname(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); +} + +let sharedGrepEmbedClient: EmbedClient | null = null; +function getGrepEmbedClient(): EmbedClient { + if (!sharedGrepEmbedClient) { + sharedGrepEmbedClient = new EmbedClient({ + daemonEntry: resolveGrepEmbedDaemonPath(), + timeoutMs: SEMANTIC_EMBED_TIMEOUT_MS, + }); + } + return sharedGrepEmbedClient; +} + +/** + * Plain-text-ish pattern → candidate for semantic search. + * Skip regex-heavy queries (many metachars) where cosine similarity is not + * what the user asked for. + */ +function patternIsSemanticFriendly(pattern: string, fixedString: boolean): boolean { + if (!pattern || pattern.length < 2) return false; + if (fixedString) return true; + // Literal-ish patterns with only occasional `.*` are still fine for semantic. + const metaMatches = pattern.match(/[|()\[\]{}+?^$\\]/g); + if (!metaMatches) return true; + return metaMatches.length <= 1; +} + const MAX_FALLBACK_CANDIDATES = 500; /** @@ -71,12 +107,25 @@ export function createGrepCommand( countOnly: Boolean(parsed.c || parsed["count"]), }; + // Try semantic search first (daemon-backed embedding of the pattern). + // Falls back to lexical LIKE if the daemon is unreachable, disabled by + // env flag, or the pattern is regex-heavy. + let queryEmbedding: number[] | null = null; + if (SEMANTIC_SEARCH_ENABLED && patternIsSemanticFriendly(pattern, matchParams.fixedString)) { + try { + queryEmbedding = await getGrepEmbedClient().embed(pattern, "query"); + } catch { + queryEmbedding = null; + } + } + let rows: ContentRow[] = []; try { const searchOptions = { ...buildGrepSearchOptions(matchParams, targets[0] ?? ctx.cwd), pathFilter: buildPathFilterForTargets(targets), limit: 100, + queryEmbedding, }; const queryRows = await Promise.race([ searchDeeplakeTables(client, table, sessionsTable ?? "sessions", searchOptions), @@ -87,6 +136,26 @@ export function createGrepCommand( rows = []; // fall through to in-memory fallback } + // Semantic returned nothing → retry with lexical LIKE as a second shot + // before giving up to the in-memory fallback. Keeps behavior robust when + // embeddings miss but BM25 would match. + if (rows.length === 0 && queryEmbedding) { + try { + const lexicalOptions = { + ...buildGrepSearchOptions(matchParams, targets[0] ?? ctx.cwd), + pathFilter: buildPathFilterForTargets(targets), + limit: 100, + }; + const lexicalRows = await Promise.race([ + searchDeeplakeTables(client, table, sessionsTable ?? "sessions", lexicalOptions), + new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), 3000)), + ]); + rows.push(...lexicalRows); + } catch { + // fall through to in-memory fallback below + } + } + // Dedup by path (multiple targets may overlap) const seen = new Set(); rows = rows.filter(r => seen.has(r.path) ? false : (seen.add(r.path), true)); @@ -106,9 +175,25 @@ export function createGrepCommand( } } - // Normalize session JSON blobs to per-turn lines before the regex pass. + // Normalize session JSON blobs to per-turn lines. const normalized = rows.map(r => ({ path: r.path, content: normalizeContent(r.path, r.content) })); - const output = refineGrepMatches(normalized, matchParams); + + // In semantic mode, skip the regex refinement: cosine similarity has + // already done the filtering, and dropping lines whose literal text + // doesn't match the pattern would defeat the semantic retrieval. + // Toggle with HIVEMIND_SEMANTIC_EMIT_ALL=false to restore strict regex. + let output: string[]; + if (queryEmbedding && queryEmbedding.length > 0 && process.env.HIVEMIND_SEMANTIC_EMIT_ALL !== "false") { + output = []; + for (const r of normalized) { + for (const line of r.content.split("\n")) { + const trimmed = line.trim(); + if (trimmed) output.push(`${r.path}:${line}`); + } + } + } else { + output = refineGrepMatches(normalized, matchParams); + } return { stdout: output.length > 0 ? output.join("\n") + "\n" : "", diff --git a/vitest.config.ts b/vitest.config.ts index 375fd1f..08e56ba 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -86,6 +86,42 @@ export default defineConfig({ functions: 90, lines: 90, }, + // embedding_generation — nomic daemon + IPC client + SQL helper. + // Lines/statements held at 90; branches + functions are allowed to + // dip on the daemon because a few paths (SIGINT/SIGTERM handlers, + // the non-Linux `typeof process.getuid !== "function"` fallback, + // and the server "error" handler) can't be triggered from unit + // tests without forking a real subprocess. + "src/embeddings/client.ts": { + statements: 90, + branches: 80, + functions: 90, + lines: 90, + }, + "src/embeddings/daemon.ts": { + statements: 90, + branches: 75, + functions: 75, + lines: 90, + }, + "src/embeddings/nomic.ts": { + statements: 90, + branches: 90, + functions: 90, + lines: 90, + }, + "src/embeddings/protocol.ts": { + statements: 90, + branches: 90, + functions: 90, + lines: 90, + }, + "src/embeddings/sql.ts": { + statements: 90, + branches: 90, + functions: 90, + lines: 90, + }, "src/hooks/pre-tool-use.ts": { statements: 90, branches: 90,