From 9675e99c66ae06c56e27fda4598d23cec098d18f Mon Sep 17 00:00:00 2001 From: keeponlight Date: Fri, 11 Sep 2026 23:57:29 +0800 Subject: [PATCH] fix: import Hermes USER.md native memories --- .../server/routes/import-export.ts | 72 +++--- .../unit/server/hermes-native-import.test.ts | 212 ++++++++++++++++++ 2 files changed, 257 insertions(+), 27 deletions(-) create mode 100644 apps/memos-local-plugin/tests/unit/server/hermes-native-import.test.ts diff --git a/apps/memos-local-plugin/server/routes/import-export.ts b/apps/memos-local-plugin/server/routes/import-export.ts index 082aabe23..ff45cd4be 100644 --- a/apps/memos-local-plugin/server/routes/import-export.ts +++ b/apps/memos-local-plugin/server/routes/import-export.ts @@ -8,11 +8,12 @@ * non-colliding rows. * GET /api/v1/import/hermes-native/scan * → count $HERMES_HOME/memories/MEMORY.md - * entries when running as Hermes (on + * and optional USER.md entries when + * running as Hermes (on * Windows the default home is under * %LOCALAPPDATA%\hermes). * POST /api/v1/import/hermes-native/run - * → import a batch from that file. + * → import a batch from those files. * GET /api/v1/import/openclaw-native/scan * → count OpenClaw agent session JSONL * messages when running as OpenClaw. @@ -27,7 +28,7 @@ import { createHash } from "node:crypto"; import { readFile, readdir, stat } from "node:fs/promises"; import { homedir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import type { TraceDTO } from "../../agent-contract/dto.js"; import type { ServerOptions } from "../types.js"; @@ -39,10 +40,17 @@ const NATIVE_IMPORT_DEFAULT_BATCH = 25; const NATIVE_IMPORT_MAX_BATCH = 200; const NATIVE_IMPORT_CACHE_TTL_MS = 5 * 60 * 1000; +interface HermesNativeMemory { + text: string; + file: string; + /** Position within this file, independent of entries in the other file. */ + index: number; + mtimeMs: number; +} + interface HermesNativeSource { - memories: string[]; + memories: HermesNativeMemory[]; bytes: number; - mtimeMs: number; } interface OpenClawNativeSource { @@ -51,7 +59,7 @@ interface OpenClawNativeSource { sessions: number; } -const hermesNativeCache = new Map(); +const hermesNativeCache = new Map(); const openClawNativeCache = new Map(); export function registerImportExportRoutes( @@ -180,7 +188,6 @@ export function registerImportExportRoutes( const traces = buildHermesNativeTraces(batch, { offset, total, - mtimeMs: source.mtimeMs, }); const result = await deps.core.importBundle({ version: 1, @@ -401,23 +408,31 @@ async function readHermesNativeMemories( path: string, opts: { force?: boolean } = {}, ): Promise { - const info = await stat(path); + const files = [{ path, file: "MEMORY.md", info: await stat(path) }]; + const userPath = join(dirname(path), "USER.md"); + try { + files.push({ path: userPath, file: "USER.md", info: await stat(userPath) }); + } catch (err) { + // USER.md is optional; other I/O errors must not hide profile data loss. + if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err; + } + // A combined size and maximum mtime can miss same-size edits to the older file. + const fingerprint = JSON.stringify( + files.map(({ file, info }) => [file, info.size, info.mtimeMs]), + ); const cached = hermesNativeCache.get(path); - if ( - !opts.force && - cached && - cached.source.bytes === info.size && - cached.source.mtimeMs === info.mtimeMs - ) { + if (!opts.force && cached?.fingerprint === fingerprint) { return cached.source; } - const raw = await readFile(path, "utf8"); - const source = { - memories: splitHermesNativeMemories(raw), - bytes: info.size, - mtimeMs: info.mtimeMs, - }; - hermesNativeCache.set(path, { source }); + const source: HermesNativeSource = { memories: [], bytes: 0 }; + for (const { path: filePath, file, info } of files) { + const raw = await readFile(filePath, "utf8"); + for (const [index, text] of splitHermesNativeMemories(raw).entries()) { + source.memories.push({ text, file, index, mtimeMs: info.mtimeMs }); + } + source.bytes += info.size; + } + hermesNativeCache.set(path, { source, fingerprint }); return source; } @@ -443,14 +458,16 @@ function splitHermesNativeMemories(raw: string): string[] { } function buildHermesNativeTraces( - memories: readonly string[], - opts: { offset: number; total: number; mtimeMs: number }, + memories: readonly HermesNativeMemory[], + opts: { offset: number; total: number }, ): TraceDTO[] { - const baseTs = Number.isFinite(opts.mtimeMs) ? Math.floor(opts.mtimeMs) : Date.now(); return memories.map((memory, i) => { + const baseTs = Number.isFinite(memory.mtimeMs) ? Math.floor(memory.mtimeMs) : Date.now(); const index = opts.offset + i; + // Preserve existing MEMORY.md IDs; namespace profile IDs by their source file. + const identity = `${memory.index}\0${memory.text}`; const hash = createHash("sha256") - .update(`${index}\0${memory}`) + .update(memory.file === "MEMORY.md" ? identity : `${memory.file}\0${identity}`) .digest("hex") .slice(0, 24); const ts = Math.max(0, baseTs - Math.max(1, opts.total - index) * 1000); @@ -459,9 +476,10 @@ function buildHermesNativeTraces( episodeId: `ep_hm_${hash}` as never, sessionId: "se_hermes_native_memory" as never, ts: ts as never, - userText: memory, + userText: memory.text, agentText: "", - summary: memory, + summary: memory.text, + tags: [memory.file], toolCalls: [], reflection: undefined, value: 0.5 as never, diff --git a/apps/memos-local-plugin/tests/unit/server/hermes-native-import.test.ts b/apps/memos-local-plugin/tests/unit/server/hermes-native-import.test.ts new file mode 100644 index 000000000..38a6b5406 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/server/hermes-native-import.test.ts @@ -0,0 +1,212 @@ +import { createHash } from "node:crypto"; +import * as fs from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { TraceDTO } from "../../../agent-contract/dto.js"; +import type { MemoryCore } from "../../../agent-contract/memory-core.js"; +import { registerImportExportRoutes } from "../../../server/routes/import-export.js"; +import { Routes, type RouteContext } from "../../../server/routes/registry.js"; + +describe("Hermes native memory import", () => { + let hermesHome: string; + let memoryPath: string; + let userPath: string; + let routes: Routes; + const importBundle = vi.fn(async (bundle: Parameters[0]) => ({ + imported: bundle.traces?.length ?? 0, + skipped: 0, + })); + + beforeEach(() => { + hermesHome = fs.mkdtempSync(join(tmpdir(), "memos-hermes-import-")); + const memoriesDir = join(hermesHome, "memories"); + fs.mkdirSync(memoriesDir); + memoryPath = join(memoriesDir, "MEMORY.md"); + userPath = join(memoriesDir, "USER.md"); + vi.stubEnv("HERMES_HOME", hermesHome); + importBundle.mockClear(); + routes = new Routes(); + registerImportExportRoutes(routes, { core: { importBundle } as unknown as MemoryCore }, { + agent: "hermes", + }); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + fs.rmSync(hermesHome, { recursive: true, force: true }); + }); + + function context(body: unknown = {}): RouteContext { + return { + body: Buffer.from(JSON.stringify(body)), + res: { writeHead: vi.fn(), end: vi.fn() }, + } as unknown as RouteContext; + } + + async function scan() { + return await routes.getExact("GET /api/v1/import/hermes-native/scan")!(context()); + } + + async function run(offset = 0, limit = 25) { + return await routes.getExact("POST /api/v1/import/hermes-native/run")!( + context({ offset, limit }), + ); + } + + function lastTraces(): TraceDTO[] { + return importBundle.mock.calls.at(-1)![0].traces as TraceDTO[]; + } + + it("scans and pages through both files with source tags and each file's timestamp", async () => { + const memoryText = "first memory\n§\nshared fact\n"; + const userText = "preferred language: 中文\r\n§\r\nshared fact\r\n§\r\n"; + fs.writeFileSync(memoryPath, memoryText); + fs.writeFileSync(userPath, userText); + const memoryTime = new Date("2026-01-02T00:00:00Z"); + const userTime = new Date("2026-01-01T00:00:00Z"); + fs.utimesSync(memoryPath, memoryTime, memoryTime); + fs.utimesSync(userPath, userTime, userTime); + + expect(await scan()).toMatchObject({ + found: true, + total: 4, + path: memoryPath, + bytes: Buffer.byteLength(memoryText) + Buffer.byteLength(userText), + }); + expect(await run(0, 3)).toMatchObject({ + total: 4, nextOffset: 3, imported: 3, done: false, + }); + const firstPage = lastTraces(); + expect(firstPage).toMatchObject([ + { userText: "first memory", summary: "first memory", tags: ["MEMORY.md"] }, + { userText: "shared fact", summary: "shared fact", tags: ["MEMORY.md"] }, + { + userText: "preferred language: 中文", + summary: "preferred language: 中文", + tags: ["USER.md"], + }, + ]); + expect(firstPage[0].ts).toBeGreaterThan(memoryTime.getTime() - 10_000); + expect(firstPage[0].ts).toBeLessThanOrEqual(memoryTime.getTime()); + expect(firstPage[2].ts).toBeGreaterThan(userTime.getTime() - 10_000); + expect(firstPage[2].ts).toBeLessThanOrEqual(userTime.getTime()); + + expect(await run(3, 3)).toMatchObject({ + total: 4, nextOffset: 4, imported: 1, done: true, + }); + const lastPage = lastTraces(); + expect(lastPage).toMatchObject([ + { userText: "shared fact", summary: "shared fact", tags: ["USER.md"] }, + ]); + expect(lastPage[0].id).not.toBe(firstPage[1].id); + + expect(await run(4, 3)).toMatchObject({ + total: 4, nextOffset: 4, imported: 0, skipped: 0, done: true, + }); + expect(importBundle).toHaveBeenCalledTimes(2); + }); + + it.each([undefined, "", " \r\n§\r\n\r\n"])( + "keeps MEMORY.md imports working with missing or empty USER.md (%j)", + async (userText) => { + const memoryText = "first memory\n§\nsecond memory"; + fs.writeFileSync(memoryPath, memoryText); + if (userText !== undefined) fs.writeFileSync(userPath, userText); + + expect(await scan()).toMatchObject({ + found: true, + total: 2, + bytes: Buffer.byteLength(memoryText) + Buffer.byteLength(userText ?? ""), + }); + expect(await run()).toMatchObject({ total: 2, imported: 2, done: true }); + expect(lastTraces().map((trace) => trace.userText)).toEqual([ + "first memory", "second memory", + ]); + }, + ); + + it("preserves legacy MEMORY.md IDs and keeps USER.md IDs stable when MEMORY.md grows", async () => { + fs.writeFileSync(memoryPath, "shared fact"); + fs.writeFileSync(userPath, "shared fact"); + expect(await run(0, 1)).toMatchObject({ total: 2, imported: 1, done: false }); + const originalMemory = lastTraces()[0]; + // Existing imports use this identity; changing it would duplicate their rows. + const legacyHash = createHash("sha256").update("0\0shared fact").digest("hex").slice(0, 24); + expect(originalMemory).toMatchObject({ + id: `tr_hm_${legacyHash}`, + episodeId: `ep_hm_${legacyHash}`, + sessionId: "se_hermes_native_memory", + }); + await run(1, 1); + const originalUser = lastTraces()[0]; + expect(originalUser.id).not.toBe(originalMemory.id); + + fs.appendFileSync(memoryPath, "\n§\nanother memory"); + expect(await run()).toMatchObject({ total: 3, imported: 3, done: true }); + expect(lastTraces()[0].id).toBe(originalMemory.id); + expect(lastTraces()[2]).toMatchObject({ + id: originalUser.id, + episodeId: originalUser.episodeId, + userText: "shared fact", + tags: ["USER.md"], + }); + }); + + it.each(["MEMORY.md", "USER.md"])( + "refreshes a cached page after a same-size edit to the older %s", + async (file) => { + fs.writeFileSync(memoryPath, "first memory\n§\nsecond memory"); + fs.writeFileSync(userPath, "first profile"); + const editedPath = file === "MEMORY.md" ? memoryPath : userPath; + const newerPath = file === "MEMORY.md" ? userPath : memoryPath; + const oldTime = new Date("2026-01-01T00:00:00Z"); + const changedTime = new Date("2026-01-02T00:00:00Z"); + const newerTime = new Date("2026-01-03T00:00:00Z"); + fs.utimesSync(editedPath, oldTime, oldTime); + fs.utimesSync(newerPath, newerTime, newerTime); + await run(); + + const previousSize = fs.statSync(editedPath).size; + const changedText = file === "MEMORY.md" + ? "first memory\n§\nlatest memory" + : "other profile"; + fs.writeFileSync(editedPath, changedText); + fs.utimesSync(editedPath, changedTime, changedTime); + expect(fs.statSync(editedPath).size).toBe(previousSize); + + expect(await run(1)).toMatchObject({ total: 3, imported: 2, done: true }); + expect(lastTraces().map((trace) => trace.userText)).toEqual( + file === "MEMORY.md" + ? ["latest memory", "first profile"] + : ["second memory", "other profile"], + ); + }, + ); + + it("refreshes cached pages when USER.md appears or disappears", async () => { + fs.writeFileSync(memoryPath, "first memory\n§\nsecond memory"); + await run(); + + fs.writeFileSync(userPath, "new profile"); + expect(await run(2)).toMatchObject({ total: 3, imported: 1, done: true }); + expect(lastTraces()[0].userText).toBe("new profile"); + + fs.unlinkSync(userPath); + expect(await run(1)).toMatchObject({ total: 2, imported: 1, done: true }); + expect(lastTraces().map((trace) => trace.userText)).toEqual(["second memory"]); + }); + + it("reports USER.md read errors instead of silently dropping profile entries", async () => { + fs.writeFileSync(memoryPath, "first memory"); + fs.mkdirSync(userPath); + + expect(await scan()).toMatchObject({ found: false, total: 0, error: expect.any(String) }); + const ctx = context({ offset: 0 }); + await routes.getExact("POST /api/v1/import/hermes-native/run")!(ctx); + expect(ctx.res.writeHead).toHaveBeenCalledWith(404, expect.any(Object)); + expect(importBundle).not.toHaveBeenCalled(); + }); +});