diff --git a/.changeset/fix-ipv6-web-origin.md b/.changeset/fix-ipv6-web-origin.md new file mode 100644 index 0000000000..d25284e3ce --- /dev/null +++ b/.changeset/fix-ipv6-web-origin.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Fix `kimi web` startup when binding to an IPv6 host. diff --git a/.changeset/journal-fswatch-reliability-fixes.md b/.changeset/journal-fswatch-reliability-fixes.md new file mode 100644 index 0000000000..be1b0a5d30 --- /dev/null +++ b/.changeset/journal-fswatch-reliability-fixes.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Fix `.gitignore` edits not taking effect until restart for file listing and file watching. diff --git a/.changeset/multi-instance-session-sync.md b/.changeset/multi-instance-session-sync.md new file mode 100644 index 0000000000..a546025b9c --- /dev/null +++ b/.changeset/multi-instance-session-sync.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +web: When several server instances share one home, opening a session held by another instance now redirects to that instance, and the session list refreshes automatically when a peer adds or removes sessions. diff --git a/.changeset/session-event-release-barrier.md b/.changeset/session-event-release-barrier.md new file mode 100644 index 0000000000..44e83a9883 --- /dev/null +++ b/.changeset/session-event-release-barrier.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +web: Restore live event delivery after archiving and reopening a session. diff --git a/.changeset/session-lease-write-fencing.md b/.changeset/session-lease-write-fencing.md new file mode 100644 index 0000000000..5d004d9484 --- /dev/null +++ b/.changeset/session-lease-write-fencing.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Opening the same session from a second instance now fails with a clear ownership error, while shutdown blocks late writes and releases ambiguous closes through a dirty fallback. diff --git a/.changeset/skill-hot-reload.md b/.changeset/skill-hot-reload.md new file mode 100644 index 0000000000..854aff101f --- /dev/null +++ b/.changeset/skill-hot-reload.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Newly added or updated skills now take effect on the next turn without restarting, and the web skill list refreshes automatically. diff --git a/.changeset/stale-write-fencing.md b/.changeset/stale-write-fencing.md new file mode 100644 index 0000000000..90aa5b1479 --- /dev/null +++ b/.changeset/stale-write-fencing.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +The agent now refuses to edit a file that changed on disk since it was last read in the session, instead of silently overwriting external changes. diff --git a/apps/kimi-code/.gitignore b/apps/kimi-code/.gitignore index 901b7a6d26..762220ab4c 100644 --- a/apps/kimi-code/.gitignore +++ b/apps/kimi-code/.gitignore @@ -8,4 +8,4 @@ agents/ src/generated/vis-web-asset.ts # Copied from packages/pi-tui/native at build time by scripts/copy-native-assets.mjs -native/ +/native/ diff --git a/apps/kimi-code/package.json b/apps/kimi-code/package.json index 56c2f9cbae..0bbe681d48 100644 --- a/apps/kimi-code/package.json +++ b/apps/kimi-code/package.json @@ -77,6 +77,9 @@ "e2e:real": "pnpm -w run build:packages && KIMI_E2E_REAL=1 vitest run test/e2e/real-llm-smoke.e2e.test.ts", "postinstall": "node scripts/postinstall.mjs" }, + "dependencies": { + "fs-native-extensions": "1.5.0" + }, "optionalDependencies": { "@mariozechner/clipboard": "^0.3.9", "node-pty": "^1.1.0" @@ -85,6 +88,7 @@ "@moonshot-ai/acp-adapter": "workspace:^", "@moonshot-ai/agent-core-v2": "workspace:^", "@moonshot-ai/kap-server": "workspace:^", + "@moonshot-ai/kernel-file-lock": "workspace:^", "@moonshot-ai/kimi-code-oauth": "workspace:^", "@moonshot-ai/kimi-code-sdk": "workspace:^", "@moonshot-ai/kimi-telemetry": "workspace:^", diff --git a/apps/kimi-code/scripts/native/check-bundle.mjs b/apps/kimi-code/scripts/native/check-bundle.mjs index 3cd10c278d..bfef0f4938 100644 --- a/apps/kimi-code/scripts/native/check-bundle.mjs +++ b/apps/kimi-code/scripts/native/check-bundle.mjs @@ -23,7 +23,7 @@ const optionalRuntimeRequires = new Set([ 'utf-8-validate', ]); const optionalRelativeRuntimeRequires = new Set(['./crypto/build/Release/sshcrypto.node']); -const handledNativeRuntimeRequires = new Set(); +const handledNativeRuntimeRequires = new Set(['fs-native-extensions']); function isAllowedSpecifier(specifier) { if (builtins.has(specifier) || specifier.startsWith('node:')) return true; diff --git a/apps/kimi-code/scripts/native/native-deps.mjs b/apps/kimi-code/scripts/native/native-deps.mjs index 8e26d9229d..1d12f3f196 100644 --- a/apps/kimi-code/scripts/native/native-deps.mjs +++ b/apps/kimi-code/scripts/native/native-deps.mjs @@ -39,6 +39,15 @@ const piTuiNativeFileByTarget = Object.freeze({ 'win32-x64': ['native/win32/prebuilds/win32-x64/win32-console-mode.node'], }); +const fsNativeExtensionsFileByTarget = Object.freeze( + Object.fromEntries( + SUPPORTED_TARGETS.map((target) => [ + target, + [`prebuilds/${target}/fs-native-extensions.node`], + ]), + ), +); + export function isSupportedTarget(target) { return SUPPORTED_TARGETS.includes(target); } @@ -84,6 +93,13 @@ export const nativeDeps = Object.freeze([ parent: null, nativeFileRelatives: (target) => piTuiNativeFileByTarget[target] ?? [], }, + { + id: 'fs-native-extensions', + name: () => 'fs-native-extensions', + collect: 'native-file-only', + parent: null, + nativeFileRelatives: (target) => fsNativeExtensionsFileByTarget[target] ?? [], + }, ]); /** diff --git a/apps/kimi-code/src/cli/sub/web/run.ts b/apps/kimi-code/src/cli/sub/web/run.ts index 1a4fcbaa84..36e22394d2 100644 --- a/apps/kimi-code/src/cli/sub/web/run.ts +++ b/apps/kimi-code/src/cli/sub/web/run.ts @@ -41,6 +41,7 @@ import { DEFAULT_SERVER_HOST, DEFAULT_SERVER_PORT, parseServerOptions, + serverOrigin, tryResolveServerToken, VALID_LOG_LEVELS, type ParsedServerOptions, @@ -288,7 +289,7 @@ async function runServerInProcess( }); logger.info('serving the REST/WS API and the bundled web UI'); running = { - address: `http://${v2.host}:${v2.port}`, + address: serverOrigin(v2.host, v2.port), logger, close: () => v2.close(), }; diff --git a/apps/kimi-code/src/cli/sub/web/shared.ts b/apps/kimi-code/src/cli/sub/web/shared.ts index 79dfff7d41..84f1654e90 100644 --- a/apps/kimi-code/src/cli/sub/web/shared.ts +++ b/apps/kimi-code/src/cli/sub/web/shared.ts @@ -7,13 +7,14 @@ import { readFileSync } from 'node:fs'; import { join } from 'node:path'; -import type { ServerLogLevel } from '@moonshot-ai/kap-server'; +import { formatServerOrigin, type ServerLogLevel } from '@moonshot-ai/kap-server'; export const LOCAL_SERVER_HOST = '127.0.0.1'; export const DEFAULT_LAN_HOST = '0.0.0.0'; export const DEFAULT_SERVER_HOST = LOCAL_SERVER_HOST; export const DEFAULT_SERVER_PORT = 58627; -export const DEFAULT_SERVER_ORIGIN = serverOrigin(DEFAULT_SERVER_HOST, DEFAULT_SERVER_PORT); +export const DEFAULT_SERVER_ORIGIN = formatServerOrigin(DEFAULT_SERVER_HOST, DEFAULT_SERVER_PORT); +export { formatServerOrigin as serverOrigin }; /** Filename (under KIMI_CODE_HOME) of the persistent server bearer token. */ export const SERVER_TOKEN_FILE = 'server.token'; @@ -112,10 +113,6 @@ export function parseLogLevel(raw: string | undefined): ServerLogLevel { ); } -export function serverOrigin(host: string, port: number): string { - return `http://${host}:${port}`; -} - /** Strip `/api/v1` and trailing slashes so user-supplied origins are uniform. */ export function normalizeServerOrigin(value: string): string { const url = new URL(value); diff --git a/apps/kimi-code/src/main.ts b/apps/kimi-code/src/main.ts index 28ed63a900..567d889329 100644 --- a/apps/kimi-code/src/main.ts +++ b/apps/kimi-code/src/main.ts @@ -36,6 +36,7 @@ import { runUpdatePreflight } from './cli/update/preflight'; import { createKimiCodeHostIdentity, getVersion } from './cli/version'; import { CLI_SHUTDOWN_TIMEOUT_MS, CLI_UI_MODE, PROCESS_NAME } from './constant/app'; import { cleanupStaleNativeCacheForCurrent } from './native/native-assets'; +import { installKernelFileLockNativeBinding } from './native/kernel-file-lock'; import { installNativeModuleHook } from './native/module-hook'; import { runNativeAssetSmokeIfRequested } from './native/smoke'; @@ -142,6 +143,7 @@ export function main(): void { // invalid proxy URL is reported and ignored rather than aborting startup. installGlobalProxyDispatcher(); installNativeModuleHook(); + installKernelFileLockNativeBinding(); if (runNativeAssetSmokeIfRequested()) return; // Start the background cleanup of stale native cache. Fire-and-forget; must not block startup or throw. diff --git a/apps/kimi-code/src/native/kernel-file-lock.ts b/apps/kimi-code/src/native/kernel-file-lock.ts new file mode 100644 index 0000000000..1717e3afd6 --- /dev/null +++ b/apps/kimi-code/src/native/kernel-file-lock.ts @@ -0,0 +1,44 @@ +import { join } from 'node:path'; + +import { + setKernelFileLockBindingLoader, + type KernelFileLockBinding, +} from '@moonshot-ai/kernel-file-lock'; + +import { loadNativePackageFile } from './native-require'; + +interface NativeFileLockBinding { + tryLock(fd: number, offset: number, length: number, exclusive: boolean): void; + unlock(fd: number, offset: number, length: number): void; +} + +function errorCode(error: unknown): string | undefined { + return (error as NodeJS.ErrnoException | undefined)?.code; +} + +export function loadKernelFileLockNativeBinding(): KernelFileLockBinding | undefined { + const target = `${process.platform}-${process.arch}`; + const binding = loadNativePackageFile( + 'fs-native-extensions', + join('prebuilds', target, 'fs-native-extensions.node'), + ); + if (binding === null) return undefined; + return { + tryLock: (fd) => { + try { + binding.tryLock(fd, 0, 0, true); + return true; + } catch (error) { + if (errorCode(error) === 'EAGAIN') return false; + throw error; + } + }, + unlock: (fd) => { + binding.unlock(fd, 0, 0); + }, + }; +} + +export function installKernelFileLockNativeBinding(): void { + setKernelFileLockBindingLoader(loadKernelFileLockNativeBinding); +} diff --git a/apps/kimi-code/src/native/native-require.ts b/apps/kimi-code/src/native/native-require.ts index 6840d610b5..3287ffba98 100644 --- a/apps/kimi-code/src/native/native-require.ts +++ b/apps/kimi-code/src/native/native-require.ts @@ -28,3 +28,13 @@ export function loadNativePackage( if (nativeRequire === null) return null; return nativeRequire(packageName) as T; } + +export function loadNativePackageFile( + packageName: string, + relativePath: string, + options: NativeAssetOptions = {}, +): T | null { + const packageRoot = getNativePackageRoot(packageName, options); + if (packageRoot === null) return null; + return createRequire(import.meta.url)(join(packageRoot, relativePath)) as T; +} diff --git a/apps/kimi-code/src/native/smoke.ts b/apps/kimi-code/src/native/smoke.ts index c77f1419d0..b6bef56c0e 100644 --- a/apps/kimi-code/src/native/smoke.ts +++ b/apps/kimi-code/src/native/smoke.ts @@ -2,8 +2,9 @@ import { createRequire } from 'node:module'; import { dirname, join } from 'node:path'; import { getEmbeddedNativeAssetManifest, getNativePackageRoot } from './native-assets'; +import { loadKernelFileLockNativeBinding } from './kernel-file-lock'; -const smokePackages = ['@mariozechner/clipboard', '@moonshot-ai/pi-tui']; +const smokePackages = ['@mariozechner/clipboard', '@moonshot-ai/pi-tui', 'fs-native-extensions']; // Verify pi-tui's native helper can actually be loaded through the module hook. // pi-tui computes native helper paths from process.execPath and require()s them; @@ -34,6 +35,13 @@ function smokePiTuiNativeLoad(): void { } } +function smokeKernelFileLockNativeLoad(): void { + const binding = loadKernelFileLockNativeBinding(); + if (binding === undefined) { + throw new Error('fs-native-extensions binding is unavailable.'); + } +} + export function runNativeAssetSmokeIfRequested(): boolean { if (process.env['KIMI_CODE_NATIVE_ASSET_SMOKE'] !== '1') return false; @@ -49,6 +57,7 @@ export function runNativeAssetSmokeIfRequested(): boolean { } } smokePiTuiNativeLoad(); + smokeKernelFileLockNativeLoad(); process.stdout.write(`Native asset smoke passed: ${manifest.target}\n`); process.exit(0); } catch (error) { diff --git a/apps/kimi-code/test/cli/web/web.test.ts b/apps/kimi-code/test/cli/web/web.test.ts index d23497b5d1..6cefa44aa2 100644 --- a/apps/kimi-code/test/cli/web/web.test.ts +++ b/apps/kimi-code/test/cli/web/web.test.ts @@ -18,7 +18,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { registerWebCommand } from '#/cli/sub/web'; import type { LegacyKillDeps } from '#/cli/sub/web/legacy-kill'; import type { WebCommandDeps } from '#/cli/sub/web/run'; -import type { ParsedServerOptions } from '#/cli/sub/web/shared'; +import { serverOrigin, type ParsedServerOptions } from '#/cli/sub/web/shared'; import { darkColors } from '#/tui/theme/colors'; vi.mock('node:child_process', async (importOriginal) => { @@ -952,6 +952,15 @@ describe('formatHostForUrl', () => { }); }); +describe('serverOrigin', () => { + it('bracket-wraps an IPv6 host so the origin remains parseable', () => { + const origin = serverOrigin('::1', 58627); + + expect(origin).toBe('http://[::1]:58627'); + expect(new URL(origin).port).toBe('58627'); + }); +}); + describe('filterDisplayAddresses', () => { it('drops IPv6 link-local, de-duplicates, and orders IPv4 before IPv6', async () => { const { filterDisplayAddresses } = await import('#/cli/sub/web/networks'); diff --git a/apps/kimi-code/test/native/native-assets.test.ts b/apps/kimi-code/test/native/native-assets.test.ts index 1946a6ccc7..9d976336e0 100644 --- a/apps/kimi-code/test/native/native-assets.test.ts +++ b/apps/kimi-code/test/native/native-assets.test.ts @@ -12,7 +12,7 @@ import { type NativeAssetManifest, type NativeAssetSource, } from '#/native/native-assets'; -import { loadNativePackage } from '#/native/native-require'; +import { loadNativePackage, loadNativePackageFile } from '#/native/native-require'; function sha256(bytes: Buffer | string): string { return createHash('sha256').update(bytes).digest('hex'); @@ -125,4 +125,30 @@ describe('native assets', () => { rmSync(dir, { recursive: true, force: true }); } }); + + it('loads a package file from extracted native assets', () => { + const dir = mkdtempSync(join(tmpdir(), 'kimi-native-file-require-')); + try { + const { manifest, source } = fakeManifest({ + 'node_modules/fake-native/package.json': '{}', + 'node_modules/fake-native/prebuilds/test-target/addon.cjs': + "module.exports = { value: 'native' };\n", + }); + + const addon = loadNativePackageFile<{ value: string }>( + 'fake-native', + join('prebuilds', 'test-target', 'addon.cjs'), + { + cacheBase: dir, + manifest, + source, + version: 'test', + }, + ); + + expect(addon).toEqual({ value: 'native' }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/apps/kimi-code/test/scripts/native/native-deps.test.ts b/apps/kimi-code/test/scripts/native/native-deps.test.ts index 980f1c7714..4714fc20e9 100644 --- a/apps/kimi-code/test/scripts/native/native-deps.test.ts +++ b/apps/kimi-code/test/scripts/native/native-deps.test.ts @@ -42,6 +42,7 @@ describe('resolveTargetDeps', () => { expect(names).toContain('@mariozechner/clipboard'); expect(names).toContain('@mariozechner/clipboard-darwin-arm64'); expect(names).toContain('@moonshot-ai/pi-tui'); + expect(names).toContain('fs-native-extensions'); }); it('picks the right clipboard subpackage per target', () => { @@ -75,6 +76,15 @@ describe('resolveTargetDeps', () => { ]); }); + it('encodes fs-native-extensions prebuild path per target', () => { + const fsNative = resolveTargetDeps('linux-x64').find( + (d) => d.resolvedName === 'fs-native-extensions', + ); + expect(fsNative?.nativeFileRelatives).toEqual([ + 'prebuilds/linux-x64/fs-native-extensions.node', + ]); + }); + it('throws on unsupported target', () => { expect(() => resolveTargetDeps('linux-x64-musl')).toThrow(/unsupported/i); }); @@ -97,4 +107,10 @@ describe('nativeDeps registry shape', () => { expect(piTui?.collect).toBe('native-file-only'); expect(piTui?.parent).toBe(null); }); + + it('has fs-native-extensions (collect=native-file-only, no parent)', () => { + const fsNative = nativeDeps.find((d) => d.id === 'fs-native-extensions'); + expect(fsNative?.collect).toBe('native-file-only'); + expect(fsNative?.parent).toBe(null); + }); }); diff --git a/apps/kimi-code/tsdown.native.config.ts b/apps/kimi-code/tsdown.native.config.ts index c1008cb616..c3d5642ed5 100644 --- a/apps/kimi-code/tsdown.native.config.ts +++ b/apps/kimi-code/tsdown.native.config.ts @@ -16,7 +16,7 @@ const builtins = new Set([ ...builtinModules, ...builtinModules.map((name) => `node:${name}`), ]); -const optionalNativeDependencies = new Set(['cpu-features']); +const optionalNativeDependencies = new Set(['cpu-features', 'fs-native-extensions']); function shouldAlwaysBundle(id: string): boolean { if (builtins.has(id) || id.startsWith('node:')) return false; diff --git a/apps/kimi-web/src/api/daemon/client.ts b/apps/kimi-web/src/api/daemon/client.ts index d8b1833edf..9cff10fa04 100644 --- a/apps/kimi-web/src/api/daemon/client.ts +++ b/apps/kimi-web/src/api/daemon/client.ts @@ -1463,8 +1463,16 @@ export class DaemonKimiWebApi implements KimiWebApi { handlers.onConnectionChange(connected); }, - onError: (code: number, msg: string, fatal: boolean) => { - handlers.onError(code, msg, fatal); + onError: (code: number, msg: string, fatal: boolean, details?: unknown) => { + handlers.onError(code, msg, fatal, details); + }, + + onSessionListChanged: () => { + handlers.onSessionListChanged?.(); + }, + + onSkillCatalogChanged: (sessionId: string) => { + handlers.onSkillCatalogChanged?.(sessionId); }, onTerminalOutput: (sessionId, terminalId, data, seq) => { diff --git a/apps/kimi-web/src/api/daemon/serverAuth.ts b/apps/kimi-web/src/api/daemon/serverAuth.ts index 4aa04ed8b8..97431d860c 100644 --- a/apps/kimi-web/src/api/daemon/serverAuth.ts +++ b/apps/kimi-web/src/api/daemon/serverAuth.ts @@ -18,6 +18,7 @@ const STORAGE_KEY = 'kimi-web.server-credential'; const FRAGMENT_PARAM = 'token'; +const REDIRECT_HASH_PARAM = 'redirect_hash'; const CREDENTIAL_TTL_MS = 7 * 24 * 60 * 60 * 1000; interface StoredCredential { @@ -38,18 +39,29 @@ function readFragmentToken(): string | undefined { const params = new URLSearchParams(hash.slice(1)); const token = params.get(FRAGMENT_PARAM); if (!token) return undefined; + const redirectHash = params.get(REDIRECT_HASH_PARAM); // Scrub the fragment (keep path + query) so the token is not left in the // address bar, browser history, or any screenshot of the window. const url = new URL(window.location.href); - url.hash = ''; + url.hash = redirectHash ?? ''; window.history.replaceState( window.history.state, '', - `${url.pathname}${url.search}`, + `${url.pathname}${url.search}${url.hash}`, ); return token; } +export function withServerCredentialFragment(targetUrl: string, credential: string): string { + const url = new URL(targetUrl); + const redirectHash = url.hash.startsWith('#') ? url.hash.slice(1) : url.hash; + const params = new URLSearchParams(); + params.set(FRAGMENT_PARAM, credential); + if (redirectHash !== '') params.set(REDIRECT_HASH_PARAM, redirectHash); + url.hash = params.toString(); + return url.toString(); +} + function createStoredCredential(credential: string): StoredCredential { return { version: 1, diff --git a/apps/kimi-web/src/api/daemon/sessionOwnership.ts b/apps/kimi-web/src/api/daemon/sessionOwnership.ts new file mode 100644 index 0000000000..34dc1299be --- /dev/null +++ b/apps/kimi-web/src/api/daemon/sessionOwnership.ts @@ -0,0 +1,78 @@ +// apps/kimi-web/src/api/daemon/sessionOwnership.ts +// Session-ownership (multi-instance) error details — local re-implementation +// of the `details` payload carried under envelope code 40921 +// (`session.held_by_peer`). kimi-web must not depend on @moonshot-ai/agent-core +// or @moonshot-ai/protocol, so the zod schema in +// packages/protocol/src/session-ownership.ts is mirrored here as plain types +// plus structural narrowing. +// +// Wire semantics (from the protocol schema): +// - creating lease file observed mid-creation; retry shortly +// - routable holder is live and registered an address; redirect +// - held-by-local-instance holder has no address (local/embedded); terminal + +import { isDaemonApiError } from '../errors'; + +/** Envelope `code` for `session.held_by_peer` (see kap-server error-handler). */ +export const SESSION_HELD_BY_PEER_CODE = 40921; + +export type SessionOwnershipPhase = + | 'creating' + | 'routable' + | 'held-by-local-instance'; + +export interface HeldByPeerDetails { + kind: 'held-by-peer'; + phase: SessionOwnershipPhase; + /** Present only when phase === 'routable'. */ + address?: string; + /** Retry hint (ms) for 'creating'. */ + retry_after_ms?: number; +} + +export type SessionOwnershipDetails = HeldByPeerDetails; + +const PHASES: ReadonlySet = new Set([ + 'creating', + 'routable', + 'held-by-local-instance', +]); + +function isSessionOwnershipPhase(value: string): value is SessionOwnershipPhase { + return PHASES.has(value as SessionOwnershipPhase); +} + +/** Structurally narrow an unknown envelope `details` payload. Returns undefined + * for anything that is not a well-formed ownership payload (defensive: the + * server contract is new, and a malformed payload must degrade to the generic + * error toast rather than a crash). */ +export function narrowSessionOwnershipDetails( + details: unknown, +): SessionOwnershipDetails | undefined { + if (typeof details !== 'object' || details === null) return undefined; + const record = details as Record; + if (record['kind'] !== 'held-by-peer') return undefined; + const phase = record['phase']; + if (typeof phase !== 'string' || !isSessionOwnershipPhase(phase)) { + return undefined; + } + const address = record['address']; + const retryAfterMs = record['retry_after_ms']; + return { + kind: 'held-by-peer', + phase, + address: typeof address === 'string' && address.length > 0 ? address : undefined, + retry_after_ms: + typeof retryAfterMs === 'number' && Number.isFinite(retryAfterMs) && retryAfterMs >= 0 + ? retryAfterMs + : undefined, + }; +} + +/** Extract the ownership details from any thrown value: a DaemonApiError with + * code 40921 carrying a well-formed details payload. Anything else → undefined. */ +export function getSessionOwnershipDetails(err: unknown): SessionOwnershipDetails | undefined { + if (!isDaemonApiError(err)) return undefined; + if (err.code !== SESSION_HELD_BY_PEER_CODE) return undefined; + return narrowSessionOwnershipDetails(err.details); +} diff --git a/apps/kimi-web/src/api/daemon/wire.ts b/apps/kimi-web/src/api/daemon/wire.ts index 900d98bcf0..f7462b8380 100644 --- a/apps/kimi-web/src/api/daemon/wire.ts +++ b/apps/kimi-web/src/api/daemon/wire.ts @@ -583,7 +583,8 @@ export interface WireInFlightTurn { /** `GET /sessions/{sid}/snapshot` — atomic rebuild state at a watermark. */ export interface WireSessionSnapshot { as_of_seq: number; - epoch: string; + /** Absent until the journal's first durable event: "no baseline" ≠ "baseline changed". */ + epoch?: string; session: WireSession; messages: { items: WireMessage[]; has_more: boolean }; in_flight_turn: WireInFlightTurn | null; diff --git a/apps/kimi-web/src/api/daemon/ws.ts b/apps/kimi-web/src/api/daemon/ws.ts index 721677df8a..a2c1bc879d 100644 --- a/apps/kimi-web/src/api/daemon/ws.ts +++ b/apps/kimi-web/src/api/daemon/ws.ts @@ -6,6 +6,7 @@ import { traceWsIn, traceWsLifecycle, traceWsOut } from '../../debug/trace'; import { classifyFrame } from './agentEventProjector'; import { getCredential } from './serverAuth'; +import { SESSION_HELD_BY_PEER_CODE } from './sessionOwnership'; import type { WireEvent, WireServerFrame } from './wire'; // Mirrors kap-server's WS_BEARER_PROTOCOL_PREFIX. The browser WebSocket API @@ -44,8 +45,16 @@ export interface DaemonEventSocketHandlers { onResync(sessionId: string, currentSeq: number, epoch?: string): void; /** Called when the WS connection opens or closes */ onConnectionState(connected: boolean): void; - /** Called on error frames or JSON parse failures */ - onError(code: number, msg: string, fatal: boolean): void; + /** Called on error frames or JSON parse failures. `details` carries the + * structured payload of connection-level error frames when the server + * sends one (e.g. SessionOwnershipDetails under 40921). */ + onError(code: number, msg: string, fatal: boolean, details?: unknown): void; + /** Volatile hint (no payload): the shared session list changed on some + * instance (multi-instance home). The listener re-pulls the list. */ + onSessionListChanged?(): void; + /** Volatile per-session hint: the session's skill catalog changed on the + * daemon. The listener re-pulls that session's skills. */ + onSkillCatalogChanged?(sessionId: string): void; onTerminalOutput?(sessionId: string, terminalId: string, data: string, seq: number): void; onTerminalExit?(sessionId: string, terminalId: string, exitCode: number | null): void; } @@ -373,13 +382,40 @@ export class DaemonEventSocket { payload: frame.payload, }); } else { - this.handlers.onError(frame.payload.code, frame.payload.msg, frame.payload.fatal); + this.handlers.onError(frame.payload.code, frame.payload.msg, frame.payload.fatal, frame.payload?.details); } break; } + // Multi-instance hint (volatile, no payload): some instance sharing this + // home created/archived/deleted a session. Consumed by name here because + // classifyFrame would route the unknown unprefixed type to the agent + // projector, which no-ops on it. The server (sessionEventBroadcaster) + // only ever emits the bare form. + case 'session.list_changed': + this.handlers.onSessionListChanged?.(); + break; + + // Volatile per-session hint: the daemon's skill catalog for this session + // changed (e.g. a skill file edited on disk). Consumed by name here for + // the same reason as session.list_changed above; the frame carries its + // own per-connection seq, never the durable watermark. The server + // (skillCatalogBridge) only ever emits the bare form. + case 'skill_catalog.changed': + this.handlers.onSkillCatalogChanged?.(frame.session_id as string); + break; + case 'ack': - // ack frames are fire-and-forget for now (no request tracking) + if (frame.code === SESSION_HELD_BY_PEER_CODE) { + const ownershipDetails = frame.payload?.ownership_details; + if (ownershipDetails && typeof ownershipDetails === 'object') { + for (const details of Object.values(ownershipDetails as Record)) { + this.handlers.onError(frame.code, frame.msg, false, details); + } + } else { + this.handlers.onError(frame.code, frame.msg, false, frame.payload?.details); + } + } break; case 'terminal_output': { diff --git a/apps/kimi-web/src/api/types.ts b/apps/kimi-web/src/api/types.ts index 5c34d26ccb..82884ebe13 100644 --- a/apps/kimi-web/src/api/types.ts +++ b/apps/kimi-web/src/api/types.ts @@ -520,7 +520,8 @@ export interface AppInFlightTurn { */ export interface AppSessionSnapshot { asOfSeq: number; - epoch: string; + /** Absent until the journal's first durable event: "no baseline" ≠ "baseline changed". */ + epoch?: string; session: AppSession; /** Most recent messages, chronological ascending. */ messages: AppMessage[]; @@ -535,8 +536,18 @@ export interface AppSessionSnapshot { export interface KimiEventHandlers { onEvent(event: AppEvent, meta: KimiEventMeta): void; onResync(sessionId: string, currentSeq: number, epoch?: string): void; - onError(code: number, msg: string, fatal: boolean): void; + /** Connection-level error frame. `details` carries the structured payload + * newer servers attach (e.g. SessionOwnershipDetails under 40921); absent + * on older servers. */ + onError(code: number, msg: string, fatal: boolean, details?: unknown): void; onConnectionChange(connected: boolean): void; + /** Volatile multi-instance hint: the shared session list changed on some + * instance — refresh the sidebar list. No payload; loss is harmless. */ + onSessionListChanged?(): void; + /** Volatile per-session hint: the session's skill catalog changed on the + * daemon — refresh that session's skills (slash menu). Carries only the + * session id; loss is harmless (the next hint or reload converges). */ + onSkillCatalogChanged?(sessionId: string): void; onTerminalOutput?(sessionId: string, terminalId: string, data: string, seq: number): void; onTerminalExit?(sessionId: string, terminalId: string, exitCode: number | null): void; } diff --git a/apps/kimi-web/src/composables/client/useWorkspaceState.ts b/apps/kimi-web/src/composables/client/useWorkspaceState.ts index 03a541fe0a..e149088f77 100644 --- a/apps/kimi-web/src/composables/client/useWorkspaceState.ts +++ b/apps/kimi-web/src/composables/client/useWorkspaceState.ts @@ -314,6 +314,9 @@ export function useWorkspaceState(rawState: ExtendedState, deps: UseWorkspaceSta fileDiffLoading, } = deps; let exportInFlight = false; + // Re-entrancy guard for volatile session.list_changed refreshes — a burst of + // events must not stack overlapping re-pulls. + let sessionsListRefreshInFlight = false; async function loadOlderMessages(sessionId: string): Promise { if (rawState.messagesLoadingMoreBySession[sessionId]) return; @@ -819,6 +822,57 @@ export function useWorkspaceState(rawState: ExtendedState, deps: UseWorkspaceSta rawState.sessionsHasMoreByWorkspace = cleared; } + // Volatile per-session `skill_catalog.changed` hint: coalesce a burst of + // frames for one session into a single skills re-pull. + const SKILLS_REFRESH_DEBOUNCE_MS = 400; + const skillRefreshTimers = new Map>(); + + /** Re-pull a session's skills after a volatile `skill_catalog.changed` WS + * hint (a skill file was added/edited/removed on the daemon). Failing + * silently is correct — the event is a bare hint, so a lost refresh is + * indistinguishable from a lost event and the next hint or session reload + * converges the slash menu. Same rationale as refreshSessionsList. */ + function scheduleSkillsRefresh(sessionId: string): void { + const existing = skillRefreshTimers.get(sessionId); + if (existing !== undefined) clearTimeout(existing); + skillRefreshTimers.set( + sessionId, + setTimeout(() => { + skillRefreshTimers.delete(sessionId); + void modelProvider.loadSkillsForSession(sessionId); + }, SKILLS_REFRESH_DEBOUNCE_MS), + ); + } + + /** Re-pull the first session pages after a volatile `session.list_changed` + * WS hint (multi-instance home: another instance created/archived/deleted a + * session). Failing silently (log only) is correct — the event carries no + * payload, so a lost refresh is indistinguishable from a lost event and the + * next hint or manual reload converges the list. */ + async function refreshSessionsList(): Promise { + if (sessionsListRefreshInFlight) return; + sessionsListRefreshInFlight = true; + try { + const sessions = await loadInitialSessionsByWorkspace(); + // undefined = every workspace request failed: keep the current list + // rather than committing a false empty one (same contract as loadAll). + if (sessions === undefined) return; + // Keep the active session even when it falls outside the re-fetched + // first pages (deep link into an old session): dropping it would cut its + // snapshot/status sync and sidebar highlight mid-use. + const activeId = rawState.activeSessionId; + if (activeId !== undefined && !sessions.some((s) => s.id === activeId)) { + const current = rawState.sessions.find((s) => s.id === activeId); + if (current) sessions.push(current); + } + setSessionsPreservingLiveUsage(sessions); + } catch (err) { + console.warn('[kimi-web] sessions list refresh failed', err); + } finally { + sessionsListRefreshInFlight = false; + } + } + /** * Re-read GET /meta and apply the server-self fields (version, open-in * apps, auth bypass, backend engine). Called on first load and on every WS @@ -2761,6 +2815,8 @@ export function useWorkspaceState(rawState: ExtendedState, deps: UseWorkspaceSta loadWorkspaces, loadMoreSessions, loadAllSessions, + refreshSessionsList, + scheduleSkillsRefresh, selectWorkspace, openWorkspace, upsertWorkspacePreserveOrder, diff --git a/apps/kimi-web/src/composables/useKimiWebClient.ts b/apps/kimi-web/src/composables/useKimiWebClient.ts index f6e3f6a694..7555fc61da 100644 --- a/apps/kimi-web/src/composables/useKimiWebClient.ts +++ b/apps/kimi-web/src/composables/useKimiWebClient.ts @@ -7,6 +7,23 @@ import { i18n } from '../i18n'; import { traceClientEvent, traceKeyEvent } from '../debug/trace'; import { getKimiWebApi } from '../api'; import { isDaemonApiError, isDaemonNetworkError } from '../api/errors'; +import { + SESSION_HELD_BY_PEER_CODE, + getSessionOwnershipDetails, + narrowSessionOwnershipDetails, + type HeldByPeerDetails, + type SessionOwnershipDetails, +} from '../api/daemon/sessionOwnership'; +import { getCredential, withServerCredentialFragment } from '../api/daemon/serverAuth'; +import { + bumpRedirectBudget, + decideSessionOwnershipAction, + readRedirectBudget, + serializeRedirectBudget, + type OwnershipDecisionContext, + type OwnershipNotifyKey, + type RedirectBudget, +} from '../lib/sessionOwnershipDecision'; import { reconcileWorkspaceOrder, sortByWorkspaceOrder, @@ -1067,12 +1084,21 @@ function connectEventsIfNeeded(): void { snapshotSyncRunner.request(sessionId); }, - onError(code: number, msg: string, fatal: boolean) { + onError(code: number, msg: string, fatal: boolean, details?: unknown) { traceKeyEvent('ws:error', { status: 'failed', errorCode: code, fatal, }); + // A connection-level 40921 frame (multi-instance surfaces) carries the + // same ownership details as the REST envelope — same decision path. + if (code === SESSION_HELD_BY_PEER_CODE) { + const ownership = narrowSessionOwnershipDetails(details); + if (ownership !== undefined) { + handleSessionOwnership(ownership, { operation: 'ws' }); + return; + } + } pushWarning({ severity: 'error', title: i18n.global.t('warnings.wsTitle'), @@ -1083,6 +1109,19 @@ function connectEventsIfNeeded(): void { }); }, + // Volatile multi-instance hint: some instance sharing this home changed + // the session list — re-pull it (debounced, bursts coalesce). + onSessionListChanged() { + scheduleSessionsListRefresh(); + }, + + // Volatile per-session hint: this session's skill catalog changed on the + // daemon (e.g. a skill file edited on disk) — re-pull the skills list so + // the composer's `/` menu picks it up (debounced, bursts coalesce). + onSkillCatalogChanged(sessionId: string) { + workspaceState.scheduleSkillsRefresh(sessionId); + }, + onConnectionChange(connected: boolean) { traceKeyEvent('ws:connection', { status: connected ? 'connected' : 'disconnected', @@ -1107,7 +1146,7 @@ function connectEventsIfNeeded(): void { // Journal epoch per session, learned from snapshots / resync frames. Not // reactive — only consulted when building the subscribe cursor. -const epochBySession: Record = {}; +const epochBySession: Record = {}; // onResync resets the event projector, so that path must apply a snapshot even // if a newer global event advances the local cursor while the GET is in flight. const sessionsRequiringSnapshot = new Set(); @@ -1306,6 +1345,10 @@ function pushOperationFailure( phase: network ? err.phase : undefined, httpStatus: network ? err.status : undefined, }); + // Session-ownership contention (40921) is an expected multi-instance outcome + // with its own UX — redirect / auto-retry / targeted message — so it never + // reaches the generic failure toast. + if (handleSessionOwnershipError(err, { operation })) return; pushWarning(operationFailureNotice(operation, err, opts)); } @@ -1326,6 +1369,163 @@ function goalErrorMessage(err: unknown): string | undefined { return key ? i18n.global.t(key) : undefined; } +// --------------------------------------------------------------------------- +// Session ownership (multi-instance, envelope code 40921) +// --------------------------------------------------------------------------- +// +// Several kap-server instances can share one home. Opening (or writing to) a +// session whose lease is held by a sibling instance is rejected with +// SESSION_HELD_BY_PEER + structured details. The client-facing decision +// (redirect to the holder / auto-retry while it is mid-creation / targeted +// terminal message) is pure and lives in lib/sessionOwnershipDecision; this +// cluster owns the side effects: sessionStorage loop guard, toasts, the +// full-page navigation, and the creating retry timer. + +/** Loop-guard persistence: redirects started inside this window count towards + * PEER_MAX_REDIRECT_ATTEMPTS; older attempts expire so legitimate future + * redirects are never blocked by ancient history. */ +const PEER_REDIRECT_STORAGE_KEY = 'kimi.peerRedirectBudget'; +const PEER_REDIRECT_WINDOW_MS = 5 * 60_000; +const PEER_MAX_REDIRECT_ATTEMPTS = 3; +const OWNERSHIP_MAX_CREATING_ATTEMPTS = 3; +const OWNERSHIP_DEFAULT_RETRY_DELAY_MS = 1_000; + +/** 'creating' retries already fired per session (counts towards the cap). */ +const ownershipCreatingAttempts = new Map(); + +function readPeerRedirectBudgetSafe(now: number): RedirectBudget { + if (typeof window === 'undefined') return { count: 0, windowStart: now }; + try { + return readRedirectBudget( + window.sessionStorage.getItem(PEER_REDIRECT_STORAGE_KEY), + now, + PEER_REDIRECT_WINDOW_MS, + ); + } catch { + return { count: 0, windowStart: now }; + } +} + +function persistPeerRedirectBudgetSafe(budget: RedirectBudget): void { + if (typeof window === 'undefined') return; + try { + window.sessionStorage.setItem(PEER_REDIRECT_STORAGE_KEY, serializeRedirectBudget(budget)); + } catch { + // Storage unavailable (private mode…): the loop guard degrades to + // per-page-load only, which is still safe. + } +} + +type OwnershipNoticeMessageKey = OwnershipNotifyKey | 'redirecting' | 'creating'; + +function ownershipNotice(key: OwnershipNoticeMessageKey, params?: Record): AppNotice { + return { + // redirecting/creating are transient progress, the rest are terminal states. + severity: key === 'redirecting' || key === 'creating' ? 'info' : 'error', + title: i18n.global.t('warnings.ownership.title'), + message: + params === undefined + ? i18n.global.t(`warnings.ownership.${key}`) + : i18n.global.t(`warnings.ownership.${key}`, params), + }; +} + +function ownershipDecisionContext(overrides?: Partial): OwnershipDecisionContext { + const loc = typeof window === 'undefined' ? undefined : window.location; + return { + currentHost: loc?.host ?? '', + currentPath: loc === undefined ? '/' : `${loc.pathname}${loc.search}${loc.hash}`, + redirectAttempts: readPeerRedirectBudgetSafe(Date.now()).count, + maxRedirectAttempts: PEER_MAX_REDIRECT_ATTEMPTS, + creatingAttempts: 0, + maxCreatingAttempts: OWNERSHIP_MAX_CREATING_ATTEMPTS, + defaultRetryDelayMs: OWNERSHIP_DEFAULT_RETRY_DELAY_MS, + ...overrides, + }; +} + +/** Run the ownership decision for an already-narrowed details payload and + * apply its side effects (redirect / notice). Auto-retry for 'creating' only + * exists on the snapshot open path (scheduleOwnershipCreatingRetry); at this + * blanket level a 'retry' decision degrades to the informational notice. */ +function handleSessionOwnership(details: SessionOwnershipDetails, ctx: { operation: string }): void { + const action = decideSessionOwnershipAction(details, ownershipDecisionContext()); + traceKeyEvent('session:ownership', { + operation: ctx.operation, + action: action.type, + kind: details.kind, + phase: details.kind === 'held-by-peer' ? details.phase : undefined, + }); + switch (action.type) { + case 'redirect': { + pushWarning(ownershipNotice('redirecting', { origin: action.origin })); + persistPeerRedirectBudgetSafe(bumpRedirectBudget(readPeerRedirectBudgetSafe(Date.now()))); + if (typeof window !== 'undefined') { + const credential = getCredential(); + const target = credential === undefined + ? action.url + : withServerCredentialFragment(action.url, credential); + window.location.assign(target); + } + return; + } + case 'retry': + pushWarning(ownershipNotice('creating')); + return; + case 'notify': + pushWarning(ownershipNotice(action.key)); + return; + } +} + +/** Intercept an operation failure: when it carries ownership details (40921) + * run the ownership UX and report the error as handled — the caller must NOT + * additionally surface its generic failure toast. */ +function handleSessionOwnershipError(err: unknown, ctx: { operation: string }): boolean { + const details = getSessionOwnershipDetails(err); + if (details === undefined) return false; + handleSessionOwnership(details, ctx); + return true; +} + +/** Re-fire the snapshot sync after a 'creating' rejection, honouring the + * server-provided retry_after_ms and capping the attempts. The first attempt + * surfaces a transient notice; exhausting the cap surfaces the timeout one. */ +function scheduleOwnershipCreatingRetry(sessionId: string, details: HeldByPeerDetails): void { + const attempts = ownershipCreatingAttempts.get(sessionId) ?? 0; + const action = decideSessionOwnershipAction( + details, + ownershipDecisionContext({ creatingAttempts: attempts }), + ); + if (action.type === 'retry') { + ownershipCreatingAttempts.set(sessionId, attempts + 1); + if (attempts === 0) pushWarning(ownershipNotice('creating')); + traceKeyEvent('session:ownership-creating-retry', { + sessionId, + attempt: attempts + 1, + delayMs: action.delayMs, + }); + setTimeout(() => { + snapshotSyncRunner.request(sessionId); + }, action.delayMs); + return; + } + ownershipCreatingAttempts.delete(sessionId); + if (action.type === 'notify') pushWarning(ownershipNotice(action.key)); +} + +// The volatile session.list_changed event only says "something changed somewhere" +// — coalesce a burst into a single sidebar list re-pull. +const SESSIONS_LIST_REFRESH_DEBOUNCE_MS = 400; +let sessionsListRefreshTimer: ReturnType | null = null; +function scheduleSessionsListRefresh(): void { + if (sessionsListRefreshTimer !== null) clearTimeout(sessionsListRefreshTimer); + sessionsListRefreshTimer = setTimeout(() => { + sessionsListRefreshTimer = null; + void workspaceState.refreshSessionsList(); + }, SESSIONS_LIST_REFRESH_DEBOUNCE_MS); +} + async function handleSessionNotFound(sessionId: string): Promise { forgetSession(sessionId); @@ -1460,6 +1660,7 @@ async function syncSessionFromSnapshot(sessionId: string): Promise= ctx.maxCreatingAttempts) { + return { type: 'notify', key: 'creatingTimeout' }; + } + return { + type: 'retry', + delayMs: details.retry_after_ms ?? ctx.defaultRetryDelayMs, + }; + } + + case 'routable': { + const origin = details.address !== undefined ? normalizePeerOrigin(details.address) : undefined; + if (origin === undefined) return { type: 'notify', key: 'redirectUnavailable' }; + // Loop guard A: the "holder" resolves to ourselves — redirecting would + // reload the same page and fail the same way forever. + try { + if (new URL(origin).host === ctx.currentHost) { + return { type: 'notify', key: 'redirectSameHost' }; + } + } catch { + return { type: 'notify', key: 'redirectUnavailable' }; + } + // Loop guard B: A → B → A → … chains (stale registry, clock skew, two + // tabs fighting) must converge to a message instead of a reload storm. + if (ctx.redirectAttempts >= ctx.maxRedirectAttempts) { + return { type: 'notify', key: 'redirectLoopGuard' }; + } + return { type: 'redirect', url: buildPeerTargetUrl(origin, ctx.currentPath), origin }; + } + + case 'held-by-local-instance': + return { type: 'notify', key: 'heldByLocalInstance' }; + } +} + +// --------------------------------------------------------------------------- +// Redirect-attempt budget (loop guard persistence) +// --------------------------------------------------------------------------- + +/** Redirects started within this window count towards the loop guard. */ +export interface RedirectBudget { + count: number; + windowStart: number; +} + +/** Parse the persisted budget. An expired or malformed record starts fresh + * (stale attempts must not block a legitimate future redirect forever). */ +export function readRedirectBudget( + raw: string | null, + now: number, + windowMs: number, +): RedirectBudget { + if (raw !== null) { + try { + const parsed = JSON.parse(raw) as { count?: unknown; windowStart?: unknown }; + if ( + typeof parsed.count === 'number' && + Number.isFinite(parsed.count) && + parsed.count >= 0 && + typeof parsed.windowStart === 'number' && + Number.isFinite(parsed.windowStart) && + now - parsed.windowStart < windowMs + ) { + return { count: parsed.count, windowStart: parsed.windowStart }; + } + } catch { + // malformed — fall through to a fresh budget + } + } + return { count: 0, windowStart: now }; +} + +/** Record one more redirect start within the budget's window. */ +export function bumpRedirectBudget(budget: RedirectBudget): RedirectBudget { + return { count: budget.count + 1, windowStart: budget.windowStart }; +} + +export function serializeRedirectBudget(budget: RedirectBudget): string { + return JSON.stringify({ count: budget.count, windowStart: budget.windowStart }); +} diff --git a/apps/kimi-web/test/server-auth.test.ts b/apps/kimi-web/test/server-auth.test.ts index 853366c83e..2be29f61c3 100644 --- a/apps/kimi-web/test/server-auth.test.ts +++ b/apps/kimi-web/test/server-auth.test.ts @@ -282,6 +282,22 @@ describe('fragment token intake', () => { expect(replaceState).toHaveBeenCalledWith(null, '', '/some/path?x=1'); }); + it('builds a credential redirect fragment and restores the original hash after intake', async () => { + const auth = await loadAuth(); + const target = auth.withServerCredentialFragment( + 'http://127.0.0.1:58628/sessions/sess_1?debug=1#top', + 'tok+/=', + ); + const { replaceState } = installWindow(new URL(target).hash); + + expect(target).toBe( + 'http://127.0.0.1:58628/sessions/sess_1?debug=1#token=tok%2B%2F%3D&redirect_hash=top', + ); + expect(auth.initServerAuth()).toBe(true); + expect(auth.getCredential()).toBe('tok+/='); + expect(replaceState).toHaveBeenCalledWith(null, '', '/some/path?x=1#top'); + }); + it('ignores an empty fragment and falls back to storage', async () => { writeStoredCredential('stored-tok'); installWindow(''); diff --git a/apps/kimi-web/test/session-ownership.test.ts b/apps/kimi-web/test/session-ownership.test.ts new file mode 100644 index 0000000000..733075e6ed --- /dev/null +++ b/apps/kimi-web/test/session-ownership.test.ts @@ -0,0 +1,207 @@ +// apps/kimi-web/test/session-ownership.test.ts +// Session-ownership (40921) client handling: wire-details narrowing and the +// pure redirect / retry / notify decision. See +// src/api/daemon/sessionOwnership.ts and src/lib/sessionOwnershipDecision.ts. + +import { describe, expect, it } from 'vitest'; + +import { DaemonApiError } from '../src/api/errors'; +import { + SESSION_HELD_BY_PEER_CODE, + getSessionOwnershipDetails, + narrowSessionOwnershipDetails, + type SessionOwnershipDetails, +} from '../src/api/daemon/sessionOwnership'; +import { + bumpRedirectBudget, + decideSessionOwnershipAction, + normalizePeerOrigin, + readRedirectBudget, + serializeRedirectBudget, + type OwnershipDecisionContext, +} from '../src/lib/sessionOwnershipDecision'; + +const ROUTABLE: SessionOwnershipDetails = { + kind: 'held-by-peer', + phase: 'routable', + address: 'http://127.0.0.1:58628', +}; + +function makeCtx(overrides?: Partial): OwnershipDecisionContext { + return { + currentHost: '127.0.0.1:58627', + currentPath: '/sessions/sess_1?debug=1#top', + redirectAttempts: 0, + maxRedirectAttempts: 3, + creatingAttempts: 0, + maxCreatingAttempts: 3, + defaultRetryDelayMs: 1_000, + ...overrides, + }; +} + +describe('narrowSessionOwnershipDetails', () => { + it.each([ + ['null', null], + ['a string', 'held-by-peer'], + ['an empty object', {}], + ['an unknown kind', { kind: 'held-by-something-else' }], + ['an unknown phase', { kind: 'held-by-peer', phase: 'mystery' }], + ['a non-string phase', { kind: 'held-by-peer', phase: 1 }], + ])('rejects %s', (_label, input) => { + expect(narrowSessionOwnershipDetails(input)).toBeUndefined(); + }); +}); + +describe('getSessionOwnershipDetails', () => { + function apiError(code: number, details: unknown): DaemonApiError { + return new DaemonApiError({ code, msg: 'held', requestId: 'req_1', details }); + } + + it('extracts details from a 40921 DaemonApiError', () => { + expect( + getSessionOwnershipDetails( + apiError(SESSION_HELD_BY_PEER_CODE, { + kind: 'held-by-peer', + phase: 'creating', + retry_after_ms: 500, + }), + ), + ).toEqual({ kind: 'held-by-peer', phase: 'creating', address: undefined, retry_after_ms: 500 }); + }); + + it('returns undefined for other codes, malformed details, and non-API errors', () => { + expect( + getSessionOwnershipDetails(apiError(40401, { kind: 'held-by-peer', phase: 'routable' })), + ).toBeUndefined(); + expect(getSessionOwnershipDetails(apiError(SESSION_HELD_BY_PEER_CODE, { nope: 1 }))).toBeUndefined(); + expect(getSessionOwnershipDetails(new Error('boom'))).toBeUndefined(); + expect(getSessionOwnershipDetails(undefined)).toBeUndefined(); + }); + + it('matches the structural DaemonApiError guard (cross-realm error)', () => { + const crossRealm = { + name: 'DaemonApiError', + code: SESSION_HELD_BY_PEER_CODE, + details: { kind: 'held-by-peer', phase: 'held-by-local-instance' }, + }; + expect(getSessionOwnershipDetails(crossRealm)).toEqual({ + kind: 'held-by-peer', + phase: 'held-by-local-instance', + address: undefined, + retry_after_ms: undefined, + }); + }); +}); + +describe('normalizePeerOrigin', () => { + it.each([ + ['http://127.0.0.1:58628', 'http://127.0.0.1:58628'], + ['127.0.0.1:58628', 'http://127.0.0.1:58628'], + ['http://127.0.0.1:58628/api/v1/sessions?x=1#y', 'http://127.0.0.1:58628'], + ['https://kimi.example.com/', 'https://kimi.example.com'], + [' localhost:58628 ', 'http://localhost:58628'], + ])('normalizes %s → %s', (input, expected) => { + expect(normalizePeerOrigin(input)).toBe(expected); + }); + + it.each([ + ['empty', ''], + ['whitespace', ' '], + ['non-http scheme', 'ftp://127.0.0.1:58628'], + ])('rejects %s', (_label, input) => { + expect(normalizePeerOrigin(input)).toBeUndefined(); + }); +}); + +describe('decideSessionOwnershipAction', () => { + it('routable with address → redirect carrying origin + current path', () => { + expect(decideSessionOwnershipAction(ROUTABLE, makeCtx())).toEqual({ + type: 'redirect', + origin: 'http://127.0.0.1:58628', + url: 'http://127.0.0.1:58628/sessions/sess_1?debug=1#top', + }); + }); + + it('routable with a bare host:port address → scheme defaults to http', () => { + const action = decideSessionOwnershipAction( + { kind: 'held-by-peer', phase: 'routable', address: '127.0.0.1:58628' }, + makeCtx(), + ); + expect(action).toMatchObject({ type: 'redirect', origin: 'http://127.0.0.1:58628' }); + }); + + it('routable resolving to the current host → same-host notice, no redirect loop', () => { + expect( + decideSessionOwnershipAction(ROUTABLE, makeCtx({ currentHost: '127.0.0.1:58628' })), + ).toEqual({ type: 'notify', key: 'redirectSameHost' }); + }); + + it('routable after too many redirects → loop-guard notice', () => { + expect(decideSessionOwnershipAction(ROUTABLE, makeCtx({ redirectAttempts: 3 }))).toEqual({ + type: 'notify', + key: 'redirectLoopGuard', + }); + }); + + it('routable without a usable address → unavailable notice', () => { + expect( + decideSessionOwnershipAction({ kind: 'held-by-peer', phase: 'routable' }, makeCtx()), + ).toEqual({ type: 'notify', key: 'redirectUnavailable' }); + expect( + decideSessionOwnershipAction( + { kind: 'held-by-peer', phase: 'routable', address: 'ftp://x' }, + makeCtx(), + ), + ).toEqual({ type: 'notify', key: 'redirectUnavailable' }); + }); + + it('creating → retry with the server hint, capped by attempts', () => { + expect( + decideSessionOwnershipAction( + { kind: 'held-by-peer', phase: 'creating', retry_after_ms: 250 }, + makeCtx(), + ), + ).toEqual({ type: 'retry', delayMs: 250 }); + // No hint → default delay. + expect(decideSessionOwnershipAction({ kind: 'held-by-peer', phase: 'creating' }, makeCtx())).toEqual({ + type: 'retry', + delayMs: 1_000, + }); + // Cap exhausted → timeout notice. + expect( + decideSessionOwnershipAction( + { kind: 'held-by-peer', phase: 'creating' }, + makeCtx({ creatingAttempts: 3 }), + ), + ).toEqual({ type: 'notify', key: 'creatingTimeout' }); + }); + + it('held-by-local-instance → terminal notice', () => { + expect( + decideSessionOwnershipAction({ kind: 'held-by-peer', phase: 'held-by-local-instance' }, makeCtx()), + ).toEqual({ type: 'notify', key: 'heldByLocalInstance' }); + }); +}); + +describe('redirect budget (loop guard persistence)', () => { + const WINDOW_MS = 5 * 60_000; + + it('starts fresh on null / malformed / expired records', () => { + expect(readRedirectBudget(null, 1_000, WINDOW_MS)).toEqual({ count: 0, windowStart: 1_000 }); + expect(readRedirectBudget('not-json', 1_000, WINDOW_MS)).toEqual({ count: 0, windowStart: 1_000 }); + expect(readRedirectBudget('{"count":5}', 1_000, WINDOW_MS)).toEqual({ count: 0, windowStart: 1_000 }); + const stale = serializeRedirectBudget({ count: 2, windowStart: 1_000 }); + expect(readRedirectBudget(stale, 1_000 + WINDOW_MS + 1, WINDOW_MS)).toEqual({ + count: 0, + windowStart: 1_000 + WINDOW_MS + 1, + }); + }); + + it('keeps counts inside the window and bumps without moving windowStart', () => { + const raw = serializeRedirectBudget({ count: 2, windowStart: 1_000 }); + const budget = readRedirectBudget(raw, 2_000, WINDOW_MS); + expect(budget).toEqual({ count: 2, windowStart: 1_000 }); + expect(bumpRedirectBudget(budget)).toEqual({ count: 3, windowStart: 1_000 }); + }); +}); diff --git a/apps/kimi-web/test/workspace-state.test.ts b/apps/kimi-web/test/workspace-state.test.ts index d8871d9f8b..c41810d482 100644 --- a/apps/kimi-web/test/workspace-state.test.ts +++ b/apps/kimi-web/test/workspace-state.test.ts @@ -2105,3 +2105,54 @@ describe('useWorkspaceState — upsertWorkspacePreserveOrder hidden roots', () = expect(state.hiddenWorkspaceRoots).toEqual(['/home/Foo']); }); }); + +// Volatile `skill_catalog.changed` hint: a burst of frames for one session +// must coalesce into a single skills re-pull (mirrors the +// session.list_changed sidebar refresh debounce). +describe('useWorkspaceState — scheduleSkillsRefresh', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + function skillsRefreshDeps(loadSkillsForSession: ReturnType): UseWorkspaceStateDeps { + return { + ...createDeps(), + modelProvider: { + draftModel: ref(null), + skillsBySession: ref({}), + loadSkillsForSession, + } as unknown as UseWorkspaceStateDeps['modelProvider'], + }; + } + + it('coalesces a burst of hints into one skills re-pull per session', () => { + const loadSkillsForSession = vi.fn().mockResolvedValue(undefined); + const ws = useWorkspaceState(createState(), skillsRefreshDeps(loadSkillsForSession)); + + ws.scheduleSkillsRefresh('sess_1'); + ws.scheduleSkillsRefresh('sess_1'); + ws.scheduleSkillsRefresh('sess_1'); + expect(loadSkillsForSession).not.toHaveBeenCalled(); + + vi.advanceTimersByTime(400); + expect(loadSkillsForSession).toHaveBeenCalledTimes(1); + expect(loadSkillsForSession).toHaveBeenCalledWith('sess_1'); + }); + + it('debounces per session, not globally', () => { + const loadSkillsForSession = vi.fn().mockResolvedValue(undefined); + const ws = useWorkspaceState(createState(), skillsRefreshDeps(loadSkillsForSession)); + + ws.scheduleSkillsRefresh('sess_1'); + ws.scheduleSkillsRefresh('sess_2'); + vi.advanceTimersByTime(400); + + expect(loadSkillsForSession).toHaveBeenCalledTimes(2); + expect(loadSkillsForSession).toHaveBeenCalledWith('sess_1'); + expect(loadSkillsForSession).toHaveBeenCalledWith('sess_2'); + }); +}); diff --git a/apps/kimi-web/test/ws-lifecycle.test.ts b/apps/kimi-web/test/ws-lifecycle.test.ts index e70686a396..edd98192e2 100644 --- a/apps/kimi-web/test/ws-lifecycle.test.ts +++ b/apps/kimi-web/test/ws-lifecycle.test.ts @@ -69,19 +69,21 @@ const SERVER_HELLO = { }, }; -describe('DaemonEventSocket reconnect + staleness', () => { +function setupFakeWebSocket(): void { let originalWebSocket: typeof globalThis.WebSocket; - beforeEach(() => { FakeWebSocket.instances = []; originalWebSocket = globalThis.WebSocket; globalThis.WebSocket = FakeWebSocket as unknown as typeof WebSocket; }); - afterEach(() => { globalThis.WebSocket = originalWebSocket; vi.useRealTimers(); }); +} + +describe('DaemonEventSocket reconnect + staleness', () => { + setupFakeWebSocket(); it('reconnect() closes the old socket, detaches it, and opens a new one', () => { const handlers = makeHandlers(); @@ -100,10 +102,6 @@ describe('DaemonEventSocket reconnect + staleness', () => { // A fresh socket was created, and we reported the transient disconnect. expect(FakeWebSocket.instances).toHaveLength(2); expect(handlers.states).toEqual([true, false]); - - // A late onclose from the stale socket must NOT schedule another connect. - first.onclose?.({ code: 1000, reason: 'reconnect', wasClean: true }); - expect(FakeWebSocket.instances).toHaveLength(2); }); it('reconnect() is a no-op after close()', () => { @@ -144,3 +142,142 @@ describe('DaemonEventSocket reconnect + staleness', () => { expect(socket.health().open).toBe(false); }); }); + +describe('DaemonEventSocket frame dispatch (multi-instance surface)', () => { + setupFakeWebSocket(); + + it('consumes bare session.list_changed via onSessionListChanged; the event.-prefixed form is not special-cased', () => { + let wireEvents = 0; + let listChanged = 0; + const handlers: DaemonEventSocketHandlers = { + onWireEvent: () => { + wireEvents += 1; + }, + onResync: () => {}, + onConnectionState: () => {}, + onError: () => {}, + onSessionListChanged: () => { + listChanged += 1; + }, + }; + const socket = new DaemonEventSocket(WS_URL, CLIENT_ID, handlers); + socket.connect(); + const ws = FakeWebSocket.instances[0]!; + ws.emitMessage(SERVER_HELLO); + + ws.emitMessage({ type: 'session.list_changed', payload: {} }); + // The server never emits the prefixed form; it must NOT reach + // onSessionListChanged (it falls through to the generic protocol path). + ws.emitMessage({ type: 'event.session.list_changed', payload: {} }); + + expect(listChanged).toBe(1); + expect(wireEvents).toBe(1); + }); + + it('consumes bare skill_catalog.changed via onSkillCatalogChanged; the event.-prefixed form is not special-cased', () => { + let wireEvents = 0; + let rawAgentEvents = 0; + const changed: string[] = []; + const handlers: DaemonEventSocketHandlers = { + onWireEvent: () => { + wireEvents += 1; + }, + onRawAgentEvent: () => { + rawAgentEvents += 1; + }, + onResync: () => {}, + onConnectionState: () => {}, + onError: () => {}, + onSkillCatalogChanged: (sessionId) => { + changed.push(sessionId); + }, + }; + const socket = new DaemonEventSocket(WS_URL, CLIENT_ID, handlers); + socket.connect(); + const ws = FakeWebSocket.instances[0]!; + ws.emitMessage(SERVER_HELLO); + + ws.emitMessage({ + type: 'skill_catalog.changed', + seq: 1, + session_id: 'sess_1', + timestamp: '2026-01-01T00:00:00.000Z', + volatile: true, + payload: { type: 'skill_catalog.changed', sourceId: 'workspace-file' }, + }); + // The server never emits the prefixed form; it must NOT reach + // onSkillCatalogChanged (it falls through to the generic protocol path). + ws.emitMessage({ + type: 'event.skill_catalog.changed', + seq: 2, + session_id: 'sess_2', + timestamp: '2026-01-01T00:00:00.000Z', + volatile: true, + payload: { type: 'skill_catalog.changed', sourceId: 'plugin' }, + }); + + expect(changed).toEqual(['sess_1']); + expect(wireEvents).toBe(1); + expect(rawAgentEvents).toBe(0); + }); + + it('passes connection-level error-frame details through to onError', () => { + const errors: Array<{ code: number; msg: string; fatal: boolean; details: unknown }> = []; + const handlers: DaemonEventSocketHandlers = { + onWireEvent: () => {}, + onResync: () => {}, + onConnectionState: () => {}, + onError: (code, msg, fatal, details) => { + errors.push({ code, msg, fatal, details }); + }, + }; + const socket = new DaemonEventSocket(WS_URL, CLIENT_ID, handlers); + socket.connect(); + const ws = FakeWebSocket.instances[0]!; + ws.emitMessage(SERVER_HELLO); + + const ownership = { kind: 'held-by-peer', phase: 'routable', address: 'http://127.0.0.1:58628' }; + ws.emitMessage({ type: 'error', payload: { code: 40921, msg: 'session held by peer', fatal: false, details: ownership } }); + + expect(errors).toEqual([{ code: 40921, msg: 'session held by peer', fatal: false, details: ownership }]); + }); + + it('keeps session-scoped error frames on the agent-event path (no details leak to onError)', () => { + const errors: unknown[] = []; + const raw: Array<{ type: string; session_id: string; payload: unknown }> = []; + const handlers: DaemonEventSocketHandlers = { + onWireEvent: () => {}, + onResync: () => {}, + onConnectionState: () => {}, + onError: (...args) => { + errors.push(args); + }, + onRawAgentEvent: (frame) => { + raw.push(frame); + }, + }; + const socket = new DaemonEventSocket(WS_URL, CLIENT_ID, handlers); + socket.connect(); + const ws = FakeWebSocket.instances[0]!; + ws.emitMessage(SERVER_HELLO); + + ws.emitMessage({ + type: 'error', + session_id: 'sess_1', + seq: 7, + timestamp: 't', + payload: { code: 40300, msg: 'provider denied' }, + }); + + expect(errors).toEqual([]); + expect(raw).toEqual([ + { + type: 'error', + seq: 7, + session_id: 'sess_1', + timestamp: 't', + payload: { code: 40300, msg: 'provider denied' }, + }, + ]); + }); +}); diff --git a/flake.nix b/flake.nix index 7f56b35994..6f2f7282d2 100644 --- a/flake.nix +++ b/flake.nix @@ -67,6 +67,7 @@ ./packages/agent-core-v2 ./packages/kap-server ./packages/kaos + ./packages/kernel-file-lock ./packages/klient ./packages/kosong ./packages/migration-legacy @@ -93,6 +94,7 @@ "@moonshot-ai/agent-core-v2" "@moonshot-ai/kap-server" "@moonshot-ai/kaos" + "@moonshot-ai/kernel-file-lock" "@moonshot-ai/kosong" "@moonshot-ai/migration-legacy" "@moonshot-ai/minidb" @@ -160,7 +162,7 @@ inherit (finalAttrs) pname version src pnpmWorkspaces; inherit pnpm; fetcherVersion = 3; - hash = "sha256-+pzJfoWJwVXIUU8oc56LVpfNjSY6MABID5g11Cm92xw="; + hash = "sha256-DG0qhbCF74KeNe0LKHuGAQufDzTdtXxvzFBYHl4xs9w="; }; nativeBuildInputs = [ diff --git a/packages/agent-core-v2/package.json b/packages/agent-core-v2/package.json index d3e4f5da3e..577824654f 100644 --- a/packages/agent-core-v2/package.json +++ b/packages/agent-core-v2/package.json @@ -60,6 +60,7 @@ "@google/genai": "^1.49.0", "@jsquash/webp": "^1.5.0", "@modelcontextprotocol/sdk": "^1.29.0", + "@moonshot-ai/kernel-file-lock": "workspace:^", "@moonshot-ai/kimi-code-oauth": "workspace:^", "@moonshot-ai/minidb": "workspace:^", "@moonshot-ai/protocol": "workspace:^", diff --git a/packages/agent-core-v2/scripts/check-domain-layers.mjs b/packages/agent-core-v2/scripts/check-domain-layers.mjs index d02848869c..1bd14a5bf0 100644 --- a/packages/agent-core-v2/scripts/check-domain-layers.mjs +++ b/packages/agent-core-v2/scripts/check-domain-layers.mjs @@ -102,6 +102,10 @@ const DOMAIN_LAYER = new Map([ // Depends only on `_base`; sits in L1 beside the other program-control // layer substrates. ['task', 1], + // `sessionLease` owns the low-level per-session write-fencing capability. + // It depends only on the L1 cross-process lock and write-authority contracts + // (plus L0 infrastructure), so consumers must not pull it up to their layer. + ['sessionLease', 1], // persistence/ and os/ — the two-level scopes. `interface` holds contracts // (same layer as the old domains they replace); `backends` holds // implementations that may depend on cross-domain services at various layers. @@ -121,6 +125,10 @@ const DOMAIN_LAYER = new Map([ ['config', 2], ['workspaceLocalConfig', 2], ['sessionFs', 2], + // `sessionFileLedger` is the Session-scope optimistic-concurrency ledger for + // the write path: it consumes the `sessionFs` dirty-tick state and the L1 + // hostFs stat, so it sits beside `sessionFs` at L2. + ['sessionFileLedger', 2], ['process', 2], ['workspaceRegistry', 2], ['hostFolderBrowser', 2], @@ -166,6 +174,10 @@ const DOMAIN_LAYER = new Map([ ['usage', 4], ['runtime', 4], ['toolDedupe', 4], + // `fileFencing` is the Agent-scope optimistic-concurrency gate: a + // tool-executor hook participant (L3) over the `sessionFileLedger` (L2), + // so it sits at L4 beside `toolDedupe`. + ['fileFencing', 4], ['toolSelect', 4], ['toolPolicy', 4], ['contextMemory', 4], diff --git a/packages/agent-core-v2/src/_base/di/instantiation.ts b/packages/agent-core-v2/src/_base/di/instantiation.ts index 5642c087b4..c13d9091a8 100644 --- a/packages/agent-core-v2/src/_base/di/instantiation.ts +++ b/packages/agent-core-v2/src/_base/di/instantiation.ts @@ -16,7 +16,7 @@ export namespace _util { export function getServiceDependencies( ctor: DI_TARGET_OBJ, // eslint-disable-next-line @typescript-eslint/no-explicit-any - ): { id: ServiceIdentifier; index: number }[] { + ): { id: ServiceIdentifier; index: number; optional: boolean }[] { return ctor[DI_DEPENDENCIES] || []; } @@ -25,7 +25,7 @@ export namespace _util { // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type [DI_TARGET]: Function; // eslint-disable-next-line @typescript-eslint/no-explicit-any - [DI_DEPENDENCIES]: { id: ServiceIdentifier; index: number }[]; + [DI_DEPENDENCIES]: { id: ServiceIdentifier; index: number; optional: boolean }[]; } } @@ -56,12 +56,13 @@ function storeServiceDependency( // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type target: Function, index: number, + optional: boolean, ): void { const t = target as _util.DI_TARGET_OBJ; if (t[_util.DI_TARGET] === target) { - t[_util.DI_DEPENDENCIES].push({ id, index }); + t[_util.DI_DEPENDENCIES].push({ id, index, optional }); } else { - t[_util.DI_DEPENDENCIES] = [{ id, index }]; + t[_util.DI_DEPENDENCIES] = [{ id, index, optional }]; t[_util.DI_TARGET] = target; } } @@ -83,7 +84,7 @@ export function createDecorator(name: string): ServiceIdentifier { '@IServiceName-decorator can only be used to decorate a parameter', ); } - storeServiceDependency(id, target, index); + storeServiceDependency(id, target, index, false); } as unknown as ServiceIdentifier; Object.defineProperty(id, 'toString', { @@ -99,6 +100,12 @@ export function createDecorator(name: string): ServiceIdentifier { return id; } +export function optional(id: ServiceIdentifier): ParameterDecorator { + return (target, _key, index): void => { + storeServiceDependency(id, target as Function, index, true); + }; +} + export function refineServiceDecorator( serviceIdentifier: ServiceIdentifier, ): ServiceIdentifier { diff --git a/packages/agent-core-v2/src/_base/di/instantiationService.ts b/packages/agent-core-v2/src/_base/di/instantiationService.ts index 52084a85b7..625442b9d9 100644 --- a/packages/agent-core-v2/src/_base/di/instantiationService.ts +++ b/packages/agent-core-v2/src/_base/di/instantiationService.ts @@ -255,8 +255,15 @@ export class InstantiationService implements IInstantiationService { const serviceDependencies = _util.getServiceDependencies(ctor).toSorted((a, b) => a.index - b.index); const serviceArgs: unknown[] = []; for (const dependency of serviceDependencies) { - const service = this._getOrCreateServiceInstance(dependency.id, _trace); - if (!service) { + if ( + dependency.optional && + this._getServiceInstanceOrDescriptor(dependency.id) === undefined + ) { + serviceArgs.push(undefined); + continue; + } + const service = this._getOrCreateServiceInstance(dependency.id, _trace); + if (!service) { this._throwIfStrict( `[createInstance] ${ctor.name} depends on UNKNOWN service ${String(dependency.id)}.`, false, @@ -349,6 +356,7 @@ export class InstantiationService implements IInstantiationService { for (const dependency of _util.getServiceDependencies(item.desc.ctor)) { const instanceOrDesc = this._getServiceInstanceOrDescriptor(dependency.id); if (!instanceOrDesc) { + if (dependency.optional) continue; this._throwIfStrict( `[createInstance] ${String(item.id)} depends on ${String(dependency.id)} which is NOT registered.`, true, diff --git a/packages/agent-core-v2/src/_base/utils/fs.ts b/packages/agent-core-v2/src/_base/utils/fs.ts index a93e9699a2..8a35d4a72e 100644 --- a/packages/agent-core-v2/src/_base/utils/fs.ts +++ b/packages/agent-core-v2/src/_base/utils/fs.ts @@ -1,14 +1,49 @@ /** - * Low-level durable file-write primitives — atomic writes plus file and - * directory fsync helpers. + * Low-level filesystem helpers — durable file-write primitives (atomic writes + * plus file and directory fsync), stat classification shared by watchers, and + * the stat-tuple comparison shared by every stat-only staleness check. */ import { randomBytes } from 'node:crypto'; import { closeSync, fsyncSync, openSync } from 'node:fs'; import * as nodeFs from 'node:fs'; +import type { Stats } from 'node:fs'; import { open, rename, unlink } from 'node:fs/promises'; import { dirname } from 'pathe'; +/** + * True for directory entries that are neither a regular file, a directory, + * nor a symlink (unix sockets, fifos, devices). File watchers must skip + * these: chokidar attaches `fs.watch` to every scanned entry and that call + * throws `UNKNOWN` on special files. + */ +export function isSpecialFileStat(stats: Stats | undefined): boolean { + return stats !== undefined && !stats.isFile() && !stats.isDirectory() && !stats.isSymbolicLink(); +} + +/** + * The (`ino`, `mtimeMs`, `size`) tuple identifying one on-disk revision of a + * file — the comparison unit of every stat-only staleness check (write-path + * fencing, storage-watch fingerprints, read/edit TOCTOU revalidation). + * `ino`/`mtimeMs` are optional because not every stat source supplies them. + */ +export interface FileStatTuple { + readonly ino?: number; + readonly mtimeMs?: number; + readonly size: number; +} + +/** + * Tuple equality tolerant of a missing stat (`undefined`): two missing + * tuples are equal, exactly one missing is not. + */ +export function fileStatTuplesEqual( + a: FileStatTuple | undefined, + b: FileStatTuple | undefined, +): boolean { + return a?.ino === b?.ino && a?.mtimeMs === b?.mtimeMs && a?.size === b?.size; +} + export async function syncDir(dirPath: string): Promise { if (process.platform === 'win32') return; const dirFh = await open(dirPath, 'r'); diff --git a/packages/agent-core-v2/src/_base/utils/promise.ts b/packages/agent-core-v2/src/_base/utils/promise.ts index 811bbda9d5..203b3394f0 100644 --- a/packages/agent-core-v2/src/_base/utils/promise.ts +++ b/packages/agent-core-v2/src/_base/utils/promise.ts @@ -31,3 +31,26 @@ export function timeoutOutcome( }, }); } + +/** + * Keyed promise-chain exclusion over a shared `id → tail promise` map: `op` + * runs after every previously enqueued operation for `id` settles, a + * rejection does not poison later operations for the same key, and the map + * entry is dropped once the queue drains. + */ +export function enqueueKeyedOperation( + queues: Map>, + id: string, + op: () => Promise, +): Promise { + const previous = queues.get(id) ?? Promise.resolve(); + const result = previous.then(op); + const tail = result.then( + () => undefined, + () => undefined, + ); + queues.set(id, tail); + return result.finally(() => { + if (queues.get(id) === tail) queues.delete(id); + }); +} diff --git a/packages/agent-core-v2/src/agent/fileFencing/fileFencing.ts b/packages/agent-core-v2/src/agent/fileFencing/fileFencing.ts new file mode 100644 index 0000000000..c69163976c --- /dev/null +++ b/packages/agent-core-v2/src/agent/fileFencing/fileFencing.ts @@ -0,0 +1,19 @@ +/** + * `fileFencing` domain (L4) — `IAgentFileFencingService` contract. + * + * The Agent-scope tool-hook participant that gates `Write`/`Edit` calls on + * the `sessionFileLedger` optimistic-concurrency verdict and re-baselines + * the ledger after successful `Read`/`Write`/`Edit` executions. The service + * exists for its constructor side effects (it registers by name on the + * `toolExecutor` hook slots); nothing calls its methods. Bound at Agent + * scope. + */ + +import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; + +export interface IAgentFileFencingService { + readonly _serviceBrand: undefined; +} + +export const IAgentFileFencingService: ServiceIdentifier = + createDecorator('agentFileFencingService'); diff --git a/packages/agent-core-v2/src/agent/fileFencing/fileFencingService.ts b/packages/agent-core-v2/src/agent/fileFencing/fileFencingService.ts new file mode 100644 index 0000000000..2c7ce70be9 --- /dev/null +++ b/packages/agent-core-v2/src/agent/fileFencing/fileFencingService.ts @@ -0,0 +1,145 @@ +/** + * `fileFencing` domain (L4) — `IAgentFileFencingService` implementation. + * + * Registers the `writeFencing` participant on the `toolExecutor` + * `onBeforeExecuteTool` / `onDidExecuteTool` hook slots, matching + * `Read`/`Write`/`Edit` by tool name and letting every other tool pass + * through. For Edit and overwrite Write the before hook injects an execution + * wrapper; append Write remains non-destructive and bypasses read-first. The + * wrapper re-checks the ledger only after the scheduler grants the declared + * file access, so a queued write cannot rely on a stale preflight verdict. The + * target path comes from the resolved execution's file accesses + * — the exact canonical path the tool itself computed. `stale` blocks with an + * outside-modification conflict and `no-baseline` blocks with a read-first + * reason (Edit-over-existing, or overwrite Write over an existing file). The + * did-hook baselines the ledger from the revision the successful call captured + * on its own result (ranged Reads excepted — per the ledger contract they + * never count as full reads); direct creation of a new file is verdict-`clean`, + * so it never blocks. Checked after `permission` (ignition order is set by + * `agentLifecycle`). Bound at Agent scope. + */ + +import { InstantiationType } from '#/_base/di/extensions'; +import { Disposable } from '#/_base/di/lifecycle'; +import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; +import { IAgentToolExecutorService } from '#/agent/toolExecutor/toolExecutor'; +import type { + ToolBeforeExecuteContext, + ToolDidExecuteContext, + ToolExecutionHookContext, +} from '#/agent/toolExecutor/toolHooks'; +import { + ISessionFileLedger, + type FileLedgerVerdict, +} from '#/session/sessionFileLedger/fileLedger'; +import { toolFileRevision, type ToolAccesses } from '#/tool/toolContract'; + +import { IAgentFileFencingService } from './fileFencing'; + +const READ_TOOL = 'Read'; +const WRITE_TOOLS = new Set(['Write', 'Edit']); + +function isFenced(ctx: ToolExecutionHookContext): boolean { + return ctx.toolCall.name === READ_TOOL || WRITE_TOOLS.has(ctx.toolCall.name); +} + +function targetPathOf(ctx: ToolBeforeExecuteContext): string | undefined { + return fileAccessPath(ctx.execution.accesses); +} + +function fileAccessPath(accesses: ToolAccesses | undefined): string | undefined { + if (accesses === undefined) return undefined; + for (const access of accesses) { + if (access.kind === 'file') return access.path; + } + return undefined; +} + +function isRangedRead(args: unknown): boolean { + if (typeof args !== 'object' || args === null) return false; + const input = args as { readonly line_offset?: unknown; readonly n_lines?: unknown }; + return input.line_offset !== undefined || input.n_lines !== undefined; +} + +function isAppendWrite(ctx: ToolExecutionHookContext): boolean { + if (ctx.toolCall.name !== 'Write' || typeof ctx.args !== 'object' || ctx.args === null) { + return false; + } + return (ctx.args as { readonly mode?: unknown }).mode === 'append'; +} + +function blockReason(toolName: string, path: string, verdict: FileLedgerVerdict): string { + if (verdict === 'no-baseline') { + return toolName === 'Edit' + ? `Editing "${path}" is blocked: the file exists but has not been read in this session. Read it first, then retry the edit.` + : `Writing "${path}" is blocked: the file already exists but has not been read in this session. Read it first, then retry the write.`; + } + const verb = toolName === 'Edit' ? 'Editing' : 'Writing'; + return ( + `${verb} "${path}" is blocked: the file changed on disk since it was last read or written ` + + 'in this session. Read it again, then retry.' + ); +} + +export class AgentFileFencingService extends Disposable implements IAgentFileFencingService { + declare readonly _serviceBrand: undefined; + + constructor( + @ISessionFileLedger private readonly ledger: ISessionFileLedger, + @IAgentToolExecutorService toolExecutor: IAgentToolExecutorService, + ) { + super(); + toolExecutor.hooks.onBeforeExecuteTool.register('writeFencing', async (ctx, next) => { + this.onBefore(ctx); + if (ctx.decision?.block === true) return; + await next(); + }); + toolExecutor.hooks.onDidExecuteTool.register('writeFencing', async (ctx, next) => { + this.onDid(ctx); + await next(); + }); + } + + private onBefore(ctx: ToolBeforeExecuteContext): void { + if (!isFenced(ctx)) return; + const path = targetPathOf(ctx); + if (path === undefined) return; + if (!WRITE_TOOLS.has(ctx.toolCall.name)) return; + if (isAppendWrite(ctx)) return; + const execute = ctx.decision?.execute ?? ctx.execution.execute; + ctx.decision = { + ...ctx.decision, + execute: async (executeCtx) => { + const verdict = await this.ledger.compare(path); + if (verdict !== 'clean') { + const reason = blockReason(ctx.toolCall.name, path, verdict); + return { output: reason, isError: true }; + } + return execute(executeCtx); + }, + }; + } + + private onDid(ctx: ToolDidExecuteContext): void { + if (!isFenced(ctx)) return; + if (ctx.result.isError === true) return; + if (ctx.toolCall.name === READ_TOOL && isRangedRead(ctx.args)) return; + const revision = ctx.result[toolFileRevision]; + if (revision !== undefined) { + this.ledger.recordBaseline(revision.path, { + exists: true, + ino: revision.ino, + mtimeMs: revision.mtimeMs, + size: revision.size, + }); + } + } +} + +registerScopedService( + LifecycleScope.Agent, + IAgentFileFencingService, + AgentFileFencingService, + InstantiationType.Eager, + 'fileFencing', +); diff --git a/packages/agent-core-v2/src/agent/profile/agentSkillListingReminder.ts b/packages/agent-core-v2/src/agent/profile/agentSkillListingReminder.ts new file mode 100644 index 0000000000..852ef2fdf9 --- /dev/null +++ b/packages/agent-core-v2/src/agent/profile/agentSkillListingReminder.ts @@ -0,0 +1,22 @@ +/** + * `profile` domain (L4) — `IAgentSkillListingReminderService` contract. + * + * Turn-boundary model notification for skill hot reload: the skill listing is + * baked into the agent's system prompt at bind time, so after the session + * skill catalog merges a filesystem- or config-driven reload the provider + * registered by this service fires once at the agent's next new turn, appending + * a system reminder with the refreshed `catalog.getModelSkillListing()` (whose + * first line already supersedes earlier listings). Bound at Agent scope — each + * agent carries its own pending flag. + */ + +import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; + +export const SKILL_CATALOG_CHANGED_INJECTION_VARIANT = 'skill_catalog_changed'; + +export interface IAgentSkillListingReminderService { + readonly _serviceBrand: undefined; +} + +export const IAgentSkillListingReminderService: ServiceIdentifier = + createDecorator('agentSkillListingReminderService'); diff --git a/packages/agent-core-v2/src/agent/profile/agentSkillListingReminderService.ts b/packages/agent-core-v2/src/agent/profile/agentSkillListingReminderService.ts new file mode 100644 index 0000000000..994e0202e2 --- /dev/null +++ b/packages/agent-core-v2/src/agent/profile/agentSkillListingReminderService.ts @@ -0,0 +1,72 @@ +/** + * `profile` domain (L4) — `IAgentSkillListingReminderService` implementation. + * + * Subscribes the session skill catalog's `onDidChange` and registers one + * context-injection provider into the agent's `contextInjector`; the provider + * stays silent until an injection pass runs at a new turn (`isNewTurn`), then + * clears the pending flag and returns the change note plus the current + * `sessionSkillCatalog` model listing, which the injector appends as a system + * reminder. Bound at Agent scope. + */ + +import { Disposable } from '#/_base/di/lifecycle'; +import { InstantiationType } from '#/_base/di/extensions'; +import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; +import { IAgentContextInjectorService } from '#/agent/contextInjector/contextInjector'; +import { ISessionSkillCatalog } from '#/session/sessionSkillCatalog/skillCatalog'; + +import { + IAgentSkillListingReminderService, + SKILL_CATALOG_CHANGED_INJECTION_VARIANT, +} from './agentSkillListingReminder'; + +export class AgentSkillListingReminderService + extends Disposable + implements IAgentSkillListingReminderService +{ + declare readonly _serviceBrand: undefined; + + private pending = false; + + constructor( + @IAgentContextInjectorService injector: IAgentContextInjectorService, + @ISessionSkillCatalog private readonly skillCatalog: ISessionSkillCatalog, + ) { + super(); + this._register( + this.skillCatalog.onDidChange(() => { + this.pending = true; + }), + ); + this._register( + injector.register(SKILL_CATALOG_CHANGED_INJECTION_VARIANT, ({ isNewTurn }) => { + if (!isNewTurn || !this.pending) return undefined; + this.pending = false; + return this.renderReminder(); + }), + ); + } + + private renderReminder(): string { + const listing = this.skillCatalog.catalog.getModelSkillListing(); + if (listing === '') { + return ( + 'The skill catalog changed during this session; there are currently no invocable skills. ' + + 'This supersedes any earlier skill listing reminder in this session.' + ); + } + return ( + 'The skill catalog changed during this session; the listing below supersedes any earlier ' + + 'skill listing reminder in this session.' + + `\n\n${listing}` + ); + } +} + +registerScopedService( + LifecycleScope.Agent, + IAgentSkillListingReminderService, + AgentSkillListingReminderService, + InstantiationType.Eager, + 'profile', +); diff --git a/packages/agent-core-v2/src/agent/task/persist.ts b/packages/agent-core-v2/src/agent/task/persist.ts index 9fc37d94fb..fe096696cb 100644 --- a/packages/agent-core-v2/src/agent/task/persist.ts +++ b/packages/agent-core-v2/src/agent/task/persist.ts @@ -73,7 +73,7 @@ export class AgentTaskPersistence { private readonly agentScope: string, private readonly docs: IAtomicDocumentStore, private readonly bytes: IFileSystemStorageService, - private readonly fallbackRoot?: AgentTaskPersistenceRoot, + private readonly fallbackRoot: AgentTaskPersistenceRoot | undefined, ) {} private primaryRoot(): AgentTaskPersistenceRoot { @@ -122,6 +122,7 @@ export class AgentTaskPersistence { async appendTaskOutput(taskId: string, chunk: string): Promise { if (chunk.length === 0) return; + validateTaskId(taskId); await this.bytes.append(this.taskOutputScope(taskId), OUTPUT_LOG_KEY, textEncoder.encode(chunk)); } diff --git a/packages/agent-core-v2/src/agent/task/task.ts b/packages/agent-core-v2/src/agent/task/task.ts index 9e091502fa..c97502fc4a 100644 --- a/packages/agent-core-v2/src/agent/task/task.ts +++ b/packages/agent-core-v2/src/agent/task/task.ts @@ -83,7 +83,9 @@ export interface IAgentTaskService { registerTask(task: AgentTask, options?: RegisterAgentTaskOptions): string; getTask(taskId: string): AgentTaskInfo | undefined; list(activeOnly?: boolean, limit?: number): readonly AgentTaskInfo[]; + beginClose(): void; persistOutput(taskId: string): void; + flushPersistence(): Promise; getOutputSnapshot( taskId: string, maxPreviewBytes: number, diff --git a/packages/agent-core-v2/src/agent/task/taskService.ts b/packages/agent-core-v2/src/agent/task/taskService.ts index ff59ca0bf1..8b5811a721 100644 --- a/packages/agent-core-v2/src/agent/task/taskService.ts +++ b/packages/agent-core-v2/src/agent/task/taskService.ts @@ -36,6 +36,7 @@ import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; import type { ContentPart } from '#/kosong/contract/message'; import { Disposable } from '#/_base/di/lifecycle'; +import { toErrorMessage } from '#/_base/errors/errorMessage'; import { abortable, userCancellationReason, @@ -218,6 +219,7 @@ export class AgentTaskService extends Disposable implements IAgentTaskService { private readonly deliveredNotificationKeys = new Set(); private readonly persistence: AgentTaskPersistence; private activeTaskReminderPending = false; + private _closing = false; constructor( @ITelemetryService private readonly telemetry: ITelemetryService, @@ -294,6 +296,10 @@ export class AgentTaskService extends Disposable implements IAgentTaskService { } } + beginClose(): void { + this._closing = true; + } + registerTask(task: AgentTask, options: RegisterAgentTaskOptions = {}): string { const detached = options.detached ?? true; const timeoutMs = options.timeoutMs ?? task.timeoutMs; @@ -476,6 +482,38 @@ export class AgentTaskService extends Disposable implements IAgentTaskService { this.startOutputPersist(entry); } + async flushPersistence(): Promise { + let firstFailure: unknown; + for (;;) { + const entries = [...this.tasks.values()]; + for (const entry of entries) { + if (entry.pendingOutput.length > 0) this.startOutputPersist(entry); + } + const queues = entries.flatMap((entry) => [ + { entry, kind: 'state' as const, promise: entry.persistWriteQueue }, + { entry, kind: 'output' as const, promise: entry.outputWriteQueue }, + ]); + const results = await Promise.allSettled(queues.map(({ promise }) => promise)); + for (const result of results) { + if (result.status === 'rejected' && firstFailure === undefined) { + firstFailure = result.reason; + } + } + const changed = queues.some(({ entry, kind, promise }) => { + return kind === 'state' + ? entry.persistWriteQueue !== promise + : entry.outputWriteQueue !== promise; + }); + if (changed) continue; + if (firstFailure !== undefined) { + throw firstFailure instanceof Error + ? firstFailure + : new Error(toErrorMessage(firstFailure)); + } + return; + } + } + async loadFromDisk(options: AgentTaskLoadOptions = {}): Promise { const persistence = this.persistence; if (options.replace !== false) { @@ -828,6 +866,7 @@ export class AgentTaskService extends Disposable implements IAgentTaskService { } private assertCanRegister(detached: boolean): void { + if (this._closing) throw new Error('Agent task service is closing.'); const maxRunningTasks = resolveAgentTaskConfig(this.config)?.maxRunningTasks; if (maxRunningTasks === undefined) return; if (!detached) return; @@ -872,8 +911,8 @@ export class AgentTaskService extends Disposable implements IAgentTaskService { const persistence = this.persistence; const info = this.toInfo(entry); entry.persistWriteQueue = entry.persistWriteQueue - .then(() => persistence.writeTask(info)) - .catch(() => { }); + .then(() => persistence.writeTask(info)); + void entry.persistWriteQueue.catch(() => {}); return entry.persistWriteQueue; } @@ -907,8 +946,8 @@ export class AgentTaskService extends Disposable implements IAgentTaskService { private appendTaskOutput(entry: ManagedTask, chunk: string): void { const persistence = this.persistence; entry.outputWriteQueue = entry.outputWriteQueue - .then(() => persistence.appendTaskOutput(entry.taskId, chunk)) - .catch(() => { }); + .then(() => persistence.appendTaskOutput(entry.taskId, chunk)); + void entry.outputWriteQueue.catch(() => {}); } private startOutputPersist(entry: ManagedTask): void { diff --git a/packages/agent-core-v2/src/agent/toolExecutor/toolExecutorService.ts b/packages/agent-core-v2/src/agent/toolExecutor/toolExecutorService.ts index b1ef5b6984..74ee677ae9 100644 --- a/packages/agent-core-v2/src/agent/toolExecutor/toolExecutorService.ts +++ b/packages/agent-core-v2/src/agent/toolExecutor/toolExecutorService.ts @@ -30,8 +30,12 @@ import { type ToolExecution, type ToolResult, type ToolUpdate, + toolFileRevision, } from '#/tool/toolContract'; -import type { ToolDidExecuteContext, ToolBeforeExecuteContext } from '#/agent/toolExecutor/toolHooks'; +import type { + ToolBeforeExecuteContext, + ToolDidExecuteContext, +} from '#/agent/toolExecutor/toolHooks'; import { IAgentToolRegistryService } from '#/agent/toolRegistry/toolRegistry'; import { ILogService } from '#/_base/log/log'; import type { ToolCallEvent } from '#/app/telemetry/events'; @@ -375,14 +379,22 @@ export class AgentToolExecutorService implements IAgentToolExecutorService { } const executionMetadata = decision?.executionMetadata; + const runnableExecution = + decision?.execute === undefined ? execution : { ...execution, execute: decision.execute }; this.dispatchToolCall(call, call.args, options, displayFields); return { task: { - accesses: execution.accesses ?? ToolAccesses.all(), + accesses: runnableExecution.accesses ?? ToolAccesses.all(), execute: async (taskSignal) => - this.runSingleExecution(call, execution, executionMetadata, options, taskSignal), + this.runSingleExecution( + call, + runnableExecution, + executionMetadata, + options, + taskSignal, + ), }, stopBatchAfterThis: execution.stopBatchAfterThis, }; @@ -826,7 +838,12 @@ function normalizeToolResult(result: ExecutableToolResult): ToolResult { stopTurn?: boolean; truncated?: true; note?: string; - } = { output, stopTurn: result.stopTurn }; + [toolFileRevision]?: ToolResult[typeof toolFileRevision]; + } = { + output, + stopTurn: result.stopTurn, + [toolFileRevision]: result[toolFileRevision], + }; if (result.truncated === true) base.truncated = true; if (typeof result.note === 'string' && result.note.length > 0) base.note = result.note; if (result.isError === true) { diff --git a/packages/agent-core-v2/src/agent/toolExecutor/toolHooks.ts b/packages/agent-core-v2/src/agent/toolExecutor/toolHooks.ts index 21a420605e..3aa629f44c 100644 --- a/packages/agent-core-v2/src/agent/toolExecutor/toolHooks.ts +++ b/packages/agent-core-v2/src/agent/toolExecutor/toolHooks.ts @@ -33,6 +33,7 @@ export interface AuthorizeToolExecutionResult { readonly reason?: string | undefined; readonly syntheticResult?: ExecutableToolResult | undefined; readonly executionMetadata?: unknown; + readonly execute?: RunnableToolExecution['execute']; } export interface PrepareToolExecutionResult extends AuthorizeToolExecutionResult { diff --git a/packages/agent-core-v2/src/app/bootstrap/bootstrapService.ts b/packages/agent-core-v2/src/app/bootstrap/bootstrapService.ts index 381b00ddd8..2334fc2a98 100644 --- a/packages/agent-core-v2/src/app/bootstrap/bootstrapService.ts +++ b/packages/agent-core-v2/src/app/bootstrap/bootstrapService.ts @@ -13,7 +13,7 @@ * Bound at App scope. */ -import { basename, join, relative } from 'pathe'; +import { join, relative } from 'pathe'; import { InstantiationType } from '#/_base/di/extensions'; import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; @@ -58,7 +58,7 @@ export class BootstrapService implements IBootstrapService { this.storeDir = join(options.homeDir, 'store'); this.cacheDir = join(options.homeDir, 'cache'); this.logsDir = join(options.homeDir, 'logs'); - this.configKey = basename(options.configPath); + this.configKey = relative(options.homeDir, options.configPath); this.scopes = { config: '', sessions: relative(options.homeDir, this.sessionsDir), diff --git a/packages/agent-core-v2/src/app/config/configService.ts b/packages/agent-core-v2/src/app/config/configService.ts index d9a9ebaab7..75c8f363e5 100644 --- a/packages/agent-core-v2/src/app/config/configService.ts +++ b/packages/agent-core-v2/src/app/config/configService.ts @@ -10,9 +10,13 @@ * — plus a `delivered` snapshot per domain used as the diff base for * `onDidSectionChange`. Reads config paths and the environment overlay through * `bootstrap`, persists the TOML document through the `storage` TOML - * atomic-document store (reloading when the document changes on disk), and logs - * through `log`. Late section / overlay registration re-validates the - * already-loaded raw value and re-runs overlays. Bound at App scope. + * atomic-document store (reloading when the document changes on disk), + * serializes every persist through the atomic-document store's transactional + * update primitive (backed by a cross-process lock for file storage), re-reading + * inside the lock and applying only the touched section so writes from other + * processes survive, and logs through `log`. + * Late section / overlay registration re-validates the already-loaded raw + * value and re-runs overlays. Bound at App scope. */ import { InstantiationType } from '#/_base/di/extensions'; @@ -25,7 +29,6 @@ import { IAtomicTomlDocumentStore, type IAtomicDocumentStore, } from '#/persistence/interface/atomicDocumentStore'; - import { type AnyEnvBindings, type ConfigChangedEvent, @@ -331,8 +334,10 @@ export class ConfigService extends Disposable implements IConfigService { } else { this.raw[domain] = stripped; } + const previousRaw = this.raw; await this.persist(domain); this.rebuildEffective('set', [domain]); + this.commitAbsorbed(previousRaw); }); } @@ -358,8 +363,10 @@ export class ConfigService extends Disposable implements IConfigService { } else { this.raw[domain] = this.registry.validate(domain, stripped); } + const previousRaw = this.raw; await this.persist(domain); this.rebuildEffective('set', [domain]); + this.commitAbsorbed(previousRaw); }); } @@ -549,8 +556,24 @@ export class ConfigService extends Disposable implements IConfigService { } private async persist(domain: string): Promise { - applySectionToToml(this.rawSnake, domain, this.raw[domain], this.registry); - await this.documentStore.set(CONFIG_SCOPE, this.configKey, this.rawSnake); + const freshBase = await this.documentStore.update( + CONFIG_SCOPE, + this.configKey, + (data) => { + const next = data !== undefined && isPlainObject(data) ? cloneRecord(data) : {}; + applySectionToToml(next, domain, this.raw[domain], this.registry); + return next; + }, + ); + this.rawSnake = freshBase; + this.raw = transformTomlData(freshBase, this.registry); + } + + private commitAbsorbed(previousRaw: ResolvedConfig): void { + const absorbed = [...new Set([...Object.keys(previousRaw), ...Object.keys(this.raw)])].filter( + (domain) => !deepEqual(previousRaw[domain], this.raw[domain]), + ); + if (absorbed.length > 0) this.commit('reload', absorbed); } } diff --git a/packages/agent-core-v2/src/app/edit/fileEdit.ts b/packages/agent-core-v2/src/app/edit/fileEdit.ts index c8f7c3c08d..69eb371f61 100644 --- a/packages/agent-core-v2/src/app/edit/fileEdit.ts +++ b/packages/agent-core-v2/src/app/edit/fileEdit.ts @@ -10,6 +10,7 @@ */ import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; +import type { HostFileStat } from '#/os/interface/hostFileSystem'; export interface FileEditInput { readonly path: string; @@ -20,7 +21,7 @@ export interface FileEditInput { } export type FileEditResult = - | { readonly ok: true; readonly count: number } + | { readonly ok: true; readonly count: number; readonly stat: HostFileStat } | { readonly ok: false; readonly error: string }; export interface IFileEditService { diff --git a/packages/agent-core-v2/src/app/edit/fileEditService.ts b/packages/agent-core-v2/src/app/edit/fileEditService.ts index 7601c12445..6cf29aa42f 100644 --- a/packages/agent-core-v2/src/app/edit/fileEditService.ts +++ b/packages/agent-core-v2/src/app/edit/fileEditService.ts @@ -2,15 +2,17 @@ * `edit` domain (L4) — `IFileEditService` implementation. * * Reads the file through the os `hostFs` domain (`IHostFileSystem`), runs the - * pure edit logic (`TextModel` + `EditService`), and writes the re-materialized - * content back. Maps host-level failures (e.g. `EISDIR`) to the domain-neutral - * `FileEditResult`; it owns no tool-facing message, which the Agent `EditTool` - * adapter supplies. Bound at App scope. + * pure edit logic (`TextModel` + `EditService`), verifies that the stat tuple + * stayed stable across the read/transform window, and writes the + * re-materialized content back. Maps host-level failures (e.g. `EISDIR`) to + * the domain-neutral `FileEditResult`; it owns no tool-facing message, which + * the Agent `EditTool` adapter supplies. Bound at App scope. */ import { InstantiationType } from '#/_base/di/extensions'; import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; import { unwrapErrorCause } from '#/_base/errors/errors'; +import { fileStatTuplesEqual } from '#/_base/utils/fs'; import { IHostFileSystem } from '#/os/interface/hostFileSystem'; import { EditService } from './editService'; @@ -28,7 +30,15 @@ export class FileEditService implements IFileEditService { async edit(input: FileEditInput): Promise { try { + const beforeRead = await this.fs.stat(input.path); const raw = await this.fs.readText(input.path, { errors: 'strict' }); + const afterRead = await this.fs.stat(input.path); + if (!fileStatTuplesEqual(beforeRead, afterRead)) { + return { + ok: false, + error: `${input.displayPath} changed on disk while the edit was being prepared. Read it again, then retry.`, + }; + } const model = new TextModel(raw); const result = this.editor.apply(model, { path: input.displayPath, @@ -39,8 +49,15 @@ export class FileEditService implements IFileEditService { if (!result.ok) { return { ok: false, error: result.error }; } - await this.fs.writeText(input.path, result.rawContent); - return { ok: true, count: result.count }; + const beforeWrite = await this.fs.stat(input.path); + if (!fileStatTuplesEqual(afterRead, beforeWrite)) { + return { + ok: false, + error: `${input.displayPath} changed on disk while the edit was being prepared. Read it again, then retry.`, + }; + } + const stat = await this.fs.writeText(input.path, result.rawContent); + return { ok: true, count: result.count, stat }; } catch (error) { const code = (unwrapErrorCause(error) as { code?: unknown } | null)?.code; if (code === 'EISDIR') { diff --git a/packages/agent-core-v2/src/app/edit/tools/edit.ts b/packages/agent-core-v2/src/app/edit/tools/edit.ts index 8eda6b50a0..001d7bee42 100644 --- a/packages/agent-core-v2/src/app/edit/tools/edit.ts +++ b/packages/agent-core-v2/src/app/edit/tools/edit.ts @@ -32,9 +32,11 @@ import { IHostEnvironment } from '#/os/interface/hostEnvironment'; import { ISessionSkillCatalog } from '#/session/sessionSkillCatalog/skillCatalog'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; import { + attachToolFileRevision, ToolAccesses, type BuiltinTool, type ExecutableToolResult, + makeToolFileRevision, type ToolExecution, } from '#/tool/toolContract'; import { registerTool } from '#/agent/toolRegistry/toolContribution'; @@ -135,7 +137,10 @@ export class EditTool implements BuiltinTool { return { isError: true, output: result.error }; } const word = result.count === 1 ? 'occurrence' : 'occurrences'; - return { output: `Replaced ${String(result.count)} ${word} in ${args.path}` }; + return attachToolFileRevision( + { output: `Replaced ${String(result.count)} ${word} in ${args.path}` }, + makeToolFileRevision(safePath, result.stat), + ); } } diff --git a/packages/agent-core-v2/src/app/sessionLifecycle/sessionLifecycle.ts b/packages/agent-core-v2/src/app/sessionLifecycle/sessionLifecycle.ts index c57bb8d214..f01128f3af 100644 --- a/packages/agent-core-v2/src/app/sessionLifecycle/sessionLifecycle.ts +++ b/packages/agent-core-v2/src/app/sessionLifecycle/sessionLifecycle.ts @@ -5,8 +5,10 @@ * `ForkSessionOptions`, `CreateChildSessionOptions`, and the * `ISessionLifecycleService` used to create sessions (`create`), look up the * live ones (`get` / `list`), close them (`close`), archive/restore them, - * fork them (`fork`), and fork-then-tag them as direct children (`createChild`). Announces - * lifecycle transitions through ordered hook slots plus + * fork them (`fork`), and fork-then-tag them as direct children (`createChild`). + * Close and archive always release their lease; a release failure is + * dirty-marked and abandoned internally. Lifecycle transitions run through + * ordered hook slots plus * `onDidCreateSession` / `onDidCloseSession` / `onDidArchiveSession` / * `onDidForkSession`. App-scoped — a single * process-wide instance owns the live session scope tree. Persisted @@ -63,9 +65,17 @@ export interface SessionWillCloseEvent { readonly reason: SessionCloseReason; } +export type SessionReleaseReason = 'close' | 'archive' | 'dirty-abort'; + +export interface SessionWillReleaseEvent { + readonly sessionId: string; + readonly reason: SessionReleaseReason; +} + export type SessionLifecycleHooks = { readonly onDidCreateSession: SessionCreatedEvent; readonly onWillCloseSession: SessionWillCloseEvent; + readonly onWillReleaseSession: SessionWillReleaseEvent; }; export interface SessionArchivedEvent { @@ -86,11 +96,13 @@ export interface ISessionLifecycleService { readonly onDidArchiveSession: Event; readonly onDidForkSession: Event; readonly hooks: Hooks; + beginClose(): Promise; create(opts: CreateSessionOptions): Promise; get(sessionId: string): ISessionScopeHandle | undefined; list(): readonly ISessionScopeHandle[]; resume(sessionId: string): Promise; close(sessionId: string): Promise; + closeAll(): Promise; archive(sessionId: string): Promise; restore(sessionId: string): Promise; fork(opts: ForkSessionOptions): Promise; diff --git a/packages/agent-core-v2/src/app/sessionLifecycle/sessionLifecycleService.ts b/packages/agent-core-v2/src/app/sessionLifecycle/sessionLifecycleService.ts index 311370523c..f0cccda8aa 100644 --- a/packages/agent-core-v2/src/app/sessionLifecycle/sessionLifecycleService.ts +++ b/packages/agent-core-v2/src/app/sessionLifecycle/sessionLifecycleService.ts @@ -32,6 +32,14 @@ * kicked fire-and-forget). The session-level eager services whose * subscriptions must exist before the first agent / turn (external hooks, * cron) are force-instantiated at the same point. + * + * Every materialization (create/resume/fork-target) first takes the session's + * cross-process write lease under `session-leases/` and registers that lease + * as the session's write admission. Close/archive stop producers, flush the + * session's append-log tail, seal new write admission, await already-admitted + * I/O, and then release the lease. Any release failure converges internally to + * a dirty-marked abandoned release; callers never need a second teardown + * operation. Lease loss follows the same fail-closed release path. */ import { randomUUID } from 'node:crypto'; @@ -41,7 +49,7 @@ import { ulid } from 'ulid'; import { InstantiationType } from '#/_base/di/extensions'; import { IInstantiationService } from '#/_base/di/instantiation'; -import { Disposable } from '#/_base/di/lifecycle'; +import { Disposable, type IDisposable } from '#/_base/di/lifecycle'; import { createScopedChildHandle, type ISessionScopeHandle, @@ -50,6 +58,7 @@ import { } from '#/_base/di/scope'; import { unwrapErrorCause } from '#/_base/errors/errors'; import { Emitter, type Event } from '#/_base/event'; +import { ILogService } from '#/_base/log/log'; import { DEFAULT_PLAN_MODE_SECTION } from '#/agent/plan/configSection'; import { IAgentPlanService } from '#/agent/plan/plan'; import { IBootstrapService } from '#/app/bootstrap/bootstrap'; @@ -72,13 +81,29 @@ import { IHostEnvironment } from '#/os/interface/hostEnvironment'; import { IHostFileSystem, type HostDirEntry } from '#/os/interface/hostFileSystem'; import { IAppendLogStore } from '#/persistence/interface/appendLogStore'; import { IAtomicDocumentStore } from '#/persistence/interface/atomicDocumentStore'; +import { IStorageWriteAdmission } from '#/persistence/interface/storageWriteAdmission'; +import { + type CrossProcessLockInspection, + ICrossProcessLockService, + OsLockErrors, +} from '#/os/interface/crossProcessLock'; import { IAgentLifecycleService, MAIN_AGENT_ID } from '#/session/agentLifecycle/agentLifecycle'; +import { IAgentTaskService } from '#/agent/task/task'; import { ensureMainAgent } from '#/session/agentLifecycle/mainAgent'; import { ISessionMcpService } from '#/session/mcp/sessionMcp'; import { labelsFromAgentMeta } from '#/session/agentLifecycle/subagentMetadata'; import { ISessionExternalHooksService } from '#/session/externalHooks/externalHooks'; import { ISessionContext, sessionContextSeed } from '#/session/sessionContext/sessionContext'; import { ISessionCronService } from '#/session/cron/sessionCronService'; +import { + type HeldByPeerDetails, + HELD_BY_PEER_CREATING_DETAILS, + heldByPeerDetailsFromInspection, + SessionLease, + sessionLeasePath, + sessionLeaseSeed, +} from '#/session/sessionLease/sessionLease'; +import { ISessionLeaseContactProvider } from '#/session/sessionLease/sessionLeaseContactProvider'; import { ISessionMetadata, type SessionMeta } from '#/session/sessionMetadata/sessionMetadata'; import { ISessionSkillCatalog } from '#/session/sessionSkillCatalog/skillCatalog'; import { ISessionAgentProfileCatalog } from '#/session/sessionAgentProfileCatalog/sessionAgentProfileCatalog'; @@ -101,6 +126,7 @@ import { type SessionForkedEvent, type SessionLifecycleHooks, type SessionWillCloseEvent, + type SessionWillReleaseEvent, ISessionLifecycleService, } from './sessionLifecycle'; @@ -109,9 +135,33 @@ type MaterializeSessionOptions = Omit & { readonly workspaceId?: string; }; +type SessionEntryState = 'opening' | 'active' | 'closing'; +type SessionReleaseKind = 'close' | 'archive'; +type SessionReleaseStage = + | 'set-archived' + | 'drain-agents' + | 'publish-archive' + | 'will-close' + | 'dispose-scope' + | 'will-release' + | 'flush' + | 'drain-writes' + | 'lease-lost' + | 'shutdown'; + +interface SessionEntry { + state: SessionEntryState; + readonly handle: ISessionScopeHandle; + readonly lease: SessionLease; + readonly registration: IDisposable; + readonly scope: string; + disposed: boolean; + releasePromise?: Promise; +} + export class SessionLifecycleService extends Disposable implements ISessionLifecycleService { declare readonly _serviceBrand: undefined; - private readonly sessions = new Map(); + private readonly entries = new Map(); private readonly _onDidCreateSession = this._register(new Emitter()); readonly onDidCreateSession: Event = this._onDidCreateSession.event; private readonly _onDidCloseSession = this._register(new Emitter()); @@ -123,8 +173,11 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec readonly hooks = createHooks([ 'onDidCreateSession', 'onWillCloseSession', + 'onWillReleaseSession', ]); private readonly resuming = new Map>(); + private readonly inFlightOperations = new Set>(); + private closing = false; constructor( @IInstantiationService private readonly instantiation: IInstantiationService, @@ -141,13 +194,29 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec private readonly workspaceLocalConfig: IWorkspaceLocalConfigService, @IEventService private readonly event: IEventService, @ITelemetryService private readonly telemetry: ITelemetryService, + @ILogService private readonly log: ILogService, + @ICrossProcessLockService private readonly locks: ICrossProcessLockService, + @IStorageWriteAdmission private readonly writeAdmission: IStorageWriteAdmission, + @ISessionLeaseContactProvider + private readonly leaseContact: ISessionLeaseContactProvider, ) { super(); } + beginClose(): Promise { + this.closing = true; + return Promise.allSettled([...this.inFlightOperations, ...this.resuming.values()]).then(() => undefined); + } + async create(opts: CreateSessionOptions): Promise { + this.assertOpen(); + return this.trackOperation(this.doCreate(opts)); + } + + private async doCreate(opts: CreateSessionOptions): Promise { const sessionId = opts.sessionId ?? createSessionId(); - const handle = await this.materializeSession({ ...opts, sessionId }); + const entry = await this.materializeSession({ ...opts, sessionId }); + const handle = entry.handle; try { const main = opts.mainAgentBinding === undefined @@ -168,19 +237,19 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec opts.workDir, handle.accessor.get(ISessionContext).workspaceId, ); + this.activateSession(entry); + await this.announceCreated({ sessionId, handle, source: 'startup' }); + return handle; } catch (error) { const sessionDir = handle.accessor.get(ISessionContext).sessionDir; - this.sessions.delete(sessionId); await this.drainAgents(handle).catch(() => {}); - handle.dispose(); + this.rollbackSession(entry); await this.hostFs.remove(sessionDir).catch(() => {}); throw error; } - await this.announceCreated({ sessionId, handle, source: 'startup' }); - return handle; } - private async materializeSession(opts: MaterializeSessionOptions): Promise { + private async materializeSession(opts: MaterializeSessionOptions): Promise { const workspace = await this.workspaceRegistry.createOrTouch(opts.workDir); const workspaceId = opts.workspaceId ?? workspace.id; const sessionScope = this.bootstrap.sessionScope(workspaceId, opts.sessionId); @@ -203,31 +272,55 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec ); const additionalDirs = [...localWorkspaceDirs.additionalDirs, ...callerAdditionalDirs]; await this.hostEnv.ready; - const handle = createScopedChildHandle( - this.instantiation, - LifecycleScope.Session, - opts.sessionId, - { - extra: [...sessionContextSeed(ctx)], - }, - ) as ISessionScopeHandle; - if (additionalDirs.length > 0) { - handle.accessor.get(ISessionWorkspaceContext).setAdditionalDirs(additionalDirs); + const lease = await this.acquireSessionLease(opts.sessionId); + let registration: IDisposable; + try { + registration = this.writeAdmission.registerSession(sessionScope, lease); + } catch (error) { + lease.release(); + throw error; } + let entry: SessionEntry | undefined; try { + const handle = createScopedChildHandle( + this.instantiation, + LifecycleScope.Session, + opts.sessionId, + { + extra: [...sessionContextSeed(ctx), ...sessionLeaseSeed(lease)], + }, + ) as ISessionScopeHandle; + if (additionalDirs.length > 0) { + handle.accessor.get(ISessionWorkspaceContext).setAdditionalDirs(additionalDirs); + } + entry = { + state: 'opening', + handle, + lease, + registration, + scope: sessionScope, + disposed: false, + }; + this.entries.set(opts.sessionId, entry); + // Force-instantiate the session-level eager services whose subscriptions + // must exist before the first agent / turn (external hooks, cron). + handle.accessor.get(ISessionExternalHooksService); + handle.accessor.get(ISessionCronService); await handle.accessor.get(ISessionMetadata).ready; await handle.accessor.get(ISessionToolPolicy).ready; void handle.accessor.get(ISessionSkillCatalog).ready; await handle.accessor.get(ISessionAgentProfileCatalog).ready; await handle.accessor.get(ISessionMcpService).ensureMcpReady(opts.mcpServers); - handle.accessor.get(ISessionExternalHooksService); - handle.accessor.get(ISessionCronService); + return entry; } catch (error) { - handle.dispose(); + if (entry !== undefined) { + this.rollbackSession(entry); + } else { + registration.dispose(); + lease.release(); + } throw error; } - this.sessions.set(opts.sessionId, handle); - return handle; } /** @@ -248,7 +341,7 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec sessionDir, workDir, }); - await this.appendLogStore.flush(); + await this.appendLogStore.flush(''); } private async announceCreated(event: SessionCreatedEvent): Promise { @@ -259,14 +352,15 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec } get(sessionId: string): ISessionScopeHandle | undefined { - if (this.resuming.has(sessionId)) return undefined; - return this.sessions.get(sessionId); + const entry = this.entries.get(sessionId); + return entry?.state === 'active' ? entry.handle : undefined; } resume(sessionId: string): Promise { + if (this.closing) return Promise.reject(this.lifecycleClosingError()); const inflight = this.resuming.get(sessionId); if (inflight !== undefined) return inflight; - const live = this.sessions.get(sessionId); + const live = this.get(sessionId); if (live !== undefined) return Promise.resolve(live); const promise = this.doResume(sessionId) .catch((error: unknown) => { @@ -282,7 +376,7 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec } private async doResume(sessionId: string): Promise { - const live = this.sessions.get(sessionId); + const live = this.get(sessionId); if (live !== undefined) return live; const summary = await this.index.get(sessionId); @@ -292,51 +386,45 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec const workDir = summary.cwd ?? workspace?.root; if (workDir === undefined) return undefined; - const handle = await this.materializeSession({ + const entry = await this.materializeSession({ sessionId, workDir, workspaceId: summary.workspaceId, }); - const agents = handle.accessor.get(IAgentLifecycleService); - if (agents.get(MAIN_AGENT_ID) === undefined) { - await agents.create({ agentId: MAIN_AGENT_ID }); + const handle = entry.handle; + try { + const agents = handle.accessor.get(IAgentLifecycleService); + if (agents.get(MAIN_AGENT_ID) === undefined) { + await agents.create({ agentId: MAIN_AGENT_ID }); + } + this.activateSession(entry); + await this.announceCreated({ sessionId, handle, source: 'resume' }); + return handle; + } catch (error) { + this.rollbackSession(entry); + throw error; } - await this.announceCreated({ sessionId, handle, source: 'resume' }); - return handle; } list(): readonly ISessionScopeHandle[] { const ready: ISessionScopeHandle[] = []; - for (const [id, handle] of this.sessions) { - if (!this.resuming.has(id)) ready.push(handle); + for (const entry of this.entries.values()) { + if (entry.state === 'active') ready.push(entry.handle); } return ready; } async close(sessionId: string): Promise { - const handle = this.sessions.get(sessionId); - if (handle === undefined) return; - await this.announceWillClose({ sessionId, handle, reason: 'exit' }); - this.sessions.delete(sessionId); - await this.drainAgents(handle); - handle.dispose(); - this._onDidCloseSession.fire({ sessionId }); + await this.releaseSession(sessionId, 'close'); + } + + async closeAll(): Promise { + await this.beginClose(); + await Promise.allSettled([...this.entries].map(([sessionId]) => this.close(sessionId))); } async archive(sessionId: string): Promise { - const handle = this.sessions.get(sessionId); - if (handle === undefined) return; - const meta = handle.accessor.get(ISessionMetadata); - await meta.setArchived(true); - await this.drainAgents(handle); - this.event.publish({ - type: 'event.session.archived', - payload: { sessionId }, - }); - await this.announceWillClose({ sessionId, handle, reason: 'exit' }); - this.sessions.delete(sessionId); - handle.dispose(); - this._onDidArchiveSession.fire({ sessionId }); + await this.releaseSession(sessionId, 'archive'); } async restore(sessionId: string): Promise { @@ -350,6 +438,18 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec await this.hooks.onWillCloseSession.run(event); } + private async announceWillRelease(event: SessionWillReleaseEvent): Promise { + try { + await this.hooks.onWillReleaseSession.run(event); + } catch (error) { + this.log.warn('session release hook failed', { + sessionId: event.sessionId, + reason: event.reason, + error: String(error), + }); + } + } + private async drainAgents(handle: ISessionScopeHandle): Promise { const agentLifecycle = handle.accessor.get(IAgentLifecycleService); for (const agent of agentLifecycle.list()) { @@ -357,10 +457,210 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec } } + private releaseSession( + sessionId: string, + kind: SessionReleaseKind, + ): Promise { + const entry = this.entries.get(sessionId); + if (entry === undefined) return Promise.resolve(); + if (entry.state === 'opening') { + this.rollbackSession(entry); + return Promise.resolve(); + } + if (entry.releasePromise !== undefined) return entry.releasePromise; + entry.state = 'closing'; + const releasePromise = this.runSessionRelease(sessionId, entry, kind); + entry.releasePromise = releasePromise; + return releasePromise; + } + + private async runSessionRelease( + sessionId: string, + entry: SessionEntry, + kind: SessionReleaseKind, + ): Promise { + let stage: SessionReleaseStage = kind === 'archive' ? 'set-archived' : 'will-close'; + try { + if (kind === 'archive') { + await entry.handle.accessor.get(ISessionMetadata).setArchived(true); + stage = 'drain-agents'; + await this.drainAgents(entry.handle); + stage = 'publish-archive'; + this.event.publish({ + type: 'event.session.archived', + payload: { sessionId }, + }); + } else { + await this.announceWillClose({ sessionId, handle: entry.handle, reason: 'exit' }); + stage = 'drain-agents'; + await this.drainAgents(entry.handle); + } + + stage = 'will-close'; + if (kind === 'archive') { + await this.announceWillClose({ sessionId, handle: entry.handle, reason: 'exit' }); + } + stage = 'dispose-scope'; + this.disposeSessionHandle(entry); + stage = 'will-release'; + await this.announceWillRelease({ sessionId, reason: kind }); + stage = 'flush'; + await this.flushSessionTail(sessionId, entry.scope); + stage = 'drain-writes'; + await entry.lease.sealAndDrain(); + } catch (error) { + await this.abandonSession(entry, stage, error); + throw new Error2( + ErrorCodes.SESSION_DURABILITY_FAILED, + `session ${sessionId} was abandoned while releasing at ${stage}`, + { + details: { sessionId, stage }, + cause: error, + }, + ); + } + + this.finishSessionRelease(entry); + if (kind === 'archive') { + this._onDidArchiveSession.fire({ sessionId }); + } else { + this._onDidCloseSession.fire({ sessionId }); + } + } + + private async abandonSession( + entry: SessionEntry, + stage: SessionReleaseStage, + cause: unknown, + ): Promise { + const sessionId = entry.handle.id; + const reason = stage === 'flush' ? 'flush-failed' : 'release-failed'; + this.log.warn('abandoning session after release failed', { + sessionId, + stage, + error: String(cause), + }); + + const taskServices = this.collectTaskServices(entry); + try { + this.disposeSessionHandle(entry); + } catch { + } + + await this.writeDirtyMarker(entry, reason, stage); + await this.announceWillRelease({ sessionId, reason: 'dirty-abort' }); + await Promise.allSettled(taskServices.map((tasks) => tasks.flushPersistence())); + await entry.lease.sealAndDrain(); + this.finishSessionRelease(entry); + if (reason === 'flush-failed') { + this.telemetry.track2('session_dirty_abort', { + session_id: sessionId, + reason, + }); + } + } + + private async writeDirtyMarker( + entry: SessionEntry, + reason: 'flush-failed' | 'release-failed', + stage: SessionReleaseStage, + ): Promise { + try { + await this.docs.update(entry.scope, 'state.json', (current) => { + if (current === undefined) { + throw new Error2( + ErrorCodes.SESSION_DURABILITY_FAILED, + `session ${entry.handle.id} metadata is missing while writing dirty marker`, + { details: { sessionId: entry.handle.id } }, + ); + } + return { + ...current, + custom: { + ...current.custom, + dirtyAbort: { reason, stage, at: Date.now() }, + }, + }; + }); + } catch (error) { + this.log.warn('failed to persist session dirty marker', { + sessionId: entry.handle.id, + error: String(error), + }); + } + } + + private collectTaskServices(entry: SessionEntry): IAgentTaskService[] { + try { + return entry.handle + .accessor.get(IAgentLifecycleService) + .list() + .map((agent) => agent.accessor.get(IAgentTaskService)); + } catch { + return []; + } + } + + private finishSessionRelease(entry: SessionEntry): void { + try { + entry.registration.dispose(); + } catch (error) { + this.log.warn('failed to unregister session write admission', { + sessionId: entry.handle.id, + error: String(error), + }); + } + try { + entry.lease.release(); + } catch (error) { + this.log.warn('failed to release session lease', { + sessionId: entry.handle.id, + error: String(error), + }); + } + if (this.entries.get(entry.handle.id) === entry) this.entries.delete(entry.handle.id); + } + + private activateSession(entry: SessionEntry): void { + if (this.entries.get(entry.handle.id) !== entry || entry.state !== 'opening') { + throw new Error2( + ErrorCodes.SESSION_LEASE_LOST, + `session ${entry.handle.id} was torn down before activation`, + { details: { sessionId: entry.handle.id } }, + ); + } + entry.lease.assertWritable(); + entry.state = 'active'; + } + + private rollbackSession(entry: SessionEntry): void { + if (this.entries.get(entry.handle.id) === entry) this.entries.delete(entry.handle.id); + try { + this.disposeSessionHandle(entry); + } catch { + } + try { + entry.registration.dispose(); + } catch { + } + entry.lease.release(); + } + + private disposeSessionHandle(entry: SessionEntry): void { + if (entry.disposed) return; + entry.handle.dispose(); + entry.disposed = true; + } + async fork(opts: ForkSessionOptions): Promise { + this.assertOpen(); + return this.trackOperation(this.doFork(opts)); + } + + private async doFork(opts: ForkSessionOptions): Promise { const sourceId = opts.sourceSessionId; - const sourceHandle = this.sessions.get(sourceId); + const sourceHandle = this.get(sourceId); const indexSummary = await this.index.get(sourceId); if (sourceHandle === undefined && indexSummary === undefined) { throw new Error2(ErrorCodes.SESSION_NOT_FOUND, `session ${sourceId} does not exist`); @@ -378,6 +678,7 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec // quiesce: the only requirement is a durable copy point, which // `copyAgentWire`'s flush provides. let targetId: string | undefined; + let targetEntry: SessionEntry | undefined; let target: ISessionScopeHandle | undefined; let targetSessionDir: string | undefined; try { @@ -392,7 +693,7 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec : await this.readMetaFromDisk(workspaceId, sourceId); targetId = opts.newSessionId ?? createSessionId(); - if (this.sessions.has(targetId) || (await this.index.get(targetId)) !== undefined) { + if (this.entries.has(targetId) || (await this.index.get(targetId)) !== undefined) { throw new Error2( ErrorCodes.SESSION_ALREADY_EXISTS, `Session "${targetId}" already exists`, @@ -405,10 +706,11 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec targetSessionDir, ); - target = await this.materializeSession({ + targetEntry = await this.materializeSession({ sessionId: targetId, workDir: workspace.root, }); + target = targetEntry.handle; const targetCtx = target.accessor.get(ISessionContext); const targetMeta = target.accessor.get(ISessionMetadata); @@ -447,6 +749,7 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec } await this.appendSessionIndexEntry(targetId, workspace.root, targetCtx.workspaceId); + this.activateSession(targetEntry); this._onDidForkSession.fire({ sourceSessionId: sourceId, sessionId: targetId, @@ -455,15 +758,7 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec await this.announceCreated({ sessionId: targetId, handle: target, source: 'fork' }); return target; } catch (error) { - if (targetId !== undefined) { - this.sessions.delete(targetId); - } - if (target !== undefined) { - try { - target.dispose(); - } catch { - } - } + if (targetEntry !== undefined) this.rollbackSession(targetEntry); if (targetSessionDir !== undefined) { await this.hostFs.remove(targetSessionDir).catch(() => {}); } @@ -472,6 +767,7 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec } async createChild(opts: CreateChildSessionOptions): Promise { + this.assertOpen(); const title = opts.title ?? `Child: ${(await this.resolveSourceTitle(opts.sourceSessionId)) ?? opts.sourceSessionId}`; @@ -489,7 +785,7 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec } private async resolveSourceTitle(sourceId: string): Promise { - const live = this.sessions.get(sourceId); + const live = this.get(sourceId); if (live !== undefined) { return (await live.accessor.get(ISessionMetadata).read()).title; } @@ -601,6 +897,118 @@ export class SessionLifecycleService extends Disposable implements ISessionLifec } } + override dispose(): void { + this.closing = true; + for (const entry of this.entries.values()) { + this.startAbandon(entry, 'shutdown', new Error('session lifecycle disposed')); + } + super.dispose(); + } + + private assertOpen(): void { + if (this.closing) throw this.lifecycleClosingError(); + } + + private lifecycleClosingError(): Error2 { + return new Error2(ErrorCodes.SESSION_CLOSED, 'session lifecycle is closing'); + } + + private trackOperation(promise: Promise): Promise { + this.inFlightOperations.add(promise); + const remove = (): void => { + this.inFlightOperations.delete(promise); + }; + promise.then(remove, remove); + return promise; + } + + private onLeaseLost(sessionId: string): void { + this.log.error('session lease lost; tearing the session down', { sessionId }); + const entry = this.entries.get(sessionId); + if (entry !== undefined) { + this.startAbandon( + entry, + 'lease-lost', + new Error2(ErrorCodes.SESSION_LEASE_LOST, `session ${sessionId} lost its write lease`, { + details: { sessionId }, + }), + ); + } + } + + private startAbandon( + entry: SessionEntry, + stage: SessionReleaseStage, + cause: unknown, + ): void { + if (entry.releasePromise !== undefined) return; + entry.state = 'closing'; + const releasePromise = this.abandonSession(entry, stage, cause); + entry.releasePromise = releasePromise; + void releasePromise.catch((error) => { + this.log.error('unexpected failure while abandoning session', { + sessionId: entry.handle.id, + error: String(error), + }); + }); + } + + private async acquireSessionLease(sessionId: string): Promise { + const leasePath = sessionLeasePath(this.bootstrap.homeDir, sessionId); + const contact = this.leaseContact.contact(); + try { + const handle = await this.locks.acquire(leasePath, { + address: contact.type === 'address' ? contact.address : undefined, + }); + const lease = new SessionLease(sessionId, handle, (id) => { + this.onLeaseLost(id); + }); + this.telemetry.track2('session_lease_acquired', { session_id: sessionId }); + return lease; + } catch (error) { + if ( + isError2(error) && + (error.code === OsLockErrors.codes.OS_LOCK_HELD || + error.code === OsLockErrors.codes.OS_LOCK_WAIT_TIMEOUT) + ) { + this.throwHeldByPeer(sessionId, error); + } + throw error; + } + } + + private throwHeldByPeer(sessionId: string, cause: unknown): never { + const inspection = this.locks.inspect(sessionLeasePath(this.bootstrap.homeDir, sessionId)); + const details = this.heldByPeerDetails(inspection); + this.telemetry.track2('session_held_by_peer_returned', { + session_id: sessionId, + phase: details.phase, + }); + throw new Error2( + ErrorCodes.SESSION_HELD_BY_PEER, + `session ${sessionId} is held by another instance (${details.phase})`, + { details, cause }, + ); + } + + private heldByPeerDetails(inspection: CrossProcessLockInspection): HeldByPeerDetails { + // A free lease here means the holder vanished between the failed acquire + // and this probe; that race converges by retrying, same as 'creating'. + return heldByPeerDetailsFromInspection(inspection) ?? HELD_BY_PEER_CREATING_DETAILS; + } + + private async flushSessionTail(sessionId: string, scope: string): Promise { + try { + await this.appendLogStore.flush(scope); + } catch (error) { + this.log.warn('final journal flush failed while closing session', { + sessionId, + error: String(error), + }); + throw error; + } + } + private async readMetaFromDisk( workspaceId: string, sessionId: string, diff --git a/packages/agent-core-v2/src/app/skillCatalog/skillRootWatcher.ts b/packages/agent-core-v2/src/app/skillCatalog/skillRootWatcher.ts new file mode 100644 index 0000000000..8aae0467cd --- /dev/null +++ b/packages/agent-core-v2/src/app/skillCatalog/skillRootWatcher.ts @@ -0,0 +1,180 @@ +/** + * `skillCatalog` domain (L3) — filesystem watcher for skill-root directories. + * + * Watches candidate skill roots through `IHostFsWatchService`, probes them + * through `IHostFileSystem`, and fires a 300 ms debounced change callback + * whenever any of them changes. Candidates may not exist yet (skill + * directories are opt-in). chokidar 4 (verified + * 4.0.3 on darwin): a recursive watch on a path whose immediate parent + * exists picks the path up when it is created, but a path with two or more + * missing leading segments reports NOTHING when the chain appears. So an + * absent root is tracked by a depth-0 sentinel on the nearest existing + * ancestor that re-anchors down the chain as segments appear — sentinel + * events only trigger an existence re-probe (a `mkdir -p` chain is never + * missed) — and once the root exists a recursive watch is armed in its + * place. Root deletion while armed falls back to sentinel mode on the next + * advance, so delete/recreate cycles stay live. Pure helper owned by the + * file-backed skill sources; not a DI service. + */ + +import { dirname } from 'pathe'; + +import { Disposable } from '#/_base/di/lifecycle'; +import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import type { HostFsChange, IHostFsWatchHandle, IHostFsWatchService } from '#/os/interface/hostFsWatch'; + +import { isDir } from './skillRoots'; + +const SKILL_WATCH_DEBOUNCE_MS = 300; + +interface RootWatchState { + readonly root: string; + rootWatch: IHostFsWatchHandle | undefined; + sentinel: IHostFsWatchHandle | undefined; + sentinelDir: string | undefined; + advanceTail: Promise; +} + +export class SkillRootWatcher extends Disposable { + private readonly states = new Map(); + private armTail: Promise = Promise.resolve(); + private disposed = false; + private debounceTimer: ReturnType | undefined; + + readonly ready: Promise; + + constructor( + private readonly hostFs: IHostFileSystem, + private readonly hostFsWatch: IHostFsWatchService, + private readonly resolveRoots: () => Promise, + private readonly onDidChange: () => void, + ) { + super(); + this.ready = this.rearm(); + } + + refresh(): Promise { + return this.rearm(); + } + + override dispose(): void { + if (this.disposed) return; + this.disposed = true; + if (this.debounceTimer !== undefined) { + clearTimeout(this.debounceTimer); + this.debounceTimer = undefined; + } + for (const state of this.states.values()) this.teardownState(state); + this.states.clear(); + super.dispose(); + } + + private rearm(): Promise { + const tail = this.armTail.then(() => this.rearmNow()); + this.armTail = tail.catch(() => undefined); + return tail; + } + + private async rearmNow(): Promise { + if (this.disposed) return; + for (const state of this.states.values()) this.teardownState(state); + this.states.clear(); + const roots = await this.resolveRoots(); + if (this.disposed) return; + const advances: Promise[] = []; + for (const root of new Set(roots)) { + const state: RootWatchState = { + root, + rootWatch: undefined, + sentinel: undefined, + sentinelDir: undefined, + advanceTail: Promise.resolve(), + }; + this.states.set(root, state); + this.advance(state); + advances.push(state.advanceTail); + } + await Promise.all(advances); + } + + private teardownState(state: RootWatchState): void { + state.rootWatch?.dispose(); + state.rootWatch = undefined; + state.sentinel?.dispose(); + state.sentinel = undefined; + state.sentinelDir = undefined; + } + + private advance(state: RootWatchState): void { + const tail = state.advanceTail.then(async () => { + if (this.disposed || this.states.get(state.root) !== state) return; + if (await isDir(this.hostFs, state.root)) { + if (state.rootWatch !== undefined) return; + // A previously armed sentinel means the root just appeared (possibly + // with content already inside): the transition itself is a change. + const appeared = state.sentinel !== undefined; + state.sentinel?.dispose(); + state.sentinel = undefined; + state.sentinelDir = undefined; + if (this.disposed || this.states.get(state.root) !== state) return; + const handle = this.hostFsWatch.watch(state.root); + state.rootWatch = handle; + handle.onDidChange(() => { + this.scheduleFire(); + }); + if (appeared) this.scheduleFire(); + return; + } + state.rootWatch?.dispose(); + state.rootWatch = undefined; + const anchor = await nearestExistingDir(this.hostFs, state.root); + if (this.disposed || this.states.get(state.root) !== state) return; + if (state.sentinel !== undefined && state.sentinelDir === anchor) return; + state.sentinel?.dispose(); + const sentinel = this.hostFsWatch.watch(anchor, { recursive: false }); + state.sentinel = sentinel; + state.sentinelDir = anchor; + sentinel.onDidChange((event) => { + this.onSentinelEvent(state, event); + }); + }); + state.advanceTail = tail.catch(() => undefined); + } + + private onSentinelEvent(state: RootWatchState, event: HostFsChange): void { + if (isOnRootChain(state.root, event.path)) this.advance(state); + } + + private scheduleFire(): void { + if (this.disposed) return; + if (this.debounceTimer !== undefined) clearTimeout(this.debounceTimer); + const timer = setTimeout(() => { + this.debounceTimer = undefined; + if (this.disposed) return; + this.onDidChange(); + // Re-probe every root: a deleted armed root falls back to sentinel mode + // here, and a missed sentinel transition is re-armed on the new root. + for (const state of this.states.values()) this.advance(state); + }, SKILL_WATCH_DEBOUNCE_MS); + timer.unref?.(); + this.debounceTimer = timer; + } +} + +function isOnRootChain(root: string, eventPath: string): boolean { + if (eventPath === root) return true; + return ( + root.startsWith(eventPath) && + (root[eventPath.length] === '/' || root[eventPath.length] === '\\') + ); +} + +async function nearestExistingDir(fs: IHostFileSystem, root: string): Promise { + let current = root; + while (true) { + if (await isDir(fs, current)) return current; + const parent = dirname(current); + if (parent === current) return current; + current = parent; + } +} diff --git a/packages/agent-core-v2/src/app/skillCatalog/skillRoots.ts b/packages/agent-core-v2/src/app/skillCatalog/skillRoots.ts index c0a1c76330..3ad408a3a4 100644 --- a/packages/agent-core-v2/src/app/skillCatalog/skillRoots.ts +++ b/packages/agent-core-v2/src/app/skillCatalog/skillRoots.ts @@ -5,14 +5,18 @@ * user (home) and project (workspace) skill locations. Brand directories are * preferred over generic ones (`.kimi-code/skills` before `.agents/skills`), * and the project root is found by walking up to `.git`. Plugin roots are no - * longer folded in here — plugins are a separate `ISkillSource`. These helpers - * are exported so the edge can compose a workspace's skills without a Session. - * Pure path/fs probes; no scoped state. + * longer folded in here — plugins are a separate `ISkillSource`. The + * `*Candidates` helpers return the same locations WITHOUT the existence filter + * or realpath resolution, for file watchers that must observe roots appearing + * later. These helpers are exported so the edge can compose a workspace's + * skills without a Session. Pure path probes through `IHostFileSystem`; no + * scoped state. */ -import { promises as fs } from 'node:fs'; import path from 'pathe'; +import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; + import type { SkillRoot, SkillSource } from './types'; const USER_BRAND_DIRS = ['skills'] as const; @@ -25,48 +29,86 @@ export interface SkillRootsOptions { } export async function userRoots( + fs: IHostFileSystem, homeDir: string, osHomeDir: string, options: SkillRootsOptions = {}, ): Promise { const roots: SkillRoot[] = []; const mergeAllAvailableSkills = options.mergeAllAvailableSkills ?? true; - await pushBrandGroup(roots, USER_BRAND_DIRS, homeDir, 'user', mergeAllAvailableSkills); - await pushFirstExisting(roots, USER_GENERIC_DIRS, osHomeDir, 'user'); + await pushBrandGroup(fs, roots, USER_BRAND_DIRS, homeDir, 'user', mergeAllAvailableSkills); + await pushFirstExisting(fs, roots, USER_GENERIC_DIRS, osHomeDir, 'user'); return roots; } export async function projectRoots( + fs: IHostFileSystem, workDir: string, options: SkillRootsOptions = {}, ): Promise { - const projectRoot = await findProjectRoot(workDir); + const projectRoot = await findProjectRoot(fs, workDir); const roots: SkillRoot[] = []; const mergeAllAvailableSkills = options.mergeAllAvailableSkills ?? true; - await pushBrandGroup(roots, PROJECT_BRAND_DIRS, projectRoot, 'project', mergeAllAvailableSkills); - await pushFirstExisting(roots, PROJECT_GENERIC_DIRS, projectRoot, 'project'); + await pushBrandGroup( + fs, + roots, + PROJECT_BRAND_DIRS, + projectRoot, + 'project', + mergeAllAvailableSkills, + ); + await pushFirstExisting(fs, roots, PROJECT_GENERIC_DIRS, projectRoot, 'project'); return roots; } export async function configuredRoots( + fs: IHostFileSystem, dirs: readonly string[], workDir: string, osHomeDir: string, source: SkillSource, ): Promise { - const projectRoot = await findProjectRoot(workDir); + const projectRoot = await findProjectRoot(fs, workDir); const roots: SkillRoot[] = []; for (const dir of dirs) { - await pushExistingRoot(roots, resolveConfiguredDir(dir, projectRoot, osHomeDir), source); + await pushExistingRoot(fs, roots, resolveConfiguredDir(dir, projectRoot, osHomeDir), source); } return roots; } -async function findProjectRoot(workDir: string): Promise { +export function userRootCandidates(homeDir: string, osHomeDir: string): readonly string[] { + return [ + ...USER_BRAND_DIRS.map((dir) => path.join(homeDir, dir)), + ...USER_GENERIC_DIRS.map((dir) => path.join(osHomeDir, dir)), + ]; +} + +export async function projectRootCandidates( + fs: IHostFileSystem, + workDir: string, +): Promise { + const projectRoot = await findProjectRoot(fs, workDir); + return [ + ...PROJECT_BRAND_DIRS.map((dir) => path.join(projectRoot, dir)), + ...PROJECT_GENERIC_DIRS.map((dir) => path.join(projectRoot, dir)), + ]; +} + +export async function configuredRootCandidates( + fs: IHostFileSystem, + dirs: readonly string[], + workDir: string, + osHomeDir: string, +): Promise { + const projectRoot = await findProjectRoot(fs, workDir); + return dirs.map((dir) => resolveConfiguredDir(dir, projectRoot, osHomeDir)); +} + +async function findProjectRoot(fs: IHostFileSystem, workDir: string): Promise { const start = path.resolve(workDir); let current = start; while (true) { - if (await exists(path.join(current, '.git'))) return current; + if (await exists(fs, path.join(current, '.git'))) return current; const parent = path.dirname(current); if (parent === current) return start; current = parent; @@ -74,17 +116,19 @@ async function findProjectRoot(workDir: string): Promise { } async function pushFirstExisting( + fs: IHostFileSystem, out: SkillRoot[], dirs: readonly string[], base: string, source: SkillSource, ): Promise { for (const dir of dirs) { - if (await pushExistingRoot(out, path.join(base, dir), source)) return; + if (await pushExistingRoot(fs, out, path.join(base, dir), source)) return; } } async function pushBrandGroup( + fs: IHostFileSystem, out: SkillRoot[], dirs: readonly string[], base: string, @@ -92,21 +136,22 @@ async function pushBrandGroup( mergeAllAvailableSkills: boolean, ): Promise { if (!mergeAllAvailableSkills) { - await pushFirstExisting(out, dirs, base, source); + await pushFirstExisting(fs, out, dirs, base, source); return; } for (const dir of dirs) { - await pushExistingRoot(out, path.join(base, dir), source); + await pushExistingRoot(fs, out, path.join(base, dir), source); } } async function pushExistingRoot( + fs: IHostFileSystem, out: SkillRoot[], dir: string, source: SkillSource, ): Promise { - if (!(await isDir(dir))) return false; - const resolved = await realpath(dir); + if (!(await isDir(fs, dir))) return false; + const resolved = await realpath(fs, dir); if (!out.some((root) => root.path === resolved)) out.push({ path: resolved, source }); return true; } @@ -118,19 +163,19 @@ function resolveConfiguredDir(dir: string, projectRoot: string, osHomeDir: strin return path.resolve(projectRoot, dir); } -async function isDir(p: string): Promise { +export async function isDir(fs: IHostFileSystem, p: string): Promise { try { - return (await fs.stat(p)).isDirectory(); + return (await fs.stat(p)).isDirectory; } catch { return false; } } -async function realpath(p: string): Promise { +async function realpath(fs: IHostFileSystem, p: string): Promise { return (await fs.realpath(p)).replaceAll('\\', '/'); } -async function exists(p: string): Promise { +async function exists(fs: IHostFileSystem, p: string): Promise { try { await fs.stat(p); return true; diff --git a/packages/agent-core-v2/src/app/skillCatalog/userFileSkillSource.ts b/packages/agent-core-v2/src/app/skillCatalog/userFileSkillSource.ts index fc7842a52b..a5f4a13cc2 100644 --- a/packages/agent-core-v2/src/app/skillCatalog/userFileSkillSource.ts +++ b/packages/agent-core-v2/src/app/skillCatalog/userFileSkillSource.ts @@ -3,7 +3,10 @@ * * Discovers user skills from the bootstrap home directories through * `ISkillDiscovery`, contributing them at priority 20 (above extra / plugin / - * builtin, below workspace). Reads home paths from `bootstrap`. Bound at App scope. + * builtin, below workspace). Reads home paths from `bootstrap` and hot-reloads + * on both its config section and filesystem changes in the user skill roots, + * probing them through `hostFs` and watching them through `hostFsWatch` via + * `SkillRootWatcher`. Bound at App scope. */ import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; @@ -13,6 +16,8 @@ import { Emitter, type Event } from '#/_base/event'; import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; import { IBootstrapService } from '#/app/bootstrap/bootstrap'; import { IConfigService } from '#/app/config/config'; +import { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { IHostFsWatchService } from '#/os/interface/hostFsWatch'; import { MERGE_ALL_AVAILABLE_SKILLS_SECTION, @@ -20,7 +25,8 @@ import { } from './configSection'; import { ISkillCatalogRuntimeOptions } from './skillCatalogRuntimeOptions'; import { ISkillDiscovery } from './skillDiscovery'; -import { userRoots } from './skillRoots'; +import { SkillRootWatcher } from './skillRootWatcher'; +import { userRootCandidates, userRoots } from './skillRoots'; import { SKILL_SOURCE_PRIORITY, type ISkillSource, type SkillContribution } from './skillSource'; export interface IUserFileSkillSource extends ISkillSource { @@ -43,6 +49,8 @@ export class UserFileSkillSource extends Disposable implements IUserFileSkillSou @IBootstrapService private readonly bootstrap: IBootstrapService, @IConfigService private readonly config: IConfigService, @ISkillCatalogRuntimeOptions private readonly runtimeOptions: ISkillCatalogRuntimeOptions, + @IHostFileSystem private readonly hostFs: IHostFileSystem, + @IHostFsWatchService hostFsWatch: IHostFsWatchService, ) { super(); this._register( @@ -50,6 +58,16 @@ export class UserFileSkillSource extends Disposable implements IUserFileSkillSou if (event.domain === MERGE_ALL_AVAILABLE_SKILLS_SECTION) this.onDidChangeEmitter.fire(); }), ); + if ((this.runtimeOptions.explicitDirs?.length ?? 0) === 0) { + this._register( + new SkillRootWatcher( + this.hostFs, + hostFsWatch, + async () => userRootCandidates(this.bootstrap.homeDir, this.bootstrap.osHomeDir), + () => this.onDidChangeEmitter.fire(), + ), + ); + } } async load(): Promise { @@ -60,7 +78,9 @@ export class UserFileSkillSource extends Disposable implements IUserFileSkillSou const mergeAllAvailableSkills = this.config.get(MERGE_ALL_AVAILABLE_SKILLS_SECTION) ?? true; return this.discovery.discover( - await userRoots(this.bootstrap.homeDir, this.bootstrap.osHomeDir, { mergeAllAvailableSkills }), + await userRoots(this.hostFs, this.bootstrap.homeDir, this.bootstrap.osHomeDir, { + mergeAllAvailableSkills, + }), ); } } diff --git a/packages/agent-core-v2/src/app/telemetry/events.ts b/packages/agent-core-v2/src/app/telemetry/events.ts index 8c41cf5147..b94aa142d9 100644 --- a/packages/agent-core-v2/src/app/telemetry/events.ts +++ b/packages/agent-core-v2/src/app/telemetry/events.ts @@ -435,6 +435,20 @@ export interface SessionLoadFailedEvent { reason: string; } +export interface SessionLeaseAcquiredEvent { + session_id: string; +} + +export interface SessionHeldByPeerReturnedEvent { + session_id: string; + phase: 'creating' | 'routable' | 'held-by-local-instance'; +} + +export interface SessionDirtyAbortEvent { + session_id: string; + reason: 'flush-failed'; +} + export interface FirstLaunchEvent {} export interface ExitEvent { @@ -924,6 +938,27 @@ export const telemetryEventDefinitions = { comment: 'A session resume fails.', properties: { reason: 'Error code, error name, or unknown' }, }), + session_lease_acquired: defineTelemetryEvent({ + owner: 'kimi-code', + comment: "This instance takes a session's write lease.", + properties: { session_id: 'Session the lease covers' }, + }), + session_held_by_peer_returned: defineTelemetryEvent({ + owner: 'kimi-code', + comment: 'A session materialization is refused because a peer instance holds the lease.', + properties: { + session_id: 'Session that was refused', + phase: 'Ownership phase reported to the client (routable, creating, …)', + }, + }), + session_dirty_abort: defineTelemetryEvent({ + owner: 'kimi-code', + comment: 'A session is force-aborted after its final durability barrier became ambiguous.', + properties: { + session_id: 'Session whose lease is being released after an ambiguous flush failure', + reason: 'Why the dirty abort was required', + }, + }), first_launch: defineTelemetryEvent({ owner: 'kimi-code', comment: 'The CLI runs for the first time on this device.', diff --git a/packages/agent-core-v2/src/app/workspaceRegistry/fileWorkspacePersistence.ts b/packages/agent-core-v2/src/app/workspaceRegistry/fileWorkspacePersistence.ts index 7f04827d06..2cce2473db 100644 --- a/packages/agent-core-v2/src/app/workspaceRegistry/fileWorkspacePersistence.ts +++ b/packages/agent-core-v2/src/app/workspaceRegistry/fileWorkspacePersistence.ts @@ -1,13 +1,15 @@ /** - * `workspaceRegistry` domain (L1) — `FileWorkspacePersistence` implementation. + * `workspaceRegistry` domain (L2) — `FileWorkspacePersistence` implementation. * * File backend of `IWorkspacePersistence`. Persists the catalog as a single * v1-compatible `workspaces.json` document at the storage root * (`/workspaces.json`, via `scope = ''`) through the * `IAtomicDocumentStore` access-pattern Store. The `deleted_workspace_ids` * tombstone list round-trips with the catalog so soft deletions survive - * regardless of which engine (v1 or v2) last wrote the file. Bound at App - * scope. + * regardless of which engine (v1 or v2) last wrote the file, and the parsed + * document rides along in `WorkspaceCatalog.raw` so `save` re-applies the + * semantic view onto it — unknown top-level and entry fields written by other + * engine versions are preserved. Bound at App scope. */ import { InstantiationType } from '#/_base/di/extensions'; @@ -18,7 +20,6 @@ import type { Workspace } from './workspaceRegistry'; import { IWorkspacePersistence, type PersistedWorkspaceEntry, - type PersistedWorkspaceFile, type WorkspaceCatalog, } from './workspacePersistence'; @@ -31,23 +32,25 @@ export class FileWorkspacePersistence implements IWorkspacePersistence { constructor(@IAtomicDocumentStore private readonly docs: IAtomicDocumentStore) {} + runExclusive(op: () => Promise): Promise { + return this.docs.withExclusiveKeyMutation( + WORKSPACE_REGISTRY_SCOPE, + WORKSPACE_REGISTRY_KEY, + op, + ); + } + async load(): Promise { - const file = await this.docs.get( + const file = await this.docs.get>( WORKSPACE_REGISTRY_SCOPE, WORKSPACE_REGISTRY_KEY, ); - if (file === undefined) return undefined; - if ( - typeof file !== 'object' || - file === null || - typeof (file as { workspaces?: unknown }).workspaces !== 'object' || - (file as { workspaces?: unknown }).workspaces === null - ) { - return undefined; - } + if (!isRecord(file)) return undefined; + const rawWorkspaces = file['workspaces']; + if (!isRecord(rawWorkspaces)) return undefined; const now = Date.now(); const workspaces: Workspace[] = []; - for (const [id, raw] of Object.entries(file.workspaces)) { + for (const [id, raw] of Object.entries(rawWorkspaces)) { const entry = sanitizeEntry(raw); if (entry === null) continue; workspaces.push({ @@ -58,24 +61,29 @@ export class FileWorkspacePersistence implements IWorkspacePersistence { lastOpenedAt: parseTime(entry.last_opened_at, now), }); } - const rawDeleted = (file as { deleted_workspace_ids?: unknown }).deleted_workspace_ids; + const rawDeleted = file['deleted_workspace_ids']; const deletedIds = Array.isArray(rawDeleted) ? rawDeleted.filter((id): id is string => typeof id === 'string') : []; - return { workspaces, deletedIds }; + return { workspaces, deletedIds, raw: file }; } async save(catalog: WorkspaceCatalog): Promise { - const record: Record = {}; + const rawWorkspaces = catalog.raw['workspaces']; + const previousWorkspaces = isRecord(rawWorkspaces) ? rawWorkspaces : {}; + const record: Record = {}; for (const ws of catalog.workspaces) { + const previous = previousWorkspaces[ws.id]; record[ws.id] = { + ...(isPlainRecord(previous) ? previous : {}), root: ws.root, name: ws.name, created_at: new Date(ws.createdAt).toISOString(), last_opened_at: new Date(ws.lastOpenedAt).toISOString(), }; } - const file: PersistedWorkspaceFile = { + const file = { + ...catalog.raw, version: WORKSPACE_REGISTRY_VERSION, workspaces: record, deleted_workspace_ids: [...catalog.deletedIds], @@ -84,6 +92,14 @@ export class FileWorkspacePersistence implements IWorkspacePersistence { } } +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function isPlainRecord(value: unknown): value is Record { + return isRecord(value) && !Array.isArray(value); +} + function sanitizeEntry(value: unknown): PersistedWorkspaceEntry | null { if (typeof value !== 'object' || value === null) return null; const v = value as Partial; diff --git a/packages/agent-core-v2/src/app/workspaceRegistry/workspacePersistence.ts b/packages/agent-core-v2/src/app/workspaceRegistry/workspacePersistence.ts index 46ae19c201..88910f236f 100644 --- a/packages/agent-core-v2/src/app/workspaceRegistry/workspacePersistence.ts +++ b/packages/agent-core-v2/src/app/workspaceRegistry/workspacePersistence.ts @@ -1,5 +1,5 @@ /** - * `workspaceRegistry` domain (L1) — `IWorkspacePersistence` contract. + * `workspaceRegistry` domain (L2) — `IWorkspacePersistence` contract. * * Domain-specific persistence Store for the known-workspaces catalog. It hides * the on-disk document layout (`/workspaces.json`, the v1-compatible @@ -17,6 +17,11 @@ * `load()` returns `undefined` to mean "no usable catalog" so the registry can * trigger a one-shot rebuild from the legacy session index; an empty catalog * is a valid, already-materialized state and must NOT trigger a rebuild. + * + * `WorkspaceCatalog.raw` carries the opaque document the catalog was loaded + * from; `save` re-applies the semantic view onto it so unknown top-level and + * entry fields written by other engine versions survive the round-trip under + * the shared file's read-modify-write contract. */ import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; @@ -30,20 +35,19 @@ export interface PersistedWorkspaceEntry { readonly last_opened_at: string; } -export interface PersistedWorkspaceFile { - readonly version: number; - readonly workspaces: Record; - readonly deleted_workspace_ids: string[]; -} - export interface WorkspaceCatalog { readonly workspaces: readonly Workspace[]; readonly deletedIds: readonly string[]; + /** Opaque snapshot of the document this catalog was loaded from (empty when + the file was absent or unusable). save() re-applies the semantic view + onto it, preserving fields this engine does not know. */ + readonly raw: Readonly>; } export interface IWorkspacePersistence { readonly _serviceBrand: undefined; + runExclusive(op: () => Promise): Promise; load(): Promise; save(catalog: WorkspaceCatalog): Promise; } diff --git a/packages/agent-core-v2/src/app/workspaceRegistry/workspaceRegistryService.ts b/packages/agent-core-v2/src/app/workspaceRegistry/workspaceRegistryService.ts index 27ba9458d7..917b0e89d9 100644 --- a/packages/agent-core-v2/src/app/workspaceRegistry/workspaceRegistryService.ts +++ b/packages/agent-core-v2/src/app/workspaceRegistry/workspaceRegistryService.ts @@ -1,18 +1,24 @@ /** - * `workspaceRegistry` domain (L1) — `IWorkspaceRegistry` implementation. + * `workspaceRegistry` domain (L2) — `IWorkspaceRegistry` implementation. * * Process-wide catalog of known workspaces, durable in * `/workspaces.json` (the v1-compatible file shared with * agent-core). The service keeps NO in-memory write cache: every operation - * is a fresh read-modify-write against the file, serialized through a - * promise-chain mutex. This is required, not just tidy — the same file is - * written concurrently by other processes (the v1 TUI registers session cwds - * via `touchWorkspaceRegistry`, which also re-reads the file on every call), - * so a write-through cache would clobber external additions and tombstones - * with stale state. Atomic renames at the persistence layer plus fresh - * read-modify-write on both engines shrink the lost-update window to a - * single read-modify-write, and the next session-index merge heals anything - * still lost there. + * is a fresh read-modify-write against the file. This is required, not just + * tidy — the same file is written concurrently by other processes (the v1 + * TUI registers session cwds via `touchWorkspaceRegistry`, which also + * re-reads the file on every call), so a write-through cache would clobber + * external additions and tombstones with stale state. + * + * Two exclusion layers make each read-modify-write safe: the in-process promise + * chain serializes ops (entered first, so cross-process waiting stays minimal), + * and the persistence store's transaction primitive locks the physical + * `workspaces.json` path, making the load → mutate → save burst atomic against + * other lock-aware v2 processes. + * Unregistered writers (v1) stay best-effort: their lost updates are healed + * by the session-index merge. Tombstone logic, format and key names are + * unchanged, and unknown document fields round-trip verbatim via + * `WorkspaceCatalog.raw`. * * Once per process, the first operation triggers the startup sync with the * legacy `/session_index.jsonl`: @@ -206,7 +212,11 @@ export class WorkspaceRegistryService implements IWorkspaceRegistry { byId.set(ws.id, ws); // An explicit add clears any prior deletion tombstone. deletedIds.delete(ws.id); - await this.store.save({ workspaces: [...byId.values()], deletedIds: [...deletedIds] }); + await this.store.save({ + workspaces: [...byId.values()], + deletedIds: [...deletedIds], + raw: catalog.raw, + }); return ws; }); } @@ -224,6 +234,7 @@ export class WorkspaceRegistryService implements IWorkspaceRegistry { await this.store.save({ workspaces: catalog.workspaces.map((ws) => (ws.id === id ? updated : ws)), deletedIds: catalog.deletedIds, + raw: catalog.raw, }); return updated; }); @@ -250,6 +261,7 @@ export class WorkspaceRegistryService implements IWorkspaceRegistry { await this.store.save({ workspaces: catalog.workspaces.filter((ws) => ws.id !== id), deletedIds: [...new Set([...catalog.deletedIds, id])], + raw: catalog.raw, }); return; } @@ -258,6 +270,7 @@ export class WorkspaceRegistryService implements IWorkspaceRegistry { await this.store.save({ workspaces: catalog.workspaces.filter((ws) => workspaceRootKey(ws.root) !== rootKey), deletedIds: [...new Set([...catalog.deletedIds, ...aliasIds])], + raw: catalog.raw, }); }); } @@ -270,14 +283,18 @@ export class WorkspaceRegistryService implements IWorkspaceRegistry { const loaded = await this.store.load(); if (loaded === undefined) { const rebuilt = await this.rebuildFromSessionIndex(); - await this.store.save({ workspaces: [...rebuilt.values()], deletedIds: [] }); + await this.store.save({ workspaces: [...rebuilt.values()], deletedIds: [], raw: {} }); this.merged = true; return; } const byId = new Map(loaded.workspaces.map((ws) => [ws.id, ws])); const deletedIds = new Set(loaded.deletedIds); if (await this.mergeFromSessionIndex(byId, deletedIds)) { - await this.store.save({ workspaces: [...byId.values()], deletedIds: [...deletedIds] }); + await this.store.save({ + workspaces: [...byId.values()], + deletedIds: [...deletedIds], + raw: loaded.raw, + }); } this.merged = true; } @@ -285,7 +302,7 @@ export class WorkspaceRegistryService implements IWorkspaceRegistry { /** Read the current catalog; a missing or malformed file is an empty * catalog (mirrors v1's tolerant read). */ private async loadCatalog(): Promise { - return (await this.store.load()) ?? { workspaces: [], deletedIds: [] }; + return (await this.store.load()) ?? { workspaces: [], deletedIds: [], raw: {} }; } /** Add every distinct workDir from the legacy session index that the @@ -371,7 +388,8 @@ export class WorkspaceRegistryService implements IWorkspaceRegistry { } private runExclusive(op: () => Promise): Promise { - const next = this.opQueue.then(op, op); + const locked = (): Promise => this.store.runExclusive(op); + const next = this.opQueue.then(locked, locked); this.opQueue = next.then( () => {}, () => {}, diff --git a/packages/agent-core-v2/src/errors.ts b/packages/agent-core-v2/src/errors.ts index b5b34d6d3c..122b59c432 100644 --- a/packages/agent-core-v2/src/errors.ts +++ b/packages/agent-core-v2/src/errors.ts @@ -22,6 +22,7 @@ import { McpErrors } from '#/agent/mcp/errors'; import { MessageLegacyErrors } from '#/app/messageLegacy/errors'; import { ModelCatalogErrors } from '#/kosong/model/errors'; import { OsFsErrors } from '#/os/interface/hostFsErrors'; +import { OsLockErrors } from '#/os/interface/crossProcessLock'; import { OsProcessErrors } from '#/os/interface/hostProcess'; import { PluginErrors } from '#/app/plugin/errors'; import { ProfileErrors } from '#/agent/profile/errors'; @@ -54,6 +55,7 @@ export { McpErrors } from '#/agent/mcp/errors'; export { MessageLegacyErrors } from '#/app/messageLegacy/errors'; export { ModelCatalogErrors } from '#/kosong/model/errors'; export { OsFsErrors } from '#/os/interface/hostFsErrors'; +export { OsLockErrors } from '#/os/interface/crossProcessLock'; export { OsProcessErrors } from '#/os/interface/hostProcess'; export { PluginErrors } from '#/app/plugin/errors'; export { ProfileErrors } from '#/agent/profile/errors'; @@ -83,6 +85,7 @@ export const ErrorCodes = { ...MessageLegacyErrors.codes, ...ModelCatalogErrors.codes, ...OsFsErrors.codes, + ...OsLockErrors.codes, ...OsProcessErrors.codes, ...PluginErrors.codes, ...ProfileErrors.codes, diff --git a/packages/agent-core-v2/src/index.ts b/packages/agent-core-v2/src/index.ts index 219c55a51c..b7be503b4a 100644 --- a/packages/agent-core-v2/src/index.ts +++ b/packages/agent-core-v2/src/index.ts @@ -33,12 +33,14 @@ export * from '#/app/telemetry/consoleAppender'; export * from '#/app/telemetry/cloudAppender'; export * from '#/app/bootstrap/bootstrap'; export * from '#/app/bootstrap/bootstrapService'; +export * from '#/os/interface/crossProcessLock'; export * from '#/os/interface/hostEnvironment'; export * from '#/os/interface/hostFileSystem'; export * from '#/os/interface/hostFsWatch'; export * from '#/os/interface/hostProcess'; export * from '#/os/interface/terminal'; export * from '#/os/interface/terminalErrors'; +export * from '#/os/backends/node-local/crossProcessLockService'; export * from '#/os/backends/node-local/hostEnvironmentService'; export * from '#/os/backends/node-local/hostFsService'; export * from '#/os/backends/node-local/hostFsWatchService'; @@ -79,6 +81,8 @@ export * from '#/app/sessionIndex/sessionIndex'; export * from '#/app/sessionIndex/sessionIndexService'; export * from '#/session/sessionMetadata/sessionMetadata'; export * from '#/session/sessionMetadata/sessionMetadataService'; +export * from '#/session/sessionLease/sessionLease'; +export * from '#/session/sessionLease/sessionLeaseContactProvider'; export * from '#/session/sessionToolPolicy/sessionToolPolicy'; export * from '#/session/sessionToolPolicy/sessionToolPolicyService'; export * from '#/app/config/config'; @@ -166,6 +170,7 @@ export * from '#/app/skillCatalog/skillDiscovery'; export * from '#/app/skillCatalog/inMemorySkillDiscovery'; export * from '#/app/skillCatalog/skillSource'; export * from '#/app/skillCatalog/skillRoots'; +export * from '#/app/skillCatalog/skillRootWatcher'; export * from '#/app/skillCatalog/builtin/builtin'; export * from '#/app/skillCatalog/builtinSkillSource'; export * from '#/app/skillCatalog/userFileSkillSource'; @@ -182,6 +187,8 @@ export * from '#/session/sessionAgentProfileCatalog/extraFileAgentSource'; export * from '#/session/sessionAgentProfileCatalog/explicitFileAgentSource'; export * from '#/agent/permissionGate/permissionGate'; export * from '#/agent/permissionGate/permissionGateService'; +export * from '#/agent/fileFencing/fileFencing'; +export * from '#/agent/fileFencing/fileFencingService'; import '#/app/flag/flag'; import '#/app/flag/flagRegistry'; import '#/app/flag/flagRegistryService'; @@ -314,6 +321,8 @@ export * from '#/session/sessionFs/fsWatchService'; export * from '#/session/sessionFs/gitContext'; export * from '#/session/sessionFs/rgLocator'; export * from '#/session/sessionFs/runRg'; +export * from '#/session/sessionFileLedger/fileLedger'; +export * from '#/session/sessionFileLedger/fileLedgerService'; export * from '#/app/hostFolderBrowser/hostFolderBrowser'; export * from '#/app/hostFolderBrowser/hostFolderBrowserService'; export * from '#/persistence/interface/storage'; @@ -321,8 +330,11 @@ export * from '#/persistence/interface/appendLogStore'; export * from '#/persistence/interface/atomicDocumentStore'; export * from '#/persistence/interface/queryStore'; export * from '#/persistence/interface/blobStore'; +export * from '#/persistence/interface/sessionWriteAdmission'; +export * from '#/persistence/interface/storageWriteAdmission'; export * from '#/persistence/backends/node-fs/fileStorageService'; export * from '#/persistence/backends/node-fs/appendLogStore'; +export * from '#/persistence/backends/node-fs/storageWriteAdmissionService'; export * from '#/persistence/backends/node-fs/atomicDocumentStore'; export * from '#/persistence/backends/node-fs/blobStoreService'; export * from '#/persistence/backends/node-fs/workspaceLocalConfigService'; @@ -443,6 +455,8 @@ export * from '#/agent/permissionRules/matchesRule'; export * from '#/agent/permissionRules/permissionRulesService'; export * from '#/agent/profile/profile'; export * from '#/agent/profile/profileService'; +export * from '#/agent/profile/agentSkillListingReminder'; +export * from '#/agent/profile/agentSkillListingReminderService'; export * from '#/agent/profile/context'; export * from '#/agent/prompt/prompt'; export * from '#/agent/prompt/promptService'; diff --git a/packages/agent-core-v2/src/os/backends/node-local/crossProcessLockService.ts b/packages/agent-core-v2/src/os/backends/node-local/crossProcessLockService.ts new file mode 100644 index 0000000000..2cf4dd8f49 --- /dev/null +++ b/packages/agent-core-v2/src/os/backends/node-local/crossProcessLockService.ts @@ -0,0 +1,292 @@ +/** + * `crossProcessLock` domain (L1) — `ICrossProcessLockService` implementation. + * + * Uses `kernel-file-lock` to hold an operating-system advisory lock on a + * permanent sentinel file. Owner metadata is stored in a sibling JSON document + * for diagnostics and routing only; it is not part of the exclusion protocol. + * Bound at App scope. + */ + +import { randomUUID } from 'node:crypto'; +import { readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'; + +import { + type KernelFileLockHandle, + tryAcquireKernelFileLock, +} from '@moonshot-ai/kernel-file-lock'; +import { ulid } from 'ulid'; + +import { InstantiationType } from '#/_base/di/extensions'; +import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; +import { + CrossProcessLockError, + CrossProcessLockErrorCode, + type CrossProcessLockAcquireOptions, + type CrossProcessLockInspection, + type CrossProcessLockPayload, + type CrossProcessLockServiceDeps, + type CrossProcessLockWaitOptions, + type ICrossProcessLockHandle, + ICrossProcessLockService, +} from '#/os/interface/crossProcessLock'; + +const DEFAULT_WAIT_RETRY_INTERVAL_MS = 50; + +interface DiskLockPayload { + lock_id?: string; + instance_id?: string; + pid?: number; + address?: string; +} + +function readErrno(error: unknown): string | undefined { + if (error === null || typeof error !== 'object' || !('code' in error)) return undefined; + const code = (error as { code: unknown }).code; + return typeof code === 'string' ? code : undefined; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function ownerPath(lockPath: string): string { + return `${lockPath}.owner.json`; +} + +function toDiskPayload(payload: CrossProcessLockPayload): DiskLockPayload { + return { + lock_id: payload.lockId, + instance_id: payload.instanceId, + pid: payload.pid, + address: payload.address, + }; +} + +function fromDiskPayload(payload: DiskLockPayload): CrossProcessLockPayload | undefined { + if ( + typeof payload.lock_id !== 'string' || + typeof payload.instance_id !== 'string' || + typeof payload.pid !== 'number' + ) { + return undefined; + } + return { + lockId: payload.lock_id, + instanceId: payload.instance_id, + pid: payload.pid, + address: typeof payload.address === 'string' ? payload.address : undefined, + }; +} + +function readPayload(lockPath: string): CrossProcessLockPayload | undefined { + try { + const parsed: unknown = JSON.parse(readFileSync(ownerPath(lockPath), 'utf8')); + return parsed !== null && typeof parsed === 'object' + ? fromDiskPayload(parsed as DiskLockPayload) + : undefined; + } catch (error) { + if (readErrno(error) === 'ENOENT' || error instanceof SyntaxError) return undefined; + throw error; + } +} + +function writePayload(lockPath: string, payload: CrossProcessLockPayload): void { + const path = ownerPath(lockPath); + const tempPath = `${path}.${process.pid}.${randomUUID()}.tmp`; + try { + writeFileSync(tempPath, JSON.stringify(toDiskPayload(payload)), { mode: 0o600 }); + renameSync(tempPath, path); + } catch (error) { + rmSync(tempPath, { force: true }); + throw error; + } +} + +function toLockIoError(error: unknown, path: string, op: string): CrossProcessLockError { + if (error instanceof CrossProcessLockError) return error; + return new CrossProcessLockError( + CrossProcessLockErrorCode.Io, + `${op} failed on lock ${path}: ${errorMessage(error)}`, + { details: { path, op, errno: readErrno(error) }, cause: error }, + ); +} + +function heldError( + lockPath: string, + inspection: CrossProcessLockInspection, +): CrossProcessLockError { + return new CrossProcessLockError( + CrossProcessLockErrorCode.Held, + `cross-process lock unavailable (${inspection.state})`, + { details: { path: lockPath, reason: inspection.state, holder: inspection.payload } }, + ); +} + +function waitTimeoutError( + lockPath: string, + timeoutMs: number, + cause: unknown, +): CrossProcessLockError { + return new CrossProcessLockError( + CrossProcessLockErrorCode.WaitTimeout, + `timed out waiting for the cross-process lock (${timeoutMs}ms)`, + { details: { path: lockPath, timeoutMs }, cause }, + ); +} + +class CrossProcessLockHandle implements ICrossProcessLockHandle { + private released = false; + + constructor( + readonly lockPath: string, + readonly lockId: string, + private readonly kernelHandle: KernelFileLockHandle, + ) {} + + checkHeld(): boolean { + return !this.released && this.kernelHandle.checkHeld(); + } + + release(): void { + if (this.released) return; + this.released = true; + try { + if (this.kernelHandle.checkHeld()) { + try { + rmSync(ownerPath(this.lockPath), { force: true }); + } catch {} + } + } finally { + this.kernelHandle.release(); + } + } +} + +export class CrossProcessLockService implements ICrossProcessLockService { + declare readonly _serviceBrand: undefined; + + private readonly selfPid: number; + private readonly now: () => number; + private readonly newLockId: () => string; + private readonly instanceId: string; + private readonly sleep: (ms: number) => Promise; + + constructor(deps: CrossProcessLockServiceDeps = {}) { + this.now = deps.now ?? Date.now; + this.selfPid = deps.selfPid ?? process.pid; + this.newLockId = deps.newLockId ?? ulid; + this.instanceId = deps.instanceId ?? ulid(); + this.sleep = deps.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); + } + + async acquire( + lockPath: string, + options: CrossProcessLockAcquireOptions = {}, + ): Promise { + let kernelHandle: KernelFileLockHandle | undefined; + try { + kernelHandle = tryAcquireKernelFileLock(lockPath); + } catch (error) { + throw toLockIoError(error, lockPath, 'acquire'); + } + if (kernelHandle === undefined) throw heldError(lockPath, this.inspectHeld(lockPath)); + + const lockId = this.newLockId(); + const payload: CrossProcessLockPayload = { + lockId, + instanceId: this.instanceId, + pid: this.selfPid, + address: options.address, + }; + try { + rmSync(ownerPath(lockPath), { force: true }); + writePayload(lockPath, payload); + return new CrossProcessLockHandle(lockPath, lockId, kernelHandle); + } catch (error) { + kernelHandle.release(); + throw toLockIoError(error, lockPath, 'write-owner'); + } + } + + private async acquireWithWait( + lockPath: string, + options: CrossProcessLockAcquireOptions & { wait: CrossProcessLockWaitOptions }, + ): Promise { + const deadline = this.now() + options.wait.timeoutMs; + const retryIntervalMs = options.wait.retryIntervalMs ?? DEFAULT_WAIT_RETRY_INTERVAL_MS; + let firstAttempt = true; + let lastHeldError: CrossProcessLockError | undefined; + for (;;) { + const isFirstAttempt = firstAttempt; + if (!isFirstAttempt && this.now() >= deadline) { + throw waitTimeoutError(lockPath, options.wait.timeoutMs, lastHeldError); + } + firstAttempt = false; + try { + const handle = await this.acquire(lockPath, options); + if (!isFirstAttempt && this.now() >= deadline) { + handle.release(); + throw waitTimeoutError(lockPath, options.wait.timeoutMs, lastHeldError); + } + return handle; + } catch (error) { + if ( + !(error instanceof CrossProcessLockError) || + error.code !== CrossProcessLockErrorCode.Held + ) { + throw error; + } + lastHeldError = error; + const remainingMs = deadline - this.now(); + if (remainingMs <= 0) { + throw waitTimeoutError(lockPath, options.wait.timeoutMs, error); + } + await this.sleep(Math.min(retryIntervalMs, remainingMs)); + } + } + } + + async withLock( + lockPath: string, + options: CrossProcessLockAcquireOptions & { wait: CrossProcessLockWaitOptions }, + fn: (handle: ICrossProcessLockHandle) => T | Promise, + ): Promise { + const handle = await this.acquireWithWait(lockPath, options); + try { + return await fn(handle); + } finally { + handle.release(); + } + } + + inspect(lockPath: string): CrossProcessLockInspection { + let probe: KernelFileLockHandle | undefined; + try { + probe = tryAcquireKernelFileLock(lockPath); + } catch (error) { + throw toLockIoError(error, lockPath, 'inspect'); + } + if (probe !== undefined) { + probe.release(); + return { state: 'free' }; + } + return this.inspectHeld(lockPath); + } + + private inspectHeld(lockPath: string): CrossProcessLockInspection { + try { + const payload = readPayload(lockPath); + return payload === undefined ? { state: 'creating' } : { state: 'held', payload }; + } catch (error) { + throw toLockIoError(error, lockPath, 'read-owner'); + } + } +} + +registerScopedService( + LifecycleScope.App, + ICrossProcessLockService, + CrossProcessLockService, + InstantiationType.Eager, + 'crossProcessLock', +); diff --git a/packages/agent-core-v2/src/os/backends/node-local/hostFsService.ts b/packages/agent-core-v2/src/os/backends/node-local/hostFsService.ts index b806fa1e56..4c1795c56c 100644 --- a/packages/agent-core-v2/src/os/backends/node-local/hostFsService.ts +++ b/packages/agent-core-v2/src/os/backends/node-local/hostFsService.ts @@ -6,7 +6,6 @@ */ import { - appendFile, lstat, open, readFile, @@ -27,6 +26,17 @@ import { toHostFsError } from '#/os/interface/hostFsErrors'; const READ_CHUNK_SIZE = 64 * 1024; +function toHostFileStat(stat: Awaited>): HostFileStat { + return { + isFile: stat.isFile(), + isDirectory: stat.isDirectory(), + isSymbolicLink: stat.isSymbolicLink(), + size: Number(stat.size), + mtimeMs: Number(stat.mtimeMs), + ino: Number(stat.ino), + }; +} + function isUtf8Encoding(encoding: BufferEncoding): boolean { return encoding === 'utf-8' || encoding === 'utf8'; } @@ -64,19 +74,29 @@ export class HostFileSystem implements IHostFileSystem { } } - async writeText(path: string, data: string): Promise { + async writeText(path: string, data: string): Promise { + let handle: Awaited> | undefined; try { - await writeFile(path, data, 'utf8'); + handle = await open(path, 'w'); + await handle.writeFile(data, 'utf8'); + return toHostFileStat(await handle.stat()); } catch (error) { throw toHostFsError(error, { path, op: 'write' }); + } finally { + await handle?.close(); } } - async appendText(path: string, data: string): Promise { + async appendText(path: string, data: string): Promise { + let handle: Awaited> | undefined; try { - await appendFile(path, data, 'utf8'); + handle = await open(path, 'a'); + await handle.writeFile(data, 'utf8'); + return toHostFileStat(await handle.stat()); } catch (error) { throw toHostFsError(error, { path, op: 'append' }); + } finally { + await handle?.close(); } } @@ -109,7 +129,11 @@ export class HostFileSystem implements IHostFileSystem { async *readLines( path: string, - options?: { encoding?: BufferEncoding; errors?: TextDecodeErrors }, + options?: { + encoding?: BufferEncoding; + errors?: TextDecodeErrors; + onFileStat?: (stat: HostFileStat) => void; + }, ): AsyncGenerator { try { const encoding = options?.encoding ?? 'utf-8'; @@ -118,10 +142,11 @@ export class HostFileSystem implements IHostFileSystem { if (!isUtf8Encoding(encoding)) { const content = decodeTextWithErrors(await readFile(path), encoding, errors); yield* splitLinesKeepingTerminator(content); + options?.onFileStat?.(await this.stat(path)); return; } - yield* this._readUtf8Lines(path, errors); + yield* this._readUtf8Lines(path, errors, options?.onFileStat); } catch (error) { throw toHostFsError(error, { path, op: 'read' }); } @@ -130,6 +155,7 @@ export class HostFileSystem implements IHostFileSystem { private async *_readUtf8Lines( path: string, errors: TextDecodeErrors, + onFileStat?: (stat: HostFileStat) => void, ): AsyncGenerator { const fh = await open(path, 'r'); try { @@ -168,7 +194,11 @@ export class HostFileSystem implements IHostFileSystem { yield decodeTextWithErrors(line, 'utf-8', errors, pendingOffset !== 0); } } finally { - await fh.close(); + try { + onFileStat?.(toHostFileStat(await fh.stat())); + } finally { + await fh.close(); + } } } @@ -190,15 +220,7 @@ export class HostFileSystem implements IHostFileSystem { async stat(path: string): Promise { try { - const s = await nodeStat(path); - return { - isFile: s.isFile(), - isDirectory: s.isDirectory(), - isSymbolicLink: s.isSymbolicLink(), - size: s.size, - mtimeMs: s.mtimeMs, - ino: s.ino, - }; + return toHostFileStat(await nodeStat(path)); } catch (error) { throw toHostFsError(error, { path, op: 'stat' }); } @@ -206,15 +228,7 @@ export class HostFileSystem implements IHostFileSystem { async lstat(path: string): Promise { try { - const s = await lstat(path); - return { - isFile: s.isFile(), - isDirectory: s.isDirectory(), - isSymbolicLink: s.isSymbolicLink(), - size: s.size, - mtimeMs: s.mtimeMs, - ino: s.ino, - }; + return toHostFileStat(await lstat(path)); } catch (error) { throw toHostFsError(error, { path, op: 'lstat' }); } diff --git a/packages/agent-core-v2/src/os/backends/node-local/hostFsWatchService.ts b/packages/agent-core-v2/src/os/backends/node-local/hostFsWatchService.ts index 1014d690d6..0bed5e7164 100644 --- a/packages/agent-core-v2/src/os/backends/node-local/hostFsWatchService.ts +++ b/packages/agent-core-v2/src/os/backends/node-local/hostFsWatchService.ts @@ -12,6 +12,7 @@ import { Emitter, type Event } from '#/_base/event'; import { InstantiationType } from '#/_base/di/extensions'; import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; import { onUnexpectedError } from '#/_base/errors/unexpectedError'; +import { isSpecialFileStat } from '#/_base/utils/fs'; import { type HostFsChange, @@ -25,13 +26,22 @@ import { const DEFAULT_IGNORED = (p: string): boolean => /(?:^|[/\\])\.git(?:$|[/\\])/.test(p); class HostFsWatchHandle implements IHostFsWatchHandle { + readonly ready: Promise; readonly onDidChange: Event; private readonly emitter: Emitter; private readonly watcher: FSWatcher; private disposed = false; + private readySettled = false; + private resolveReady!: () => void; + private rejectReady!: (error: unknown) => void; constructor(path: string, options: HostFsWatchOptions | undefined) { + this.ready = new Promise((resolve, reject) => { + this.resolveReady = resolve; + this.rejectReady = reject; + }); + void this.ready.catch(() => undefined); this.emitter = new Emitter(); this.onDidChange = this.emitter.event; this.watcher = new FSWatcher({ @@ -39,21 +49,36 @@ class HostFsWatchHandle implements IHostFsWatchHandle { persistent: false, followSymlinks: false, depth: options?.recursive === false ? 0 : undefined, - ignored: options?.ignored ?? DEFAULT_IGNORED, + ignored: (path, stats) => + isSpecialFileStat(stats) || (options?.ignored?.(path) ?? DEFAULT_IGNORED(path)), }); this.watcher.on('all', (eventName: string, absPath: string) => { + if (this.disposed) return; const mapped = mapChokidarEvent(eventName, absPath); if (mapped !== undefined) this.emitter.fire(mapped); }); this.watcher.on('error', (error: unknown) => { + if (!this.readySettled) { + this.readySettled = true; + this.rejectReady(error); + } onUnexpectedError(error); }); + this.watcher.on('ready', () => { + if (this.readySettled) return; + this.readySettled = true; + this.resolveReady(); + }); this.watcher.add(path); } dispose(): void { if (this.disposed) return; this.disposed = true; + if (!this.readySettled) { + this.readySettled = true; + this.resolveReady(); + } void this.watcher.close().catch(() => undefined); this.emitter.dispose(); } diff --git a/packages/agent-core-v2/src/os/backends/node-local/tools/read.ts b/packages/agent-core-v2/src/os/backends/node-local/tools/read.ts index 95220a9e5e..b007704002 100644 --- a/packages/agent-core-v2/src/os/backends/node-local/tools/read.ts +++ b/packages/agent-core-v2/src/os/backends/node-local/tools/read.ts @@ -31,9 +31,11 @@ import { unwrapErrorCause } from '#/_base/errors/errors'; import { ISessionSkillCatalog } from '#/session/sessionSkillCatalog/skillCatalog'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; import { + attachToolFileRevision, ToolAccesses, type BuiltinTool, type ExecutableToolResult, + makeToolFileRevision, type ToolExecution, } from '#/tool/toolContract'; import { registerTool } from '#/agent/toolRegistry/toolContribution'; @@ -46,6 +48,7 @@ import { MEDIA_SNIFF_BYTES, detectFileType } from '#/agent/media/file-type'; import { toInputJsonSchema } from '#/tool/input-schema'; import { literalRulePattern, matchesPathRuleSubject } from '#/tool/rule-match'; import { makeCarriageReturnsVisible, type LineEndingStyle } from '#/_base/text/line-endings'; +import { fileStatTuplesEqual } from '#/_base/utils/fs'; import { renderPrompt } from '#/_base/utils/render-prompt'; import readDescriptionTemplate from './read.md?raw'; @@ -309,22 +312,43 @@ export class ReadTool implements BuiltinTool { const lineOffset = args.line_offset ?? 1; const requestedLines = args.n_lines ?? MAX_LINES; const effectiveLimit = Math.min(requestedLines, MAX_LINES); + let observedStat: Awaited> | undefined; + const onFileStat = (value: Awaited>): void => { + observedStat = value; + }; - if (lineOffset < 0) { - return await this.readTail( - safePath, - args.path, - lineOffset, - effectiveLimit, - requestedLines, - ); + const result = + lineOffset < 0 + ? await this.readTail( + safePath, + args.path, + lineOffset, + effectiveLimit, + requestedLines, + onFileStat, + ) + : await this.readForward( + safePath, + args.path, + lineOffset, + effectiveLimit, + requestedLines, + onFileStat, + ); + if (result.isError === true) return result; + observedStat ??= await this.fs.stat(safePath); + if (!fileStatTuplesEqual(stat, observedStat)) { + return { + isError: true, + output: `"${args.path}" changed while it was being read. Retry the read.`, + }; + } + if (args.line_offset !== undefined || args.n_lines !== undefined) { + return result; } - return await this.readForward( - safePath, - args.path, - lineOffset, - effectiveLimit, - requestedLines, + return attachToolFileRevision( + result, + makeToolFileRevision(safePath, observedStat), ); } catch (error) { if (isTextDecodeError(error)) { @@ -343,6 +367,7 @@ export class ReadTool implements BuiltinTool { lineOffset: number, effectiveLimit: number, requestedLines: number, + onFileStat: (stat: Awaited>) => void, ): Promise { const selectedEntries: ReadLineEntry[] = []; const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; @@ -350,7 +375,7 @@ export class ReadTool implements BuiltinTool { let maxLinesReached = false; let collectionClosed = false; - for await (const rawLine of this.fs.readLines(safePath, { errors: 'strict' })) { + for await (const rawLine of this.fs.readLines(safePath, { errors: 'strict', onFileStat })) { if (containsNulByte(rawLine)) { return { isError: true, output: notReadableFileOutput(displayPath) }; } @@ -400,13 +425,14 @@ export class ReadTool implements BuiltinTool { lineOffset: number, effectiveLimit: number, requestedLines: number, + onFileStat: (stat: Awaited>) => void, ): Promise { const tailCount = Math.abs(lineOffset); const entries: ReadLineEntry[] = []; const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; let currentLineNo = 0; - for await (const rawLine of this.fs.readLines(safePath, { errors: 'strict' })) { + for await (const rawLine of this.fs.readLines(safePath, { errors: 'strict', onFileStat })) { if (containsNulByte(rawLine)) { return { isError: true, output: notReadableFileOutput(displayPath) }; } diff --git a/packages/agent-core-v2/src/os/backends/node-local/tools/write.ts b/packages/agent-core-v2/src/os/backends/node-local/tools/write.ts index 821cfb80b4..8a0449443d 100644 --- a/packages/agent-core-v2/src/os/backends/node-local/tools/write.ts +++ b/packages/agent-core-v2/src/os/backends/node-local/tools/write.ts @@ -26,9 +26,11 @@ import { unwrapErrorCause } from '#/_base/errors/errors'; import { ISessionSkillCatalog } from '#/session/sessionSkillCatalog/skillCatalog'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; import { + attachToolFileRevision, ToolAccesses, type BuiltinTool, type ExecutableToolResult, + makeToolFileRevision, type ToolExecution, } from '#/tool/toolContract'; import { registerTool } from '#/agent/toolRegistry/toolContribution'; @@ -119,15 +121,17 @@ export class WriteTool implements BuiltinTool { try { const mode = args.mode ?? 'overwrite'; - if (mode === 'append') { - await this.fs.appendText(safePath, args.content); - } else { - await this.fs.writeText(safePath, args.content); - } + const stat = + mode === 'append' + ? await this.fs.appendText(safePath, args.content) + : await this.fs.writeText(safePath, args.content); const bytesWritten = Buffer.byteLength(args.content, 'utf8'); - return { - output: `${mode === 'append' ? 'Appended' : 'Wrote'} ${String(bytesWritten)} bytes to ${args.path}`, - }; + return attachToolFileRevision( + { + output: `${mode === 'append' ? 'Appended' : 'Wrote'} ${String(bytesWritten)} bytes to ${args.path}`, + }, + makeToolFileRevision(safePath, stat), + ); } catch (error) { const code = (unwrapErrorCause(error) as { code?: unknown } | null)?.code; if (code === 'ENOENT') { diff --git a/packages/agent-core-v2/src/os/interface/crossProcessLock.ts b/packages/agent-core-v2/src/os/interface/crossProcessLock.ts new file mode 100644 index 0000000000..c2c7f2970d --- /dev/null +++ b/packages/agent-core-v2/src/os/interface/crossProcessLock.ts @@ -0,0 +1,113 @@ +/** + * `crossProcessLock` domain (L1) — cross-process exclusive lock contract. + * + * Defines the App-scoped lock service used by durable read-modify-write + * transactions and long-lived session leases. The lock path is a permanent + * sentinel whose inode is never removed or replaced; ownership is enforced by + * the operating system through an exclusive advisory lock held on an open file + * descriptor. A separate owner document carries operator-facing metadata and + * is never consulted for lock correctness. + */ + +import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; +import { registerErrorDomain, type ErrorDomain } from '#/_base/errors/codes'; +import { Error2, type Error2Options } from '#/_base/errors/errors'; + +export interface CrossProcessLockPayload { + lockId: string; + instanceId: string; + pid: number; + address?: string; +} + +export interface CrossProcessLockWaitOptions { + readonly timeoutMs: number; + readonly retryIntervalMs?: number; +} + +export interface CrossProcessLockAcquireOptions { + readonly address?: string; +} + +export interface CrossProcessLockInspection { + readonly state: 'free' | 'creating' | 'held'; + readonly payload?: CrossProcessLockPayload; +} + +export interface ICrossProcessLockHandle { + readonly lockPath: string; + readonly lockId: string; + checkHeld(): boolean; + release(): void; +} + +export interface ICrossProcessLockService { + readonly _serviceBrand: undefined; + + acquire( + lockPath: string, + options?: CrossProcessLockAcquireOptions, + ): Promise; + + withLock( + lockPath: string, + options: CrossProcessLockAcquireOptions & { wait: CrossProcessLockWaitOptions }, + fn: (handle: ICrossProcessLockHandle) => T | Promise, + ): Promise; + + inspect(lockPath: string): CrossProcessLockInspection; +} + +export const ICrossProcessLockService: ServiceIdentifier = + createDecorator('crossProcessLockService'); + +export interface CrossProcessLockServiceDeps { + readonly now?: () => number; + readonly selfPid?: number; + readonly newLockId?: () => string; + readonly instanceId?: string; + readonly sleep?: (ms: number) => Promise; +} + +export const OsLockErrors = { + codes: { + OS_LOCK_HELD: 'os.lock.held', + OS_LOCK_WAIT_TIMEOUT: 'os.lock.wait_timeout', + OS_LOCK_IO: 'os.lock.io', + }, + info: { + 'os.lock.held': { + title: 'Lock is held by another process', + retryable: false, + public: true, + }, + 'os.lock.wait_timeout': { + title: 'Timed out waiting for a cross-process lock', + retryable: true, + public: true, + }, + 'os.lock.io': { + title: 'Lock file I/O failed', + retryable: true, + public: false, + }, + }, +} as const satisfies ErrorDomain; + +registerErrorDomain(OsLockErrors); + +export const CrossProcessLockErrorCode = { + Held: OsLockErrors.codes.OS_LOCK_HELD, + WaitTimeout: OsLockErrors.codes.OS_LOCK_WAIT_TIMEOUT, + Io: OsLockErrors.codes.OS_LOCK_IO, +} as const; + +export type CrossProcessLockErrorCode = + (typeof CrossProcessLockErrorCode)[keyof typeof CrossProcessLockErrorCode]; + +export class CrossProcessLockError extends Error2 { + constructor(code: CrossProcessLockErrorCode, message: string, options?: Error2Options) { + super(code, message, options); + this.name = 'CrossProcessLockError'; + } +} diff --git a/packages/agent-core-v2/src/os/interface/hostFileSystem.ts b/packages/agent-core-v2/src/os/interface/hostFileSystem.ts index 0e35262ddf..6b4bb3bf54 100644 --- a/packages/agent-core-v2/src/os/interface/hostFileSystem.ts +++ b/packages/agent-core-v2/src/os/interface/hostFileSystem.ts @@ -35,13 +35,17 @@ export interface IHostFileSystem { path: string, options?: { encoding?: BufferEncoding; errors?: TextDecodeErrors }, ): Promise; - writeText(path: string, data: string): Promise; - appendText(path: string, data: string): Promise; + writeText(path: string, data: string): Promise; + appendText(path: string, data: string): Promise; readBytes(path: string, n?: number): Promise; writeBytes(path: string, data: Uint8Array): Promise; readLines( path: string, - options?: { encoding?: BufferEncoding; errors?: TextDecodeErrors }, + options?: { + encoding?: BufferEncoding; + errors?: TextDecodeErrors; + onFileStat?: (stat: HostFileStat) => void; + }, ): AsyncGenerator; createExclusive(path: string, data: Uint8Array): Promise; /** Follows symlinks to the target (Node `stat` semantics). Use {@link lstat} when the link itself matters. */ diff --git a/packages/agent-core-v2/src/os/interface/hostFsWatch.ts b/packages/agent-core-v2/src/os/interface/hostFsWatch.ts index 890ce98c61..fff8589149 100644 --- a/packages/agent-core-v2/src/os/interface/hostFsWatch.ts +++ b/packages/agent-core-v2/src/os/interface/hostFsWatch.ts @@ -28,6 +28,7 @@ export interface HostFsWatchOptions { } export interface IHostFsWatchHandle extends IDisposable { + readonly ready: Promise; readonly onDidChange: Event; } diff --git a/packages/agent-core-v2/src/persistence/backends/memory/inMemoryStorageService.ts b/packages/agent-core-v2/src/persistence/backends/memory/inMemoryStorageService.ts index cf5f716c0e..9e80581d84 100644 --- a/packages/agent-core-v2/src/persistence/backends/memory/inMemoryStorageService.ts +++ b/packages/agent-core-v2/src/persistence/backends/memory/inMemoryStorageService.ts @@ -20,6 +20,7 @@ import { type IDisposable, } from '#/_base/di/lifecycle'; import { Emitter, type Event } from '#/_base/event'; +import { enqueueKeyedOperation } from '#/_base/utils/promise'; import { IFileSystemStorageService, @@ -38,6 +39,7 @@ export class InMemoryStorageService implements IFileSystemStorageService { private readonly scopes = new Map>(); private readonly watchers = new Map(); + private readonly operationQueues = new Map>(); async read(scope: string, key: string): Promise { return this.scopes.get(scope)?.get(key); @@ -131,6 +133,14 @@ export class InMemoryStorageService implements IFileSystemStorageService { }; } + withExclusiveKeyMutation( + scope: string, + key: string, + mutation: () => Promise, + ): Promise { + return enqueueKeyedOperation(this.operationQueues, this.watchKey(scope, key), mutation); + } + private notifyWatchers(scope: string, key: string): void { this.watchers.get(this.watchKey(scope, key))?.emitter.fire(); } diff --git a/packages/agent-core-v2/src/persistence/backends/node-fs/appendLogStore.ts b/packages/agent-core-v2/src/persistence/backends/node-fs/appendLogStore.ts index 074b98798e..66ab6ae897 100644 --- a/packages/agent-core-v2/src/persistence/backends/node-fs/appendLogStore.ts +++ b/packages/agent-core-v2/src/persistence/backends/node-fs/appendLogStore.ts @@ -9,9 +9,11 @@ * Serializes whole-log rewrites with live appends, preserves queued or * in-flight records across the atomic replacement, keeps ambiguous append and * rewrite failures sticky, keeps the shared flush pending until the - * post-rewrite drain is durable, waits every key before a global flush reports - * an error, and preserves per-key storage ordering while acquired buffers - * retire and hand off to replacement owners. Bound at App scope. + * post-rewrite drain is durable, supports scoped durability barriers, waits + * every selected key before a flush reports an error, and preserves failed + * retired buffers so their in-memory tail remains observable while replacement + * owners wait for the prior generation. Physical writes are admitted and + * tracked by the underlying byte-storage backend. Bound at App scope. */ import { InstantiationType } from '#/_base/di/extensions'; @@ -119,11 +121,14 @@ export class AppendLogStore implements IAppendLogStore { await this.ownFlush(scope, key, state, rewrite); } - async flush(): Promise { - const inFlight = [...this.logs.entries()].map(([id, state]) => { + async flush(scopePrefix?: string): Promise { + const inFlight: Promise[] = []; + for (const [id, state] of this.logs) { const { scope, key } = fromLogId(id); - return this.flushState(scope, key, state); - }); + if (!matchesScope(scope, scopePrefix)) continue; + inFlight.push(this.flushState(scope, key, state)); + if (state.retirement !== undefined) inFlight.push(state.retirement); + } const settled = await Promise.allSettled(inFlight); for (const result of settled) { if (result.status === 'rejected') throw result.reason; @@ -187,16 +192,14 @@ export class AppendLogStore implements IAppendLogStore { state.refCount--; if (state.refCount > 0) return; state.retired = true; - state.retirement = this.settleRetiredState(scope, key, state).catch(() => undefined); + state.retirement = this.settleRetiredState(scope, key, state); + void state.retirement.catch((error) => state.onError?.(error)); } private async settleRetiredState(scope: string, key: string, state: LogState): Promise { - try { - await this.flushState(scope, key, state); - } finally { - const id = logId(scope, key); - if (this.logs.get(id) === state) this.logs.delete(id); - } + await this.flushState(scope, key, state); + const id = logId(scope, key); + if (this.logs.get(id) === state) this.logs.delete(id); } private ownFlush( @@ -257,6 +260,7 @@ export class AppendLogStore implements IAppendLogStore { state.pending.splice(0, batch.length); } } + } function logId(scope: string, key: string): string { @@ -268,6 +272,10 @@ function fromLogId(id: string): { scope: string; key: string } { return { scope: id.slice(0, index), key: id.slice(index + 1) }; } +function matchesScope(scope: string, scopePrefix: string | undefined): boolean { + return scopePrefix === undefined || scope === scopePrefix || scope.startsWith(`${scopePrefix}/`); +} + function encodeBatch(records: readonly unknown[]): Uint8Array { if (records.length === 0) return new Uint8Array(0); const content = records.map((record) => JSON.stringify(record) + '\n').join(''); diff --git a/packages/agent-core-v2/src/persistence/backends/node-fs/atomicDocumentStore.ts b/packages/agent-core-v2/src/persistence/backends/node-fs/atomicDocumentStore.ts index ca9532299b..e56a4552d7 100644 --- a/packages/agent-core-v2/src/persistence/backends/node-fs/atomicDocumentStore.ts +++ b/packages/agent-core-v2/src/persistence/backends/node-fs/atomicDocumentStore.ts @@ -13,6 +13,7 @@ import { InstantiationType } from '#/_base/di/extensions'; import { toDisposable, type IDisposable } from '#/_base/di/lifecycle'; import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; import { Event } from '#/_base/event'; +import { enqueueKeyedOperation } from '#/_base/utils/promise'; import { IFileSystemStorageService, StorageError, StorageErrors } from '#/persistence/interface/storage'; import { @@ -48,6 +49,7 @@ export const tomlDocumentCodec: DocumentCodec = { class AtomicDocumentStoreBase implements IAtomicDocumentStore { declare readonly _serviceBrand: undefined; + private readonly operationQueues = new Map>(); constructor( private readonly storage: IFileSystemStorageService, @@ -75,6 +77,29 @@ class AtomicDocumentStoreBase implements IAtomicDocumentStore { await this.storage.write(scope, key, this.codec.encode(value), { atomic: true }); } + update( + scope: string, + key: string, + mutate: (current: T | undefined) => T | Promise, + ): Promise { + return this.withExclusiveKeyMutation(scope, key, async () => { + const next = await mutate(await this.get(scope, key)); + await this.set(scope, key, next); + return next; + }); + } + + withExclusiveKeyMutation( + scope: string, + key: string, + mutation: () => Promise, + ): Promise { + if (this.storage.withExclusiveKeyMutation !== undefined) { + return this.storage.withExclusiveKeyMutation(scope, key, mutation); + } + return enqueueKeyedOperation(this.operationQueues, `${scope}\0${key}`, mutation); + } + async delete(scope: string, key: string): Promise { await this.storage.delete(scope, key); } diff --git a/packages/agent-core-v2/src/persistence/backends/node-fs/fileStorageService.ts b/packages/agent-core-v2/src/persistence/backends/node-fs/fileStorageService.ts index 4bb02fcbf0..cf9f7fd427 100644 --- a/packages/agent-core-v2/src/persistence/backends/node-fs/fileStorageService.ts +++ b/packages/agent-core-v2/src/persistence/backends/node-fs/fileStorageService.ts @@ -10,6 +10,9 @@ * fsync, so the replacement is both atomic and durable. * - `append` → `open('a')` + write + `fh.sync()` (when `durable`), plus a * one-time directory fsync per scope. + * - `withExclusiveKeyMutation` → the shared cross-process lock protocol + * on `.lock`, bounding waits to + * 10 seconds. * - `watch` → chokidar on the parent directory, filtered to the exact key and * debounced, so it survives atomic-replace renames and observes a * file that does not exist yet at subscription time. @@ -18,18 +21,26 @@ * control over append offsets, fsync, atomic rename and streaming, which the * agent-execution-environment abstraction does not expose. Higher-level code * (wire journal, blob store) goes through the Store / Storage interfaces above - * this backend, never `node:fs` directly. + * this backend, never `node:fs` directly. Session-rooted mutations run through + * the App-scoped write admission, which rejects writes after sealing and + * tracks admitted physical I/O until it settles. */ -import { createReadStream, mkdirSync } from 'node:fs'; +import { createReadStream, mkdirSync, statSync } from 'node:fs'; import { mkdir, open, readFile, readdir, unlink } from 'node:fs/promises'; import { FSWatcher } from 'chokidar'; import { dirname, join, normalize } from 'pathe'; import { DisposableStore, combinedDisposable, toDisposable, type IDisposable } from '#/_base/di/lifecycle'; +import { optional } from '#/_base/di/instantiation'; import { Emitter, type Event } from '#/_base/event'; import { onUnexpectedError } from '#/_base/errors/unexpectedError'; -import { atomicWrite, syncDir } from '#/_base/utils/fs'; +import { atomicWrite, fileStatTuplesEqual, isSpecialFileStat, syncDir } from '#/_base/utils/fs'; +import { + CrossProcessLockError, + CrossProcessLockErrorCode, + ICrossProcessLockService, +} from '#/os/interface/crossProcessLock'; import type { IFileSystemStorageService, @@ -37,9 +48,22 @@ import type { StorageReadRange, StorageWriteOptions, } from '#/persistence/interface/storage'; -import { toStorageIoError } from '#/persistence/interface/storage'; +import { StorageError, StorageErrors, toStorageIoError } from '#/persistence/interface/storage'; +import { IStorageWriteAdmission } from '#/persistence/interface/storageWriteAdmission'; const WATCH_DEBOUNCE_MS = 150; +const STORAGE_LOCK_WAIT_TIMEOUT_MS = 10_000; + +type WatchFingerprint = { readonly size: number; readonly mtimeMs: number; readonly ino: number } | undefined; + +function fingerprint(path: string): WatchFingerprint { + try { + const stats = statSync(path); + return { size: stats.size, mtimeMs: stats.mtimeMs, ino: stats.ino }; + } catch { + return undefined; + } +} function isEnoent(error: unknown): boolean { return (error as NodeJS.ErrnoException).code === 'ENOENT'; @@ -54,6 +78,9 @@ export class FileStorageService implements IFileSystemStorageService { private readonly baseDir: string, private readonly dirMode?: number, private readonly fileMode?: number, + @optional(ICrossProcessLockService) private readonly locks?: ICrossProcessLockService, + @optional(IStorageWriteAdmission) + private readonly writeAdmission?: IStorageWriteAdmission, ) {} async read(scope: string, key: string): Promise { @@ -93,13 +120,15 @@ export class FileStorageService implements IFileSystemStorageService { _options: StorageWriteOptions = {}, ): Promise { const filePath = this.path(scope, key); - try { - await mkdir(dirname(filePath), { recursive: true, mode: this.dirMode }); - await atomicWrite(filePath, data, undefined, this.fileMode); - await this.syncDirOnce(dirname(filePath)); - } catch (error) { - throw toStorageIoError(error, { path: filePath, op: 'write' }); - } + await this.withPhysicalWrite(scope, async () => { + try { + await mkdir(dirname(filePath), { recursive: true, mode: this.dirMode }); + await atomicWrite(filePath, data, undefined, this.fileMode); + await this.syncDirOnce(dirname(filePath)); + } catch (error) { + throw toStorageIoError(error, { path: filePath, op: 'write' }); + } + }); } async append( @@ -110,24 +139,25 @@ export class FileStorageService implements IFileSystemStorageService { ): Promise { const filePath = this.path(scope, key); const dir = dirname(filePath); - try { - await mkdir(dir, { recursive: true, mode: this.dirMode }); - - const fh = await open(filePath, 'a', this.fileMode); + await this.withPhysicalWrite(scope, async () => { try { - if (data.byteLength > 0) { - await fh.writeFile(data); - } - if (options.durable !== false) { - await fh.sync(); + await mkdir(dir, { recursive: true, mode: this.dirMode }); + const fh = await open(filePath, 'a', this.fileMode); + try { + if (data.byteLength > 0) { + await fh.writeFile(data); + } + if (options.durable !== false) { + await fh.sync(); + } + } finally { + await fh.close(); } - } finally { - await fh.close(); + await this.syncDirOnce(dir); + } catch (error) { + throw toStorageIoError(error, { path: filePath, op: 'append' }); } - await this.syncDirOnce(dir); - } catch (error) { - throw toStorageIoError(error, { path: filePath, op: 'append' }); - } + }); } async list(scope: string, prefix?: string): Promise { @@ -143,12 +173,14 @@ export class FileStorageService implements IFileSystemStorageService { async delete(scope: string, key: string): Promise { const filePath = this.path(scope, key); - try { - await unlink(filePath); - } catch (error) { - if (isEnoent(error)) return; - throw toStorageIoError(error, { path: filePath, op: 'delete' }); - } + await this.withPhysicalWrite(scope, async () => { + try { + await unlink(filePath); + } catch (error) { + if (isEnoent(error)) return; + throw toStorageIoError(error, { path: filePath, op: 'delete' }); + } + }); } watch(scope: string, key: string): Event { @@ -169,15 +201,23 @@ export class FileStorageService implements IFileSystemStorageService { const arm = (): void => { try { mkdirSync(dir, { recursive: true, mode: this.dirMode }); + const before = fingerprint(normalizedTarget); watcher = new FSWatcher({ ignoreInitial: true, awaitWriteFinish: false, depth: 0, + // chokidar attaches fs.watch to every scanned entry; special files + // (unix sockets, fifos, devices — e.g. an ipc `klient.sock` sharing + // the home root) make that call throw UNKNOWN. Skip them up front. + ignored: (_path, stats) => isSpecialFileStat(stats), }); watcher.on('all', (_event, changedPath) => { if (normalize(changedPath) === normalizedTarget) schedule(); }); watcher.on('error', (error: unknown) => onUnexpectedError(error)); + watcher.on('ready', () => { + if (!fileStatTuplesEqual(before, fingerprint(normalizedTarget))) schedule(); + }); watcher.add(dir); } catch (error) { onUnexpectedError(error); @@ -215,6 +255,45 @@ export class FileStorageService implements IFileSystemStorageService { }; } + async withExclusiveKeyMutation( + scope: string, + key: string, + mutation: () => Promise, + ): Promise { + const filePath = this.path(scope, key); + const lockPath = `${filePath}.lock`; + this.assertCanWriteNow(scope); + if (this.locks === undefined) { + throw new StorageError( + StorageErrors.codes.STORAGE_IO_FAILED, + 'file storage transaction requires a cross-process lock service', + { details: { path: filePath, op: 'lock' } }, + ); + } + try { + return await this.locks.withLock( + lockPath, + { wait: { timeoutMs: STORAGE_LOCK_WAIT_TIMEOUT_MS } }, + async () => { + this.assertCanWriteNow(scope); + return mutation(); + }, + ); + } catch (error) { + if (!(error instanceof CrossProcessLockError)) throw error; + if ( + error.code === CrossProcessLockErrorCode.Held || + error.code === CrossProcessLockErrorCode.WaitTimeout + ) { + throw new StorageError(StorageErrors.codes.STORAGE_LOCKED, 'storage transaction is locked', { + details: { path: filePath, op: 'lock' }, + cause: error, + }); + } + throw toStorageIoError(error, { path: lockPath, op: 'lock' }); + } + } + async flush(): Promise { } @@ -228,6 +307,14 @@ export class FileStorageService implements IFileSystemStorageService { return join(this.baseDir, scope); } + private assertCanWriteNow(scope: string): void { + this.writeAdmission?.assertCanWriteNow(scope); + } + + private withPhysicalWrite(scope: string, io: () => Promise): Promise { + return this.writeAdmission?.withPhysicalWrite(scope, io) ?? io(); + } + private async syncDirOnce(dir: string): Promise { if (this.syncedDirs.has(dir)) return; try { diff --git a/packages/agent-core-v2/src/persistence/backends/node-fs/storageWriteAdmissionService.ts b/packages/agent-core-v2/src/persistence/backends/node-fs/storageWriteAdmissionService.ts new file mode 100644 index 0000000000..2e0b997f94 --- /dev/null +++ b/packages/agent-core-v2/src/persistence/backends/node-fs/storageWriteAdmissionService.ts @@ -0,0 +1,76 @@ +/** + * `storage` domain (L1) — `IStorageWriteAdmission` implementation. + * + * Routes session-rooted storage scopes to admissions registered by session + * lifecycle. Double registration is a bug, missing session admissions fail + * closed, and root or non-session storage scopes execute directly. Bound at + * App scope. + */ + +import { InstantiationType } from '#/_base/di/extensions'; +import { BugIndicatingError } from '#/_base/errors/errors'; +import { toDisposable, type IDisposable } from '#/_base/di/lifecycle'; +import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; +import { Error2, ErrorCodes } from '#/errors'; +import type { ISessionWriteAdmission } from '#/persistence/interface/sessionWriteAdmission'; +import { IStorageWriteAdmission } from '#/persistence/interface/storageWriteAdmission'; + +export class StorageWriteAdmissionService implements IStorageWriteAdmission { + declare readonly _serviceBrand: undefined; + + private readonly admissions = new Map(); + + registerSession(sessionScope: string, admission: ISessionWriteAdmission): IDisposable { + if (this.admissions.has(sessionScope)) { + throw new BugIndicatingError(`write admission already registered for ${sessionScope}`); + } + this.admissions.set(sessionScope, admission); + return toDisposable(() => { + if (this.admissions.get(sessionScope) === admission) this.admissions.delete(sessionScope); + }); + } + + assertCanWriteNow(scope: string): void { + this.admissionFor(scope)?.assertCanWriteNow(); + } + + async withPhysicalWrite(scope: string, io: () => Promise): Promise { + const admission = this.admissionFor(scope); + return admission === undefined ? io() : admission.withPhysicalWrite(io); + } + + private admissionFor(scope: string): ISessionWriteAdmission | undefined { + const sessionScope = sessionScopeFromStorageScope(scope); + if (sessionScope === undefined) return undefined; + const admission = this.admissions.get(sessionScope); + if (admission === undefined) { + throw new Error2(ErrorCodes.SESSION_LEASE_LOST, 'session has no registered write admission', { + details: { sessionId: sessionScope.slice(sessionScope.lastIndexOf('/') + 1) }, + }); + } + return admission; + } +} + +function sessionScopeFromStorageScope(scope: string): string | undefined { + if (scope === '') return undefined; + const parts = scope.split('/'); + if ( + parts.length < 3 || + parts[0] !== 'sessions' || + parts[1] === '' || + parts[2] === undefined || + parts[2] === '' + ) { + return undefined; + } + return parts.slice(0, 3).join('/'); +} + +registerScopedService( + LifecycleScope.App, + IStorageWriteAdmission, + StorageWriteAdmissionService, + InstantiationType.Eager, + 'storage', +); diff --git a/packages/agent-core-v2/src/persistence/interface/appendLogStore.ts b/packages/agent-core-v2/src/persistence/interface/appendLogStore.ts index c241836493..82352d20cb 100644 --- a/packages/agent-core-v2/src/persistence/interface/appendLogStore.ts +++ b/packages/agent-core-v2/src/persistence/interface/appendLogStore.ts @@ -15,8 +15,10 @@ * later flush cannot duplicate data by guessing whether storage committed it. * A valid explicit `rewrite` is the recovery boundary: a successful atomic * replacement clears that failure before the preserved live tail drains. - * `flush` and `close` wait for every keyed buffer to settle before reporting - * the first failure in stable key insertion order. + * `flush(scopePrefix)` waits for the matching scope and descendants; omitting + * the prefix and `close` wait for every keyed buffer. Retired generations stay + * reachable after a failed final flush so later flushes report the same sticky + * failure instead of silently losing the in-memory tail. * * This file ships the interface, error class, and DI token only. * The concrete `AppendLogStore` implementation lives in @@ -52,7 +54,7 @@ export interface IAppendLogStore { append(scope: string, key: string, record: R, options?: AppendLogOptions): void; read(scope: string, key: string): AsyncIterable; rewrite(scope: string, key: string, records: readonly R[]): Promise; - flush(): Promise; + flush(scopePrefix?: string): Promise; close(): Promise; acquire(scope: string, key: string): IDisposable; } diff --git a/packages/agent-core-v2/src/persistence/interface/atomicDocumentStore.ts b/packages/agent-core-v2/src/persistence/interface/atomicDocumentStore.ts index 2d109d49b3..88792a252b 100644 --- a/packages/agent-core-v2/src/persistence/interface/atomicDocumentStore.ts +++ b/packages/agent-core-v2/src/persistence/interface/atomicDocumentStore.ts @@ -24,6 +24,16 @@ export interface IAtomicDocumentStore { get(scope: string, key: string): Promise; set(scope: string, key: string, value: T): Promise; + update( + scope: string, + key: string, + mutate: (current: T | undefined) => T | Promise, + ): Promise; + withExclusiveKeyMutation( + scope: string, + key: string, + mutation: () => Promise, + ): Promise; delete(scope: string, key: string): Promise; list(scope: string, prefix?: string): Promise; watch(scope: string, key: string): Event; diff --git a/packages/agent-core-v2/src/persistence/interface/sessionWriteAdmission.ts b/packages/agent-core-v2/src/persistence/interface/sessionWriteAdmission.ts new file mode 100644 index 0000000000..77a7ea0f5d --- /dev/null +++ b/packages/agent-core-v2/src/persistence/interface/sessionWriteAdmission.ts @@ -0,0 +1,19 @@ +/** + * `storage` domain (L1) — per-session physical-write admission contract. + * + * Defines the Session-scoped admission capability used to reject new writes + * after sealing, track admitted physical I/O, and await its drain. + */ + +import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; + +export interface ISessionWriteAdmission { + readonly _serviceBrand: undefined; + + assertCanWriteNow(): void; + withPhysicalWrite(io: () => Promise): Promise; + sealAndDrain(): Promise; +} + +export const ISessionWriteAdmission: ServiceIdentifier = + createDecorator('sessionWriteAdmission'); diff --git a/packages/agent-core-v2/src/persistence/interface/storage.ts b/packages/agent-core-v2/src/persistence/interface/storage.ts index 83c57ca17d..c3b7392fc6 100644 --- a/packages/agent-core-v2/src/persistence/interface/storage.ts +++ b/packages/agent-core-v2/src/persistence/interface/storage.ts @@ -7,6 +7,9 @@ * * - `write` — atomic whole-value replacement (the `Config` access pattern). * - `append` — ordered, durable byte extension (the `Record` access pattern). + * - `withExclusiveKeyMutation` — keyed read-modify-write exclusion when the + * backend can coordinate independent + * processes. * * They are not interchangeable: building `append` on top of `write` is O(n) * per append, and building `write` on top of `append` yields awkward "read @@ -128,6 +131,11 @@ export interface IFileSystemStorageService { list(scope: string, prefix?: string): Promise; delete(scope: string, key: string): Promise; watch?(scope: string, key: string): Event; + withExclusiveKeyMutation?( + scope: string, + key: string, + mutation: () => Promise, + ): Promise; flush(): Promise; close(): Promise; } diff --git a/packages/agent-core-v2/src/persistence/interface/storageWriteAdmission.ts b/packages/agent-core-v2/src/persistence/interface/storageWriteAdmission.ts new file mode 100644 index 0000000000..64e9e29c5c --- /dev/null +++ b/packages/agent-core-v2/src/persistence/interface/storageWriteAdmission.ts @@ -0,0 +1,22 @@ +/** + * `storage` domain (L1) — App-scoped storage write-admission contract. + * + * Routes session-rooted storage scopes to their per-session admission while + * leaving root and non-session scopes unrestricted. Missing session + * registrations fail closed. + */ + +import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; +import type { IDisposable } from '#/_base/di/lifecycle'; +import type { ISessionWriteAdmission } from '#/persistence/interface/sessionWriteAdmission'; + +export interface IStorageWriteAdmission { + readonly _serviceBrand: undefined; + + registerSession(sessionScope: string, admission: ISessionWriteAdmission): IDisposable; + assertCanWriteNow(scope: string): void; + withPhysicalWrite(scope: string, io: () => Promise): Promise; +} + +export const IStorageWriteAdmission: ServiceIdentifier = + createDecorator('storageWriteAdmission'); diff --git a/packages/agent-core-v2/src/session/agentLifecycle/agentLifecycleService.ts b/packages/agent-core-v2/src/session/agentLifecycle/agentLifecycleService.ts index 63655cd73b..1c55946e3b 100644 --- a/packages/agent-core-v2/src/session/agentLifecycle/agentLifecycleService.ts +++ b/packages/agent-core-v2/src/session/agentLifecycle/agentLifecycleService.ts @@ -7,9 +7,9 @@ * per-agent wire records and the wire state machine, the blob store, and MCP, * and registers the agent in the session registry. Binds the agent id into the * Agent-scoped telemetry view. New logs receive a metadata - * envelope while non-empty unversioned logs are rejected. Removal awaits the - * agent task manager's graceful exit policy before draining turns and full - * compaction, then disposing the child scope. Fans session-level + * envelope while non-empty unversioned logs are rejected. Removal coordinates + * task, turn, and full-compaction teardown before disposing the child scope. + * Fans session-level * permission-mode switches out to every live agent. Bound at Session scope. * * No agent id is special here: the main agent is simply the agent created @@ -63,7 +63,9 @@ import { IAgentMediaToolsRegistrar } from '#/agent/media/mediaTools'; import { IImageConfigBridge } from '#/agent/media/imageConfigBridge'; import { IAgentMcpService } from '#/agent/mcp/mcp'; import { IAgentExternalHooksService } from '#/agent/externalHooks/externalHooks'; +import { IAgentFileFencingService } from '#/agent/fileFencing/fileFencing'; import { IAgentPluginService } from '#/agent/plugin/agentPlugin'; +import { IAgentSkillListingReminderService } from '#/agent/profile/agentSkillListingReminder'; import { ISessionInteractionService } from '#/session/interaction/interaction'; import { IWireService } from '#/wire/wire'; import { ITelemetryService } from '#/app/telemetry/telemetry'; @@ -234,10 +236,17 @@ export class AgentLifecycleService extends Disposable implements IAgentLifecycle handle.accessor.get(IImageConfigBridge); handle.accessor.get(IAgentToolDedupeService); handle.accessor.get(IAgentExternalHooksService); + // File fencing wraps Read/Write/Edit execution on the shared Session file + // ledger. Resolve after externalHooks so permission and external preflight + // participants register first. + handle.accessor.get(IAgentFileFencingService); handle.accessor.get(IAgentMcpService); // Agent plugin service: registers main-agent-only plugin session-start // guidance before the first turn (self-gates to a no-op for other agents). handle.accessor.get(IAgentPluginService); + // Skill listing reminder: arms the turn-boundary notification for skill + // catalog hot reload before the first turn. + handle.accessor.get(IAgentSkillListingReminderService); // Tool-select services: precompute tool selection and the announcements // derived from it before the first turn. handle.accessor.get(IAgentToolSelectService); @@ -329,7 +338,8 @@ export class AgentLifecycleService extends Disposable implements IAgentLifecycle const handle = this.handles.get(agentId); if (handle === undefined) return; this.handles.delete(agentId); - await handle.accessor.get(IAgentTaskService).stopAllOnExit('Session closed'); + const tasks = handle.accessor.get(IAgentTaskService); + tasks.beginClose(); const loop = handle.accessor.get(IAgentLoopService); const compaction = handle.accessor.get(IAgentFullCompactionService).compacting; const compactionSettled = compaction?.promise.catch(() => undefined) ?? Promise.resolve(); @@ -342,6 +352,8 @@ export class AgentLifecycleService extends Disposable implements IAgentLifecycle compaction.abortController.abort(reason); } await Promise.all([loop.settled(), compactionSettled]); + await tasks.stopAllOnExit('Session closed'); + await tasks.flushPersistence(); handle.dispose(); this.onDidDisposeEmitter.fire(agentId); } diff --git a/packages/agent-core-v2/src/session/errors.ts b/packages/agent-core-v2/src/session/errors.ts index 5413a89c54..e1b74998f6 100644 --- a/packages/agent-core-v2/src/session/errors.ts +++ b/packages/agent-core-v2/src/session/errors.ts @@ -14,6 +14,9 @@ export const SessionErrors = { SESSION_FORK_ACTIVE_TURN: 'session.fork_active_turn', SESSION_UNDO_UNAVAILABLE: 'session.undo_unavailable', SESSION_INIT_FAILED: 'session.init_failed', + SESSION_HELD_BY_PEER: 'session.held_by_peer', + SESSION_LEASE_LOST: 'session.lease_lost', + SESSION_DURABILITY_FAILED: 'session.durability_failed', }, retryable: ['session.fork_active_turn'], } as const satisfies ErrorDomain; diff --git a/packages/agent-core-v2/src/session/sessionFileLedger/fileLedger.ts b/packages/agent-core-v2/src/session/sessionFileLedger/fileLedger.ts new file mode 100644 index 0000000000..875bbf9b63 --- /dev/null +++ b/packages/agent-core-v2/src/session/sessionFileLedger/fileLedger.ts @@ -0,0 +1,56 @@ +/** + * `sessionFileLedger` domain (L2) — per-session optimistic-concurrency ledger. + * + * Defines the `ISessionFileLedger` that remembers, per normalized absolute + * path, the on-disk stat tuple (`ino`, `mtimeMs`, `size`, existence) this + * session last successfully read or wrote. Before every Write/Edit, `compare` + * stats the target again and compares the tuple directly. The mechanism is + * detection, not mutual exclusion: it blocks stale writes it can observe and + * deliberately accepts the residual stat→write race, which no + * cooperating-process lock could close against external editors anyway. + * + * The verdict drives the write-path policy: `clean` lets the call through, + * `stale` means the file diverged since the baseline, and `no-baseline` means + * the target exists on disk but this session never read or wrote it (the + * read-first case). New-file creation is always `clean`. + * + * The ledger is in-memory only: never persisted, never journaled, never part + * of diagnostics. `resume` and `fork` therefore start from an empty ledger + * and conservatively require a fresh read before editing existing files. + * Baselines refresh only on successful Read/Write/Edit executions. A default + * Read counts after it scans a stable file even when its model-facing output + * is capped; explicit ranged Reads do not. Keys are lexical + * `normalizeFsWatchKey`s — no `realpath`, so symlink aliases to the same inode + * are an accepted residual gap. Session-scoped. + */ + +import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; +import { fileStatTuplesEqual as statTuplesEqual } from '#/_base/utils/fs'; + +export type FileStatTuple = + | { readonly exists: false } + | { + readonly exists: true; + readonly ino?: number; + readonly mtimeMs?: number; + readonly size: number; + }; + +export function fileStatTuplesEqual(a: FileStatTuple, b: FileStatTuple): boolean { + if (a.exists !== b.exists) return false; + if (!a.exists || !b.exists) return true; + return statTuplesEqual(a, b); +} + +export type FileLedgerVerdict = 'clean' | 'stale' | 'no-baseline'; + +export interface ISessionFileLedger { + readonly _serviceBrand: undefined; + + recordBaseline(path: string, revision: FileStatTuple): void; + + compare(path: string): Promise; +} + +export const ISessionFileLedger: ServiceIdentifier = + createDecorator('sessionFileLedger'); diff --git a/packages/agent-core-v2/src/session/sessionFileLedger/fileLedgerService.ts b/packages/agent-core-v2/src/session/sessionFileLedger/fileLedgerService.ts new file mode 100644 index 0000000000..c1d65eb015 --- /dev/null +++ b/packages/agent-core-v2/src/session/sessionFileLedger/fileLedgerService.ts @@ -0,0 +1,62 @@ +/** + * `sessionFileLedger` domain (L2) — `ISessionFileLedger` implementation. + * + * In-memory per-session ledger of on-disk stat tuples keyed by + * `normalizeFsWatchKey`. `recordBaseline` stores the revision observed by + * successful file I/O and `compare` always re-stats immediately before a + * write decision. An unverifiable stat fails closed as `stale`. + * Bound at Session scope. + */ + +import { InstantiationType } from '#/_base/di/extensions'; +import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; +import { unwrapErrorCause } from '#/_base/errors/errors'; +import { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { normalizeFsWatchKey } from '#/session/sessionFs/fsWatch'; + +import { + ISessionFileLedger, + fileStatTuplesEqual, + type FileLedgerVerdict, + type FileStatTuple, +} from './fileLedger'; + +export class SessionFileLedger implements ISessionFileLedger { + declare readonly _serviceBrand: undefined; + + private readonly entries = new Map(); + + constructor(@IHostFileSystem private readonly hostFs: IHostFileSystem) {} + + recordBaseline(path: string, revision: FileStatTuple): void { + this.entries.set(normalizeFsWatchKey(path), revision); + } + + async compare(path: string): Promise { + const key = normalizeFsWatchKey(path); + const entry = this.entries.get(key); + const current = await this.tryStat(path); + if (current === undefined) return 'stale'; + if (entry === undefined) return current.exists ? 'no-baseline' : 'clean'; + return fileStatTuplesEqual(entry, current) ? 'clean' : 'stale'; + } + + private async tryStat(path: string): Promise { + try { + const stat = await this.hostFs.stat(path); + return { exists: true, ino: stat.ino, mtimeMs: stat.mtimeMs, size: stat.size }; + } catch (error) { + const code = (unwrapErrorCause(error) as { code?: unknown } | null)?.code; + if (code === 'ENOENT' || code === 'ENOTDIR') return { exists: false }; + return undefined; + } + } +} + +registerScopedService( + LifecycleScope.Session, + ISessionFileLedger, + SessionFileLedger, + InstantiationType.Eager, + 'sessionFileLedger', +); diff --git a/packages/agent-core-v2/src/session/sessionFs/fsService.ts b/packages/agent-core-v2/src/session/sessionFs/fsService.ts index cf70e6e998..f08926d812 100644 --- a/packages/agent-core-v2/src/session/sessionFs/fsService.ts +++ b/packages/agent-core-v2/src/session/sessionFs/fsService.ts @@ -94,10 +94,19 @@ const FS_BINARY_NONPRINTABLE_FRACTION = 0.3; const HIDDEN_NAME_RE = /^\./; const MACOS_NOISE = new Set(['.DS_Store', '.AppleDouble', '.LSOverride']); +interface GitignoreCacheEntry { + readonly ig: Ignore; + /** + * Fingerprint of the workspace `.gitignore` the matcher was built from: + * its `mtimeMs`, or `undefined` when the file was absent at load time. + */ + readonly mtimeMs: number | undefined; +} + export class SessionFsService implements ISessionFsService { declare readonly _serviceBrand: undefined; - private readonly gitignoreCache = new Map(); + private readonly gitignoreCache = new Map(); private rgResolution: RgResolution | null | undefined = undefined; constructor( @@ -661,16 +670,25 @@ export class SessionFsService implements ISessionFsService { private async matcher(): Promise { const cwd = this.workspace.workDir; + const ignorePath = join(cwd, '.gitignore'); + // One stat per lookup keeps the cache honest: a changed or removed + // `.gitignore` (mtime mismatch / stat failure) rebuilds the entry. + // (The rg grep path never consults this cache — rg applies ignore rules + // itself.) + const st = await this.hostFs.stat(ignorePath).catch(() => undefined); + const mtimeMs = st === undefined ? undefined : (st.mtimeMs ?? -1); const cached = this.gitignoreCache.get(cwd); - if (cached !== undefined) return cached; + if (cached !== undefined && cached.mtimeMs === mtimeMs) return cached.ig; const ig = ignore(); ig.add('.git/'); - try { - const contents = await this.hostFs.readText(join(this.workspace.workDir, '.gitignore')); - ig.add(contents); - } catch { + if (st !== undefined) { + try { + ig.add(await this.hostFs.readText(ignorePath)); + } catch { + // The file raced away between stat and read — keep the base rules. + } } - this.gitignoreCache.set(cwd, ig); + this.gitignoreCache.set(cwd, { ig, mtimeMs }); return ig; } diff --git a/packages/agent-core-v2/src/session/sessionFs/fsWatch.ts b/packages/agent-core-v2/src/session/sessionFs/fsWatch.ts index 6e44c47780..0f3f2f8cf3 100644 --- a/packages/agent-core-v2/src/session/sessionFs/fsWatch.ts +++ b/packages/agent-core-v2/src/session/sessionFs/fsWatch.ts @@ -7,8 +7,16 @@ * workspace-relative paths they care about; events outside that subtree are * dropped. Session-scoped — the scope itself is the session, so no * `sessionId` is threaded through. + * + * Also owns the lexical key helper `normalizeFsWatchKey` (lexical normalize + * only, no `realpath`; case-folded on macOS/Windows). The watch service + * itself never keys paths with it — the sole consumer is the + * optimistic-concurrency ledger (`sessionFileLedger`), which keys its + * baselines with it. */ +import { normalize, sep } from 'node:path'; + import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; import type { Event } from '#/_base/event'; @@ -31,6 +39,13 @@ export interface FsChangeEvent { count?: number | undefined; } +const FS_WATCH_KEY_CASE_FOLD = process.platform === 'darwin' || process.platform === 'win32'; + +export function normalizeFsWatchKey(path: string): string { + const normalized = normalize(path).split(sep).join('/'); + return FS_WATCH_KEY_CASE_FOLD ? normalized.toLowerCase() : normalized; +} + export interface ISessionFsWatchService { readonly _serviceBrand: undefined; diff --git a/packages/agent-core-v2/src/session/sessionFs/fsWatchService.ts b/packages/agent-core-v2/src/session/sessionFs/fsWatchService.ts index a440595bd3..a1e991aa3d 100644 --- a/packages/agent-core-v2/src/session/sessionFs/fsWatchService.ts +++ b/packages/agent-core-v2/src/session/sessionFs/fsWatchService.ts @@ -4,10 +4,9 @@ * Subscribes to the os `IHostFsWatchService` on the workspace root, confines * events to the caller-declared subtree and to non-`.gitignore`d paths, * debounces them into fixed windows and re-exposes them as workspace-relative - * `FsChangeEvent`s. The os watcher is started lazily on the first non-empty - * subscription and stopped when the subscription set becomes empty. Path - * confinement is lexical (`ISessionWorkspaceContext.isWithin`), matching - * `sessionFs`. + * `FsChangeEvent`s. Path confinement is lexical (`ISessionWorkspaceContext.isWithin`), + * matching `sessionFs`. The os workDir watcher runs while the watched set is + * non-empty. */ import { isAbsolute, join, relative, sep } from 'node:path'; @@ -26,13 +25,19 @@ import { IHostFsWatchService, } from '#/os/interface/hostFsWatch'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; -import type { FsChangeEntry, FsChangeEvent } from './fsWatch'; +import type { FsChangeEvent } from './fsWatch'; import { ISessionFsWatchService } from './fsWatch'; const DEFAULT_DEBOUNCE_MS = 200; const DEFAULT_MAX_CHANGES_PER_WINDOW = 500; +interface PendingChange { + readonly rel: string; + readonly change: HostFsChange['action']; + readonly kind: HostFsChange['kind']; +} + function readPositiveIntEnv(name: string, fallback: number): number { const raw = process.env[name]; if (raw === undefined || raw === '') return fallback; @@ -51,7 +56,7 @@ export class SessionFsWatchService extends Disposable implements ISessionFsWatch private handleSub: IDisposable | undefined; private debounceTimer: NodeJS.Timeout | undefined; - private pending: FsChangeEntry[] = []; + private pending: PendingChange[] = []; private rawCount = 0; private truncated = false; @@ -64,7 +69,11 @@ export class SessionFsWatchService extends Disposable implements ISessionFsWatch DEFAULT_MAX_CHANGES_PER_WINDOW, ); - private readonly matcher: Ignore = ignore().add('.git/'); + // The matcher is the entire filter state — a rules change swaps it in + // whole, nothing else is kept. Rules load asynchronously: until the + // initial load settles, filtering conservatively uses the base rules + // (only `.git/`), i.e. unknown-but-maybe-ignored paths still pass through. + private matcher: Ignore = ignore().add('.git/'); private gitignoreLoaded = false; constructor( @@ -112,24 +121,38 @@ export class SessionFsWatchService extends Disposable implements ISessionFsWatch private loadGitignore(): void { if (this.gitignoreLoaded) return; this.gitignoreLoaded = true; - void this.hostFs - .readText(join(this.workspace.workDir, '.gitignore')) - .then( - (content) => { - this.matcher.add(content); - }, - () => undefined, - ); + void this.reloadGitignore(); + } + + private async reloadGitignore(): Promise { + const next = ignore().add('.git/'); + try { + next.add(await this.hostFs.readText(join(this.workspace.workDir, '.gitignore'))); + } catch { + // Missing or unreadable `.gitignore` — fall back to the base rules. + } + this.matcher = next; } private onRaw(e: HostFsChange): void { const rel = this.toRel(e.path); if (rel === '.') return; + // A change to the workspace-root `.gitignore` invalidates the filter + // rules: rebuild the matcher asynchronously (the recursive watcher already + // delivers this event, so no extra watcher is needed). Events arriving + // while the rebuild is in flight are filtered by the previous matcher. + if (rel === '.gitignore' && e.kind !== 'directory') { + void this.reloadGitignore(); + } const probe = e.kind === 'directory' ? `${rel}/` : rel; if (this.matcher.ignores(probe)) return; - if (!isUnderAny(rel, this.watched)) return; + if (isUnderAny(rel, this.watched)) { + this.record(e, rel); + } + } - this.pending.push({ path: rel, change: e.action, kind: e.kind }); + private record(e: HostFsChange, rel: string): void { + this.pending.push({ rel, change: e.action, kind: e.kind }); this.rawCount += 1; if (this.pending.length > this.maxChangesPerWindow) { this.truncated = true; @@ -147,17 +170,26 @@ export class SessionFsWatchService extends Disposable implements ISessionFsWatch if (this.rawCount === 0) return; const truncated = this.truncated; const count = this.rawCount; - const changes = truncated ? [] : this.pending; + const pending = this.pending; this.pending = []; this.rawCount = 0; this.truncated = false; - const event: FsChangeEvent = { - changes, - coalesced_window_ms: this.debounceMs, - ...(truncated ? { truncated: true, count } : {}), - }; - this.emitter.fire(event); + const changes = truncated + ? [] + : pending.map((entry) => ({ + path: entry.rel, + change: entry.change, + kind: entry.kind, + })); + if (truncated || changes.length > 0) { + const event: FsChangeEvent = { + changes, + coalesced_window_ms: this.debounceMs, + ...(truncated ? { truncated: true, count } : {}), + }; + this.emitter.fire(event); + } } private clearWindow(): void { diff --git a/packages/agent-core-v2/src/session/sessionLease/sessionLease.ts b/packages/agent-core-v2/src/session/sessionLease/sessionLease.ts new file mode 100644 index 0000000000..1cc1a9c15f --- /dev/null +++ b/packages/agent-core-v2/src/session/sessionLease/sessionLease.ts @@ -0,0 +1,217 @@ +/** + * `sessionLease` domain (L1) — the per-session write lease. + * + * Defines `ISessionLeaseService`, the Session-scope seeded ownership view, + * and the `SessionLease` object that satisfies it together with the + * `ISessionWriteAdmission` used by storage: an App-owned wrapper + * (`SessionLifecycleService` builds it; it is deliberately not a DI service) + * around the cross-process lock handle at + * `/session-leases/.lock`. `assertWritable` is the hard + * gate: it checks the live kernel-lock handle — a released or replaced + * sentinel fails closed with + * `session.lease_lost`, marks the lease lost, seals write admission, and fires + * the loss callback exactly once so the owning session tears itself down. + * The admission tracks physical writes so lifecycle release can await their + * drain. + * + * No default is registered for either Session-scoped view: every production + * session scope is seeded by `sessionLifecycle` via {@link sessionLeaseSeed}; + * resolving one unseeded (a session that bypassed materialization) is a bug + * and must fail loudly rather than silently disable the fencing gate. + */ + +import { join } from 'pathe'; + +import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; +import type { ScopeSeed } from '#/_base/di/scope'; +import { Error2, ErrorCodes } from '#/errors'; +import type { + CrossProcessLockInspection, + ICrossProcessLockHandle, +} from '#/os/interface/crossProcessLock'; +import { ISessionWriteAdmission } from '#/persistence/interface/sessionWriteAdmission'; + +export const LEASE_CREATING_RETRY_AFTER_MS = 1000; + +/** `details` payload of `session.held_by_peer` errors; the zod twin lives in + packages/protocol (`sessionOwnershipDetailsSchema`) and the shapes must + stay byte-identical. Declared as `type` (not `interface`) so the payload + stays assignable to `Error2Options.details`. */ +export type SessionOwnershipPhase = + | 'creating' + | 'routable' + | 'held-by-local-instance'; + +export type HeldByPeerDetails = { + readonly kind: 'held-by-peer'; + readonly phase: SessionOwnershipPhase; + readonly address?: string; + readonly retry_after_ms?: number; +}; + +export type SessionOwnershipDetails = HeldByPeerDetails; + +/** `held-by-peer` details for the converging 'creating' phase. Shared by + `heldByPeerDetailsFromInspection` and the lifecycle's failed-acquire probe: + a holder that vanished mid-race converges by retrying, same as 'creating'. */ +export const HELD_BY_PEER_CREATING_DETAILS: HeldByPeerDetails = { + kind: 'held-by-peer', + phase: 'creating', + retry_after_ms: LEASE_CREATING_RETRY_AFTER_MS, +}; + +/** + * Classify a lease inspection into `held-by-peer` details. Shared by every + * surface that reports session ownership — the lifecycle's + * post-acquire-failure probe and kap-server's read-only probes — so all of + * them classify the same lease the same way. Returns `undefined` when the + * lease is free; a caller on a failed-acquire path should map that to + * {@link HELD_BY_PEER_CREATING_DETAILS}. + */ +export function heldByPeerDetailsFromInspection( + inspection: CrossProcessLockInspection, +): HeldByPeerDetails | undefined { + if (inspection.state === 'held' && inspection.payload !== undefined) { + const { address } = inspection.payload; + return address !== undefined + ? { kind: 'held-by-peer', phase: 'routable', address } + : { kind: 'held-by-peer', phase: 'held-by-local-instance' }; + } + if (inspection.state === 'creating') { + return HELD_BY_PEER_CREATING_DETAILS; + } + return undefined; +} + +export interface ISessionLeaseInfo { + readonly sessionId: string; + readonly lockId: string; +} + +export interface ISessionLeaseService { + readonly _serviceBrand: undefined; + + /** The held lease identity; `undefined` once the lease is released. */ + readonly info: ISessionLeaseInfo | undefined; + /** Hard gate. Throws `Error2(session.lease_lost)` when this instance no + longer holds the kernel lease — including after `release()`. */ + assertWritable(): void; +} + +export const ISessionLeaseService: ServiceIdentifier = + createDecorator('sessionLeaseService'); + +export class SessionLease implements ISessionWriteAdmission, ISessionLeaseService { + declare readonly _serviceBrand: undefined; + + readonly lockId: string; + private _released = false; + private _lost = false; + private _lossFired = false; + private _sealed = false; + private inFlightWrites = 0; + private readonly drainWaiters = new Set<() => void>(); + + constructor( + readonly sessionId: string, + private readonly handle: ICrossProcessLockHandle, + private readonly onLeaseLost: (sessionId: string) => void, + ) { + this.lockId = handle.lockId; + } + + get info(): ISessionLeaseInfo | undefined { + return this._released ? undefined : { sessionId: this.sessionId, lockId: this.lockId }; + } + + private checkHeld(): boolean { + return !this._released && this.handle.checkHeld(); + } + + assertWritable(): void { + if (this._released) { + throw new Error2( + ErrorCodes.SESSION_LEASE_LOST, + `session ${this.sessionId} write lease was released`, + { details: { sessionId: this.sessionId } }, + ); + } + if (this._lost || !this.checkHeld()) { + this.markLost(); + throw new Error2( + ErrorCodes.SESSION_LEASE_LOST, + `session ${this.sessionId} no longer holds its write lease`, + { details: { sessionId: this.sessionId } }, + ); + } + } + + assertCanWriteNow(): void { + if (this._sealed) throw this.writeAdmissionClosedError(); + this.assertWritable(); + } + + async withPhysicalWrite(io: () => Promise): Promise { + this.assertCanWriteNow(); + this.inFlightWrites++; + try { + return await io(); + } finally { + this.inFlightWrites--; + if (this.inFlightWrites === 0) { + for (const resolve of this.drainWaiters) resolve(); + this.drainWaiters.clear(); + } + } + } + + sealAndDrain(): Promise { + this.sealWrites(); + return this.whenDrained(); + } + + private sealWrites(): void { + this._sealed = true; + } + + private whenDrained(): Promise { + if (this.inFlightWrites === 0) return Promise.resolve(); + return new Promise((resolve) => { + this.drainWaiters.add(resolve); + }); + } + + private writeAdmissionClosedError(): Error2 { + return new Error2( + ErrorCodes.SESSION_LEASE_LOST, + `session ${this.sessionId} write admission is sealed`, + { details: { sessionId: this.sessionId } }, + ); + } + + private markLost(): void { + this._lost = true; + this.sealWrites(); + if (this._lossFired) return; + this._lossFired = true; + this.onLeaseLost(this.sessionId); + } + + release(): void { + if (this._released) return; + this.sealWrites(); + this._released = true; + this.handle.release(); + } +} + +export function sessionLeasePath(homeDir: string, sessionId: string): string { + return join(homeDir, 'session-leases', `${sessionId}.lock`); +} + +export function sessionLeaseSeed(lease: SessionLease): ScopeSeed { + return [ + [ISessionLeaseService as ServiceIdentifier, lease], + [ISessionWriteAdmission as ServiceIdentifier, lease], + ]; +} diff --git a/packages/agent-core-v2/src/session/sessionLease/sessionLeaseContactProvider.ts b/packages/agent-core-v2/src/session/sessionLease/sessionLeaseContactProvider.ts new file mode 100644 index 0000000000..cba0ae6d2d --- /dev/null +++ b/packages/agent-core-v2/src/session/sessionLease/sessionLeaseContactProvider.ts @@ -0,0 +1,53 @@ +/** + * `sessionLease` domain (L1) — host-provided lease contact address. + * + * Holds what a session lease payload should advertise for this instance: + * `{type: 'address', address}` when the host runs a routable network service + * (kap-server seeds its listening address), `{type: 'local'}` otherwise — + * the value is recorded in the lease payload so a blocked peer can tell a + * routable holder from a local-only one (the discriminated `contact` union + * lands as the flat `address?` payload field). Read + * lazily at every lease acquisition through the `contact` thunk, so a host + * whose address is only known after listen can seed a provider that closes + * over it. Composition roots override the local-only default through + * {@link sessionLeaseContactSeed}; embedded engines pass nothing and keep + * the default. Bound at App scope. + */ + +import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; +import { InstantiationType } from '#/_base/di/extensions'; +import { LifecycleScope, registerScopedService, type ScopeSeed } from '#/_base/di/scope'; + +export type SessionLeaseContact = { type: 'address'; address: string } | { type: 'local' }; + +export interface ISessionLeaseContactProvider { + readonly _serviceBrand: undefined; + + readonly contact: () => SessionLeaseContact; +} + +export const ISessionLeaseContactProvider: ServiceIdentifier = + createDecorator('sessionLeaseContactProvider'); + +export class SessionLeaseContactProvider implements ISessionLeaseContactProvider { + declare readonly _serviceBrand: undefined; + + constructor(readonly contact: () => SessionLeaseContact = () => ({ type: 'local' })) {} +} + +export function sessionLeaseContactSeed(contact: () => SessionLeaseContact): ScopeSeed { + return [ + [ + ISessionLeaseContactProvider as ServiceIdentifier, + new SessionLeaseContactProvider(contact), + ], + ]; +} + +registerScopedService( + LifecycleScope.App, + ISessionLeaseContactProvider, + SessionLeaseContactProvider, + InstantiationType.Eager, + 'sessionLease', +); diff --git a/packages/agent-core-v2/src/session/sessionMetadata/sessionMetadataService.ts b/packages/agent-core-v2/src/session/sessionMetadata/sessionMetadataService.ts index 239abe87b8..f1f3a21b36 100644 --- a/packages/agent-core-v2/src/session/sessionMetadata/sessionMetadataService.ts +++ b/packages/agent-core-v2/src/session/sessionMetadata/sessionMetadataService.ts @@ -13,7 +13,10 @@ * released v1 builds. Re-registering an agent whose metadata is unchanged is * a no-op (no write, no mirror, no event), so resuming a session — which * re-registers its agents as they materialize — never bumps `updatedAt` and - * never reorders session listings. Bound at Session scope. + * never reorders session listings. Every durable write is fenced by the + * storage backend's per-session write admission, so an instance that lost or + * released the session lease fails closed instead of overwriting a live + * peer's state. Bound at Session scope. * * Read-model mirroring (flag `persistence_minidb_readmodel`): after a metadata * update is persisted, the fresh summary is mirrored into the `IQueryStore` diff --git a/packages/agent-core-v2/src/session/sessionSkillCatalog/explicitFileSkillSource.ts b/packages/agent-core-v2/src/session/sessionSkillCatalog/explicitFileSkillSource.ts index 2fea07d89f..500a213b80 100644 --- a/packages/agent-core-v2/src/session/sessionSkillCatalog/explicitFileSkillSource.ts +++ b/packages/agent-core-v2/src/session/sessionSkillCatalog/explicitFileSkillSource.ts @@ -3,19 +3,26 @@ * * Mirrors v1 SDK `skillDirs`: when runtime options provide `explicitDirs`, this * source contributes those directories as the user source, resolving relative - * paths against the session project root. When no explicit dirs are configured, - * it yields nothing so default user / project discovery remains active. Bound at - * Session scope so each session resolves paths against its own workDir. + * paths against the session project root, and hot-reloads on filesystem + * changes in them (probed through `hostFs` and watched through `hostFsWatch` + * via `SkillRootWatcher`). When no explicit dirs are configured, it yields + * nothing so default user / project discovery remains active. Bound at Session + * scope so each session resolves paths against its own workDir. */ import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; import { InstantiationType } from '#/_base/di/extensions'; +import { Disposable } from '#/_base/di/lifecycle'; +import { Emitter, type Event } from '#/_base/event'; import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; import { IBootstrapService } from '#/app/bootstrap/bootstrap'; -import { configuredRoots } from '#/app/skillCatalog/skillRoots'; +import { configuredRootCandidates, configuredRoots } from '#/app/skillCatalog/skillRoots'; import { ISkillCatalogRuntimeOptions } from '#/app/skillCatalog/skillCatalogRuntimeOptions'; import { ISkillDiscovery } from '#/app/skillCatalog/skillDiscovery'; +import { SkillRootWatcher } from '#/app/skillCatalog/skillRootWatcher'; import { SKILL_SOURCE_PRIORITY, type ISkillSource, type SkillContribution } from '#/app/skillCatalog/skillSource'; +import { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { IHostFsWatchService } from '#/os/interface/hostFsWatch'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; export interface IExplicitFileSkillSource extends ISkillSource { @@ -25,18 +32,41 @@ export interface IExplicitFileSkillSource extends ISkillSource { export const IExplicitFileSkillSource: ServiceIdentifier = createDecorator('explicitFileSkillSource'); -export class ExplicitFileSkillSource implements IExplicitFileSkillSource { +export class ExplicitFileSkillSource extends Disposable implements IExplicitFileSkillSource { declare readonly _serviceBrand: undefined; readonly id = 'explicit'; readonly priority = SKILL_SOURCE_PRIORITY.user; + private readonly onDidChangeEmitter = this._register(new Emitter()); + readonly onDidChange: Event = this.onDidChangeEmitter.event; constructor( @ISkillDiscovery private readonly discovery: ISkillDiscovery, @ISkillCatalogRuntimeOptions private readonly runtimeOptions: ISkillCatalogRuntimeOptions, @ISessionWorkspaceContext private readonly workspace: ISessionWorkspaceContext, @IBootstrapService private readonly bootstrap: IBootstrapService, - ) {} + @IHostFileSystem private readonly hostFs: IHostFileSystem, + @IHostFsWatchService hostFsWatch: IHostFsWatchService, + ) { + super(); + const explicitDirs = this.runtimeOptions.explicitDirs ?? []; + if (explicitDirs.length > 0) { + this._register( + new SkillRootWatcher( + this.hostFs, + hostFsWatch, + async () => + configuredRootCandidates( + this.hostFs, + explicitDirs, + this.workspace.workDir, + this.bootstrap.osHomeDir, + ), + () => this.onDidChangeEmitter.fire(), + ), + ); + } + } async load(): Promise { const explicitDirs = this.runtimeOptions.explicitDirs ?? []; @@ -44,7 +74,13 @@ export class ExplicitFileSkillSource implements IExplicitFileSkillSource { return { skills: [] }; } return this.discovery.discover( - await configuredRoots(explicitDirs, this.workspace.workDir, this.bootstrap.osHomeDir, 'user'), + await configuredRoots( + this.hostFs, + explicitDirs, + this.workspace.workDir, + this.bootstrap.osHomeDir, + 'user', + ), ); } } diff --git a/packages/agent-core-v2/src/session/sessionSkillCatalog/extraFileSkillSource.ts b/packages/agent-core-v2/src/session/sessionSkillCatalog/extraFileSkillSource.ts index 176c3492be..0d261c3f86 100644 --- a/packages/agent-core-v2/src/session/sessionSkillCatalog/extraFileSkillSource.ts +++ b/packages/agent-core-v2/src/session/sessionSkillCatalog/extraFileSkillSource.ts @@ -4,8 +4,11 @@ * Discovers user-configured extra skill directories (`extraSkillDirs`) through * `ISkillDiscovery`, contributing them at priority 10 (above plugin / builtin, * below user / workspace). Relative paths resolve against the session project - * root; `~` and `~/...` resolve against the bootstrap home dir. Bound at Session - * scope so each session reads its own workspace root. + * root; `~` and `~/...` resolve against the bootstrap home dir. Hot-reloads on + * both its config section (re-resolving the watched directories) and + * filesystem changes in the configured roots (watched through `hostFsWatch` + * via `SkillRootWatcher` and probed through `hostFs`). Bound at Session scope + * so each session reads its own workspace root. */ import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; @@ -19,9 +22,12 @@ import { EXTRA_SKILL_DIRS_SECTION, type ExtraSkillDirsConfig, } from '#/app/skillCatalog/configSection'; -import { configuredRoots } from '#/app/skillCatalog/skillRoots'; +import { configuredRootCandidates, configuredRoots } from '#/app/skillCatalog/skillRoots'; import { ISkillDiscovery } from '#/app/skillCatalog/skillDiscovery'; +import { SkillRootWatcher } from '#/app/skillCatalog/skillRootWatcher'; import { SKILL_SOURCE_PRIORITY, type ISkillSource, type SkillContribution } from '#/app/skillCatalog/skillSource'; +import { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { IHostFsWatchService } from '#/os/interface/hostFsWatch'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; export interface IExtraFileSkillSource extends ISkillSource { @@ -38,26 +44,57 @@ export class ExtraFileSkillSource extends Disposable implements IExtraFileSkillS readonly priority = SKILL_SOURCE_PRIORITY.extra; private readonly onDidChangeEmitter = this._register(new Emitter()); readonly onDidChange: Event = this.onDidChangeEmitter.event; + private readonly watcher: SkillRootWatcher; constructor( @ISkillDiscovery private readonly discovery: ISkillDiscovery, @IConfigService private readonly config: IConfigService, @ISessionWorkspaceContext private readonly workspace: ISessionWorkspaceContext, @IBootstrapService private readonly bootstrap: IBootstrapService, + @IHostFileSystem private readonly hostFs: IHostFileSystem, + @IHostFsWatchService hostFsWatch: IHostFsWatchService, ) { super(); this._register( this.config.onDidSectionChange((event) => { - if (event.domain === EXTRA_SKILL_DIRS_SECTION) this.onDidChangeEmitter.fire(); + if (event.domain === EXTRA_SKILL_DIRS_SECTION) { + this.onDidChangeEmitter.fire(); + void this.watcher.refresh(); + } }), ); + this.watcher = this._register( + new SkillRootWatcher( + this.hostFs, + hostFsWatch, + () => this.watchCandidates(), + () => this.onDidChangeEmitter.fire(), + ), + ); } async load(): Promise { await this.config.ready; const extraSkillDirs = this.config.get(EXTRA_SKILL_DIRS_SECTION) ?? []; return this.discovery.discover( - await configuredRoots(extraSkillDirs, this.workspace.workDir, this.bootstrap.osHomeDir, 'extra'), + await configuredRoots( + this.hostFs, + extraSkillDirs, + this.workspace.workDir, + this.bootstrap.osHomeDir, + 'extra', + ), + ); + } + + private async watchCandidates(): Promise { + await this.config.ready; + const extraSkillDirs = this.config.get(EXTRA_SKILL_DIRS_SECTION) ?? []; + return configuredRootCandidates( + this.hostFs, + extraSkillDirs, + this.workspace.workDir, + this.bootstrap.osHomeDir, ); } } diff --git a/packages/agent-core-v2/src/session/sessionSkillCatalog/workspaceFileSkillSource.ts b/packages/agent-core-v2/src/session/sessionSkillCatalog/workspaceFileSkillSource.ts index 18fbc99f37..99a5ba9bfb 100644 --- a/packages/agent-core-v2/src/session/sessionSkillCatalog/workspaceFileSkillSource.ts +++ b/packages/agent-core-v2/src/session/sessionSkillCatalog/workspaceFileSkillSource.ts @@ -3,8 +3,10 @@ * * Discovers project skills from the session's current `workDir` * (`workspaceContext`) through `ISkillDiscovery`, contributing them at priority - * 30 (above user / extra / plugin / builtin). Bound at Session scope so each session reads - * its own workspace root. + * 30 (above user / extra / plugin / builtin). Hot-reloads on both its config + * section and filesystem changes in the project skill roots (watched through + * `hostFsWatch` via `SkillRootWatcher` and probed through `hostFs`). Bound at + * Session scope so each session reads its own workspace root. */ import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; @@ -19,8 +21,11 @@ import { } from '#/app/skillCatalog/configSection'; import { ISkillCatalogRuntimeOptions } from '#/app/skillCatalog/skillCatalogRuntimeOptions'; import { ISkillDiscovery } from '#/app/skillCatalog/skillDiscovery'; -import { projectRoots } from '#/app/skillCatalog/skillRoots'; +import { SkillRootWatcher } from '#/app/skillCatalog/skillRootWatcher'; +import { projectRootCandidates, projectRoots } from '#/app/skillCatalog/skillRoots'; import { SKILL_SOURCE_PRIORITY, type ISkillSource, type SkillContribution } from '#/app/skillCatalog/skillSource'; +import { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { IHostFsWatchService } from '#/os/interface/hostFsWatch'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; export interface IWorkspaceFileSkillSource extends ISkillSource { @@ -43,6 +48,8 @@ export class WorkspaceFileSkillSource extends Disposable implements IWorkspaceFi @ISessionWorkspaceContext private readonly workspace: ISessionWorkspaceContext, @IConfigService private readonly config: IConfigService, @ISkillCatalogRuntimeOptions private readonly runtimeOptions: ISkillCatalogRuntimeOptions, + @IHostFileSystem private readonly hostFs: IHostFileSystem, + @IHostFsWatchService hostFsWatch: IHostFsWatchService, ) { super(); this._register( @@ -50,6 +57,16 @@ export class WorkspaceFileSkillSource extends Disposable implements IWorkspaceFi if (event.domain === MERGE_ALL_AVAILABLE_SKILLS_SECTION) this.onDidChangeEmitter.fire(); }), ); + if ((this.runtimeOptions.explicitDirs?.length ?? 0) === 0) { + this._register( + new SkillRootWatcher( + this.hostFs, + hostFsWatch, + async () => projectRootCandidates(this.hostFs, this.workspace.workDir), + () => this.onDidChangeEmitter.fire(), + ), + ); + } } async load(): Promise { @@ -59,7 +76,9 @@ export class WorkspaceFileSkillSource extends Disposable implements IWorkspaceFi await this.config.ready; const mergeAllAvailableSkills = this.config.get(MERGE_ALL_AVAILABLE_SKILLS_SECTION) ?? true; - return this.discovery.discover(await projectRoots(this.workspace.workDir, { mergeAllAvailableSkills })); + return this.discovery.discover( + await projectRoots(this.hostFs, this.workspace.workDir, { mergeAllAvailableSkills }), + ); } } diff --git a/packages/agent-core-v2/src/tool/toolContract.ts b/packages/agent-core-v2/src/tool/toolContract.ts index 45c5f66bd5..b28604115b 100644 --- a/packages/agent-core-v2/src/tool/toolContract.ts +++ b/packages/agent-core-v2/src/tool/toolContract.ts @@ -36,22 +36,47 @@ export interface ToolDelivery { readonly message: ToolDeliveryMessage; } -export interface ExecutableToolSuccessResult { +export interface ToolFileRevision { + readonly path: string; + readonly ino?: number; + readonly mtimeMs?: number; + readonly size: number; +} + +export const toolFileRevision = Symbol('toolFileRevision'); + +export function makeToolFileRevision( + path: string, + stat: { readonly ino?: number; readonly mtimeMs?: number; readonly size: number }, +): ToolFileRevision { + return { path, ino: stat.ino, mtimeMs: stat.mtimeMs, size: stat.size }; +} + +export function attachToolFileRevision( + result: T, + revision: ToolFileRevision, +): T & { readonly [toolFileRevision]: ToolFileRevision } { + return Object.defineProperty(result, toolFileRevision, { + value: revision, + enumerable: false, + }) as T & { readonly [toolFileRevision]: ToolFileRevision }; +} + +interface ExecutableToolResultBase { readonly output: ExecutableToolOutput; - readonly isError?: false | undefined; readonly stopTurn?: boolean | undefined; readonly truncated?: boolean | undefined; readonly note?: string; readonly delivery?: ToolDelivery | undefined; + readonly [toolFileRevision]?: ToolFileRevision; } -export interface ExecutableToolErrorResult { - readonly output: ExecutableToolOutput; +export interface ExecutableToolSuccessResult extends ExecutableToolResultBase { + readonly isError?: false | undefined; +} + +export interface ExecutableToolErrorResult extends ExecutableToolResultBase { readonly isError: true; - readonly stopTurn?: boolean | undefined; - readonly truncated?: boolean | undefined; - readonly note?: string; - readonly delivery?: ToolDelivery | undefined; } export type ExecutableToolResult = ExecutableToolSuccessResult | ExecutableToolErrorResult; diff --git a/packages/agent-core-v2/test/_base/di/auto-inject.test.ts b/packages/agent-core-v2/test/_base/di/auto-inject.test.ts index c8f2eca181..f969257a62 100644 --- a/packages/agent-core-v2/test/_base/di/auto-inject.test.ts +++ b/packages/agent-core-v2/test/_base/di/auto-inject.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import { SyncDescriptor } from '#/_base/di/descriptors'; import { CyclicDependencyError } from '#/_base/di/errors'; -import { IInstantiationService, createDecorator } from '#/_base/di/instantiation'; +import { IInstantiationService, createDecorator, optional } from '#/_base/di/instantiation'; import { InstantiationService } from '#/_base/di/instantiationService'; import { ServiceCollection } from '#/_base/di/serviceCollection'; @@ -66,6 +66,50 @@ describe('@IFoo auto-injection', () => { expect(bar.baz).toBeInstanceOf(Baz); }); + it('injects undefined for an unregistered optional service in strict mode', () => { + interface IDependency { + tag: 'dependency'; + } + const IDependency = createDecorator('p1.1-IDependency-optional-missing'); + class Consumer { + constructor(@optional(IDependency) readonly dependency?: IDependency) {} + } + const IConsumer = createDecorator('p1.1-IConsumer-optional-missing'); + const ix = new InstantiationService( + new ServiceCollection([IConsumer, new SyncDescriptor(Consumer)]), + true, + ); + + const consumer = ix.invokeFunction((accessor) => accessor.get(IConsumer)); + + expect(consumer.dependency).toBeUndefined(); + }); + + it('resolves a registered optional service normally', () => { + interface IDependency { + tag: 'dependency'; + } + const IDependency = createDecorator('p1.1-IDependency-optional-present'); + class Dependency implements IDependency { + tag = 'dependency' as const; + } + class Consumer { + constructor(@optional(IDependency) readonly dependency?: IDependency) {} + } + const IConsumer = createDecorator('p1.1-IConsumer-optional-present'); + const ix = new InstantiationService( + new ServiceCollection( + [IDependency, new SyncDescriptor(Dependency)], + [IConsumer, new SyncDescriptor(Consumer)], + ), + true, + ); + + const consumer = ix.invokeFunction((accessor) => accessor.get(IConsumer)); + + expect(consumer.dependency).toBeInstanceOf(Dependency); + }); + it('@IInstantiationService self-injection resolves to the OWNING container', () => { class Widget { constructor(public readonly label: string) {} diff --git a/packages/agent-core-v2/test/agent/fileFencing/fileFencing.test.ts b/packages/agent-core-v2/test/agent/fileFencing/fileFencing.test.ts new file mode 100644 index 0000000000..29966fd86b --- /dev/null +++ b/packages/agent-core-v2/test/agent/fileFencing/fileFencing.test.ts @@ -0,0 +1,385 @@ +/** + * `fileFencing` domain (L4) — verifies the write/read-first gate end to end + * through the real DI scope tree: a real tmpdir, the real `HostFileSystem`, + * and the registered `writeFencing` hook participant over a real + * `OrderedHookSlot`. Covers the hard-block verdicts (stale outside change / + * never-read existing file), stat-only checks, and ledger isolation between + * two Session scopes sharing one workspace. + */ + +import { mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { LifecycleScope, type Scope } from '#/_base/di/scope'; +import { createScopedTestHost, stubPair, type ScopedTestHost } from '#/_base/di/test'; +import { IAgentFileFencingService } from '#/agent/fileFencing/fileFencing'; +import { AgentFileFencingService } from '#/agent/fileFencing/fileFencingService'; +import { IAgentToolExecutorService } from '#/agent/toolExecutor/toolExecutor'; +import type { + ToolBeforeExecuteContext, + ToolDidExecuteContext, +} from '#/agent/toolExecutor/toolHooks'; +import type { ToolCall } from '#/kosong/contract/message'; +import { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { ISessionFileLedger } from '#/session/sessionFileLedger/fileLedger'; +import { SessionFileLedger } from '#/session/sessionFileLedger/fileLedgerService'; +import { + ToolAccesses, + type ExecutableToolResult, + makeToolFileRevision, + toolFileRevision, +} from '#/tool/toolContract'; + +import { AgentToolExecutorService } from '#/agent/toolExecutor/toolExecutorService'; + +import { stubToolExecutor } from '../loop/stubs'; +import { countingHostFs } from '../../session/sessionFs/stubs'; + +void AgentFileFencingService; +void SessionFileLedger; +void AgentToolExecutorService; + +interface Env { + readonly host: ScopedTestHost; + readonly workDir: string; + readonly outsideDir: string; + readonly statCalls: () => number; +} + +function makeEnv(): Env { + const workDir = mkdtempSync(join(tmpdir(), 'kimi-fencing-work-')); + const outsideDir = mkdtempSync(join(tmpdir(), 'kimi-fencing-out-')); + cleanupPaths.push(workDir, outsideDir); + const { fs, statCalls } = countingHostFs(); + const host = createScopedTestHost([stubPair(IHostFileSystem, fs)]); + hosts.push(host); + return { host, workDir, outsideDir, statCalls }; +} + +interface AgentWorld { + readonly env: Env; + readonly session: Scope; + readonly agent: Scope; + readonly executor: IAgentToolExecutorService; + readonly ledger: ISessionFileLedger; +} + +function makeAgent(env: Env, session: Scope): AgentWorld { + const executor = stubToolExecutor(); + const agent = env.host.childOf(session, LifecycleScope.Agent, 'main', [ + stubPair(IAgentToolExecutorService, executor), + ]); + agent.accessor.get(IAgentFileFencingService); + return { + env, + session, + agent, + executor, + ledger: session.accessor.get(ISessionFileLedger), + }; +} + +function setup(): AgentWorld { + const env = makeEnv(); + return makeAgent(env, env.host.child(LifecycleScope.Session, 's1')); +} + +function beforeCtx( + toolName: string, + path: string, + opts: { args?: Record } = {}, +): ToolBeforeExecuteContext { + const args = + opts.args ?? + (toolName === 'Edit' + ? { path, old_string: 'a', new_string: 'b' } + : toolName === 'Write' + ? { path, content: 'x' } + : { path }); + const toolCall: ToolCall = { + type: 'function', + id: 'call-1', + name: toolName, + arguments: JSON.stringify(args), + }; + const operation = toolName === 'Read' ? 'read' : toolName === 'Write' ? 'write' : 'readwrite'; + return { + turnId: 1, + signal: new AbortController().signal, + toolCall, + toolCalls: [toolCall], + args, + execution: { + accesses: ToolAccesses.file(operation, path), + approvalRule: toolName, + execute: async () => ({ output: 'ok' }), + }, + }; +} + +async function runBefore( + world: AgentWorld, + ctx: ToolBeforeExecuteContext, +): Promise { + await world.executor.hooks.onBeforeExecuteTool.run(ctx); + await runPrepared(ctx); + return ctx; +} + +async function runPrepared( + ctx: ToolBeforeExecuteContext, +): Promise { + if (ctx.decision?.execute === undefined) return undefined; + return ctx.decision.execute({ + turnId: ctx.turnId, + toolCallId: ctx.toolCall.id, + trace: ctx.trace, + signal: ctx.signal, + }); +} + +async function runDid( + world: AgentWorld, + ctx: ToolBeforeExecuteContext, + result?: ExecutableToolResult, +): Promise { + let effectiveResult = result; + if (effectiveResult === undefined) { + const target = ctx.execution.accesses?.find((access) => access.kind === 'file'); + if (target === undefined || !['Read', 'Write', 'Edit'].includes(ctx.toolCall.name)) { + effectiveResult = { output: 'done' }; + } else { + let stat; + try { + stat = statSync(target.path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + stat = { size: 0 }; + } + effectiveResult = { + output: 'done', + [toolFileRevision]: makeToolFileRevision(target.path, stat), + }; + } + } + const did: ToolDidExecuteContext = { + turnId: ctx.turnId, + signal: ctx.signal, + toolCall: ctx.toolCall, + toolCalls: [ctx.toolCall], + args: ctx.args, + result: effectiveResult, + }; + await world.executor.hooks.onDidExecuteTool.run(did); + return did; +} + +async function runOk( + world: AgentWorld, + toolName: string, + path: string, + opts: { args?: Record } = {}, +): Promise { + const ctx = beforeCtx(toolName, path, opts); + await world.executor.hooks.onBeforeExecuteTool.run(ctx); + const prepared = await runPrepared(ctx); + expect(prepared?.isError).not.toBe(true); + if (toolName === 'Write') { + const args = ctx.args as { readonly content: string; readonly mode?: 'overwrite' | 'append' }; + writeFileSync(path, args.content, { flag: args.mode === 'append' ? 'a' : 'w' }); + } + return runDid(world, ctx); +} + +async function runBlocked( + world: AgentWorld, + toolName: string, + path: string, +): Promise { + const ctx = beforeCtx(toolName, path); + await world.executor.hooks.onBeforeExecuteTool.run(ctx); + const result = await runPrepared(ctx); + if (result?.isError !== true) { + throw new Error(`expected ${toolName} on ${path} to be blocked, got: ${JSON.stringify(result)}`); + } + return result; +} + +const hosts: ScopedTestHost[] = []; +const cleanupPaths: string[] = []; + +describe('AgentFileFencingService', () => { + afterEach(() => { + for (const host of hosts.splice(0)) host.dispose(); + for (const path of cleanupPaths.splice(0)) rmSync(path, { recursive: true, force: true }); + }); + + it('blocks Edit on an existing file that was never read, read-first', async () => { + const world = setup(); + const file = join(world.env.workDir, 'a.txt'); + writeFileSync(file, 'hello'); + + const blocked = await runBlocked(world, 'Edit', file); + expect(blocked.output).toContain('has not been read in this session'); + expect(blocked.output).toContain('Read it first'); + }); + + it('blocks Edit when the file changed on disk since the last read, and unblocks after re-read', async () => { + const world = setup(); + const file = join(world.env.workDir, 'a.txt'); + writeFileSync(file, 'hello'); + await runOk(world, 'Read', file); + + // An intervening successful Edit re-baselines, so the next Edit starts clean. + await runOk(world, 'Edit', file); + + writeFileSync(file, 'hello world'); + + const blocked = await runBlocked(world, 'Edit', file); + expect(blocked.output).toContain('changed on disk since'); + + await runOk(world, 'Read', file); + await runOk(world, 'Edit', file); + }); + + it('checks the file at execution time rather than during preflight', async () => { + const world = setup(); + const file = join(world.env.workDir, 'a.txt'); + writeFileSync(file, 'hello'); + await runOk(world, 'Read', file); + + const ctx = beforeCtx('Edit', file); + await world.executor.hooks.onBeforeExecuteTool.run(ctx); + writeFileSync(file, 'changed while queued'); + + const blocked = await runPrepared(ctx); + expect(blocked?.isError).toBe(true); + expect(blocked?.output).toContain('changed on disk since'); + }); + + it('keeps the revision captured by Read when the file changes before the did-hook', async () => { + const world = setup(); + const file = join(world.env.workDir, 'a.txt'); + writeFileSync(file, 'hello'); + const ctx = await runBefore(world, beforeCtx('Read', file)); + const readRevision = makeToolFileRevision(file, statSync(file)); + + writeFileSync(file, 'changed after read'); + await runDid(world, ctx, { + output: 'hello', + [toolFileRevision]: readRevision, + }); + + const blocked = await runBlocked(world, 'Edit', file); + expect(blocked.output).toContain('changed on disk since'); + }); + + it('blocks Write over an existing file that was never read', async () => { + const world = setup(); + const file = join(world.env.workDir, 'a.txt'); + writeFileSync(file, 'hello'); + + const blocked = await runBlocked(world, 'Write', file); + expect(blocked.output).toContain('already exists'); + expect(blocked.output).toContain('has not been read in this session'); + }); + + it('allows Write append on an existing file without a read baseline', async () => { + const world = setup(); + const file = join(world.env.workDir, 'a.txt'); + writeFileSync(file, 'before'); + + await runOk(world, 'Write', file, { + args: { path: file, content: ' after', mode: 'append' }, + }); + + expect(readFileSync(file, 'utf8')).toBe('before after'); + }); + + it('allows Write creating a new file and baselines it', async () => { + const world = setup(); + const file = join(world.env.workDir, 'new.txt'); + + await runOk(world, 'Write', file); + expect(await world.ledger.compare(file)).toBe('clean'); + + await runOk(world, 'Edit', file); + }); + + it('allows consecutive Edits and keeps them clean while the stat tuple matches the baseline', async () => { + const world = setup(); + const file = join(world.env.workDir, 'a.txt'); + writeFileSync(file, 'hello'); + await runOk(world, 'Read', file); + expect(world.env.statCalls()).toBe(0); + + await runOk(world, 'Edit', file); + expect(world.env.statCalls()).toBe(1); + + await runOk(world, 'Edit', file); + expect(world.env.statCalls()).toBe(2); + + expect(await world.ledger.compare(file)).toBe('clean'); + }); + + it('blocks ranged-Read followed by Edit because ranged reads never baseline', async () => { + const world = setup(); + const file = join(world.env.workDir, 'a.txt'); + writeFileSync(file, '1\n2\n3\n4\n5\n6\n'); + + await runOk(world, 'Read', file, { args: { path: file, line_offset: 5 } }); + const blocked = await runBlocked(world, 'Edit', file); + expect(blocked.output).toContain('has not been read in this session'); + }); + + it('keeps Edit blocked when a default Read has no complete-file revision', async () => { + const world = setup(); + const file = join(world.env.workDir, 'a.txt'); + writeFileSync(file, 'partial read'); + const ctx = await runBefore(world, beforeCtx('Read', file)); + + await runDid(world, ctx, { output: 'partial' }); + + const blocked = await runBlocked(world, 'Edit', file); + expect(blocked.output).toContain('has not been read in this session'); + }); + + it('blocks out-of-root writes through the stat-only fallback and allows them after Read', async () => { + const world = setup(); + const file = join(world.env.outsideDir, 'b.txt'); + writeFileSync(file, 'hello'); + + const first = await runBlocked(world, 'Edit', file); + expect(first.output).toContain('has not been read in this session'); + + await runOk(world, 'Read', file); + await runOk(world, 'Edit', file); + + writeFileSync(file, 'hello world'); + const changed = await runBlocked(world, 'Edit', file); + expect(changed.output).toContain('changed on disk since'); + }); + + it('records no baseline and stays blocked when the fenced call fails', async () => { + const world = setup(); + const file = join(world.env.workDir, 'a.txt'); + writeFileSync(file, 'hello'); + await runOk(world, 'Read', file); + + writeFileSync(file, 'hello world'); + + // The stale verdict blocks the Edit; the wrapper's error result must not + // be baselined by the did-hook, so the file stays stale until re-read. + const failed = beforeCtx('Edit', file); + await world.executor.hooks.onBeforeExecuteTool.run(failed); + const failedResult = await runPrepared(failed); + expect(failedResult?.isError).toBe(true); + await runDid(world, failed, { output: 'boom', isError: true }); + + const retry = await runBlocked(world, 'Edit', file); + expect(retry.output).toContain('changed on disk since'); + }); +}); diff --git a/packages/agent-core-v2/test/agent/plan/injection/planModeInjection.test.ts b/packages/agent-core-v2/test/agent/plan/injection/planModeInjection.test.ts index 87df3bd290..d38b02bf78 100644 --- a/packages/agent-core-v2/test/agent/plan/injection/planModeInjection.test.ts +++ b/packages/agent-core-v2/test/agent/plan/injection/planModeInjection.test.ts @@ -66,7 +66,7 @@ describe('PlanModeService dynamic injection content', () => { hostFs: createFakeHostFs({ mkdir: vi.fn().mockResolvedValue(undefined), readText: (path: string) => readText(path), - writeText: vi.fn(async () => undefined), + writeText: vi.fn(async () => ({ isFile: true, isDirectory: false, size: 0 })), }), })); context = ctx.get(IAgentContextMemoryService); @@ -148,7 +148,7 @@ describe('PlanModeService dynamic injection cadence', () => { hostFs: createFakeHostFs({ mkdir: vi.fn().mockResolvedValue(undefined), readText: async () => '', - writeText: vi.fn(async () => undefined), + writeText: vi.fn(async () => ({ isFile: true, isDirectory: false, size: 0 })), }), })); context = ctx.get(IAgentContextMemoryService); diff --git a/packages/agent-core-v2/test/agent/plan/plan.test.ts b/packages/agent-core-v2/test/agent/plan/plan.test.ts index ca7fa17992..53b6036c7e 100644 --- a/packages/agent-core-v2/test/agent/plan/plan.test.ts +++ b/packages/agent-core-v2/test/agent/plan/plan.test.ts @@ -11,7 +11,7 @@ import { IAgentPlanService, type PlanData } from '#/agent/plan/plan'; import { IAgentPermissionRulesService } from '#/agent/permissionRules/permissionRules'; import { IAgentProfileService } from '#/agent/profile/profile'; import { IAgentScopeContext } from '#/agent/scopeContext/scopeContext'; -import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import type { HostFileStat, IHostFileSystem } from '#/os/interface/hostFileSystem'; import { ISessionContext } from '#/session/sessionContext/sessionContext'; import type { ISessionProcessRunner } from '#/session/process/processRunner'; import { createFakeHostFs, createFakeProcessRunner } from '../../tools/fixtures/fake-exec'; @@ -27,10 +27,17 @@ interface PlanFakes { readonly runner: ISessionProcessRunner; } +function textFileStat(content: string = ''): HostFileStat { + return { isFile: true, isDirectory: false, size: Buffer.byteLength(content) }; +} + function createPlanFakes(overrides: Partial = {}): PlanFakes { const fs = createFakeHostFs({ mkdir: vi.fn().mockResolvedValue(undefined), readText: vi.fn().mockResolvedValue(''), + // Skill-root probes canonicalize through host fs; the fake fs has no + // symlinks, so identity realpath is the faithful default. + realpath: vi.fn(async (p: string) => p), ...overrides, }); const runner = createFakeProcessRunner(); @@ -56,6 +63,7 @@ function createPlanFileFakes( const readText = vi.fn(async (path: string) => files.get(path) ?? ''); const writeText = vi.fn(async (path: string, content: string) => { files.set(path, content); + return textFileStat(content); }); return { files, @@ -172,7 +180,7 @@ describe('Plan service', () => { it('enters plan mode without starting a model turn and prepares the plan directory', async () => { const mkdir = vi.fn().mockResolvedValue(undefined); - const writeText = vi.fn().mockResolvedValue(0); + const writeText = vi.fn().mockResolvedValue(textFileStat()); const cwd = await makeTempDir('kimi-plan-entry-'); useFakes(createPlanFakes({ mkdir, writeText })); profile.update({ cwd }); @@ -192,7 +200,7 @@ describe('Plan service', () => { it('derives the plan path from the agent homedir on enter and restore', async () => { const cwd = await makeTempDir('kimi-plan-path-'); useFakes(createPlanFakes({ - writeText: vi.fn(async (_path: string, _content: string): Promise => {}), + writeText: vi.fn(async () => textFileStat()), })); profile.update({ cwd }); await plan.enter('stable-plan'); @@ -219,7 +227,7 @@ describe('Plan service', () => { it('keeps the plan path under the agent homedir when the profile cwd is empty', async () => { useFakes(createPlanFakes({ - writeText: vi.fn(async (_path: string, _content: string): Promise => {}), + writeText: vi.fn(async () => textFileStat()), })); profile.update({ cwd: '' }); @@ -475,10 +483,17 @@ describe('Plan service', () => { async (toolName) => { const files = new Map(); const readText = vi.fn(async (path: string) => files.get(path) ?? ''); - const writeText = vi.fn(async (path: string, content: string): Promise => { + const writeText = vi.fn(async (path: string, content: string) => { files.set(path, content); + return textFileStat(content); + }); + const stat = vi.fn(async (path: string) => { + const content = files.get(path); + return content === undefined + ? { isFile: false, isDirectory: true, size: 0 } + : textFileStat(content); }); - useFakes(createPlanFakes({ readText, writeText })); + useFakes(createPlanFakes({ readText, writeText, stat })); const cwd = await makeTempDir('kimi-plan-write-tool-'); useTools([toolName]); profile.update({ cwd }); @@ -516,8 +531,9 @@ describe('Plan service', () => { it('keeps explicit deny rules above active plan file writes', async () => { const files = new Map(); - const writeText = vi.fn(async (path: string, content: string): Promise => { + const writeText = vi.fn(async (path: string, content: string) => { files.set(path, content); + return textFileStat(content); }); useFakes(createPlanFakes({ writeText })); const cwd = await makeTempDir('kimi-plan-deny-write-'); diff --git a/packages/agent-core-v2/test/agent/profile/binding.test.ts b/packages/agent-core-v2/test/agent/profile/binding.test.ts index 8e2f4545c9..2bf212f1e3 100644 --- a/packages/agent-core-v2/test/agent/profile/binding.test.ts +++ b/packages/agent-core-v2/test/agent/profile/binding.test.ts @@ -52,6 +52,16 @@ function createAtomicDocumentStore(): AtomicDocumentStore { set: async (scope: string, key: string, value: T) => { documents.set(documentKey(scope, key), structuredClone(value)); }, + update: async (scope: string, key: string, mutate: (current: T | undefined) => T | Promise) => { + const next = await mutate(documents.get(documentKey(scope, key)) as T | undefined); + documents.set(documentKey(scope, key), structuredClone(next)); + return next; + }, + withExclusiveKeyMutation: async ( + _scope: string, + _key: string, + mutation: () => Promise, + ) => mutation(), delete: async (scope: string, key: string) => { documents.delete(documentKey(scope, key)); }, diff --git a/packages/agent-core-v2/test/agent/task/persist.test.ts b/packages/agent-core-v2/test/agent/task/persist.test.ts index 8054fe6876..08e4a72465 100644 --- a/packages/agent-core-v2/test/agent/task/persist.test.ts +++ b/packages/agent-core-v2/test/agent/task/persist.test.ts @@ -64,7 +64,13 @@ beforeEach(async () => { ix.set(IAtomicDocumentStore, new SyncDescriptor(JsonAtomicDocumentStore)); docs = ix.get(IAtomicDocumentStore); bytes = ix.get(IFileSystemStorageService); - persistence = new AgentTaskPersistence(sessionDir, SESSION_SCOPE, docs, bytes); + persistence = new AgentTaskPersistence( + sessionDir, + SESSION_SCOPE, + docs, + bytes, + undefined, + ); }); afterEach(async () => { @@ -77,7 +83,13 @@ describe('AgentTaskPersistence', () => { scope: string, fallbackRoot?: { readonly dir: string; readonly scope: string }, ): AgentTaskPersistence { - return new AgentTaskPersistence(join(sessionDir, scope), scope, docs, bytes, fallbackRoot); + return new AgentTaskPersistence( + join(sessionDir, scope), + scope, + docs, + bytes, + fallbackRoot, + ); } function sessionRoot(): { readonly dir: string; readonly scope: string } { diff --git a/packages/agent-core-v2/test/agent/task/stubs.ts b/packages/agent-core-v2/test/agent/task/stubs.ts index 66f59de320..c8c696099a 100644 --- a/packages/agent-core-v2/test/agent/task/stubs.ts +++ b/packages/agent-core-v2/test/agent/task/stubs.ts @@ -31,5 +31,6 @@ export function createAgentTaskPersistence(homedir: string): AgentTaskPersistenc TASK_TEST_AGENT_SCOPE, new JsonAtomicDocumentStore(storage), storage, + undefined, ); } diff --git a/packages/agent-core-v2/test/agent/task/taskService.test.ts b/packages/agent-core-v2/test/agent/task/taskService.test.ts index f873dfeb97..e409a7c2ac 100644 --- a/packages/agent-core-v2/test/agent/task/taskService.test.ts +++ b/packages/agent-core-v2/test/agent/task/taskService.test.ts @@ -163,6 +163,14 @@ describe('AgentTaskService', () => { await svc.stop(id); }); + it('registerTask rejects new work after close begins', () => { + const svc = ix.get(IAgentTaskService); + + svc.beginClose(); + + expect(() => svc.registerTask(fakeProcessTask())).toThrow('Agent task service is closing.'); + }); + function stubTaskConfig(value: unknown): void { ix.stub(IConfigService, { get: ((domain: string) => (domain === 'task' ? value : undefined)) as IConfigService['get'], diff --git a/packages/agent-core-v2/test/agent/task/tools/task-tools.test.ts b/packages/agent-core-v2/test/agent/task/tools/task-tools.test.ts index 97051a38b3..ff2e63ece6 100644 --- a/packages/agent-core-v2/test/agent/task/tools/task-tools.test.ts +++ b/packages/agent-core-v2/test/agent/task/tools/task-tools.test.ts @@ -172,6 +172,10 @@ class FakeTaskService implements IAgentTaskService { persistOutput(_taskId: string): void {} + beginClose(): void {} + + async flushPersistence(): Promise {} + async getOutputSnapshot( taskId: string, _maxPreviewBytes: number, diff --git a/packages/agent-core-v2/test/agent/toolExecutor/toolExecutor.test.ts b/packages/agent-core-v2/test/agent/toolExecutor/toolExecutor.test.ts index 619d3d1ab0..4a62449a63 100644 --- a/packages/agent-core-v2/test/agent/toolExecutor/toolExecutor.test.ts +++ b/packages/agent-core-v2/test/agent/toolExecutor/toolExecutor.test.ts @@ -7,6 +7,7 @@ import { SyncDescriptor } from '#/_base/di/descriptors'; import { DisposableStore } from '#/_base/di/lifecycle'; import { createServices, type TestInstantiationService } from '#/_base/di/test'; import { + attachToolFileRevision, ToolAccesses, type ExecutableTool, type ExecutableToolContext, @@ -14,6 +15,8 @@ import { type ToolExecution, type ToolResult, type ToolUpdate, + makeToolFileRevision, + toolFileRevision, } from '#/tool/toolContract'; import { IAgentToolExecutorService } from '#/agent/toolExecutor/toolExecutor'; import { AgentToolExecutorService, parseToolCallArguments } from '#/agent/toolExecutor/toolExecutorService'; @@ -693,6 +696,27 @@ describe('AgentToolExecutorService', () => { ]); }); + it('passes internal file revisions to did-hooks without serializing them', async () => { + const revision = makeToolFileRevision('/tmp/a.txt', { + size: 5, + mtimeMs: 1000, + ino: 1, + }); + const tool = new TestTool('read-like', { + result: attachToolFileRevision({ output: 'hello' }, revision), + }); + registry.register(tool); + let observed; + executor.hooks.onDidExecuteTool.register('observe-file-revision', async (ctx) => { + observed = ctx.result[toolFileRevision]; + }); + + const results = await execute([toolCall('call_read', 'read-like', {})]); + + expect(observed).toEqual(revision); + expect(JSON.stringify(results)).not.toContain('/tmp/a.txt'); + }); + it('onDidExecuteTool can replace the final tool result', async () => { const tool = new TestTool('echo'); registry.register(tool); diff --git a/packages/agent-core-v2/test/app/bootstrap/bootstrapService.test.ts b/packages/agent-core-v2/test/app/bootstrap/bootstrapService.test.ts index d3272107e0..aa3e483f9e 100644 --- a/packages/agent-core-v2/test/app/bootstrap/bootstrapService.test.ts +++ b/packages/agent-core-v2/test/app/bootstrap/bootstrapService.test.ts @@ -3,8 +3,12 @@ import { beforeEach, describe, expect, it } from 'vitest'; import { InstantiationType } from '#/_base/di/extensions'; import { LifecycleScope, _clearScopedRegistryForTests, registerScopedService } from '#/_base/di/scope'; import { createScopedTestHost } from '#/_base/di/test'; -import { IBootstrapService, bootstrapSeed, resolveBootstrapOptions } from '#/app/bootstrap/bootstrap'; -import { bootstrap } from '#/app/bootstrap/bootstrap'; +import { + IBootstrapService, + bootstrap, + bootstrapSeed, + resolveBootstrapOptions, +} from '#/app/bootstrap/bootstrap'; import { BootstrapService } from '#/app/bootstrap/bootstrapService'; import { FileStorageService } from '#/persistence/backends/node-fs/fileStorageService'; import { IFileSystemStorageService } from '#/persistence/interface/storage'; @@ -26,10 +30,23 @@ describe('BootstrapService (scoped)', () => { const svc = host.app.accessor.get(IBootstrapService); expect(svc.homeDir).toBe('/tmp/kimi-home'); expect(svc.configPath).toBe('/tmp/kimi-home/config.toml'); + expect(svc.configKey).toBe('config.toml'); expect(svc.sessionsDir).toBe('/tmp/kimi-home/sessions'); host.dispose(); }); + it('addresses a custom config path relative to the storage root', () => { + const host = createScopedTestHost( + bootstrapSeed({ + homeDir: '/tmp/kimi-home', + configPath: '/tmp/custom/config.toml', + }), + ); + const svc = host.app.accessor.get(IBootstrapService); + expect(svc.configKey).toBe('../custom/config.toml'); + host.dispose(); + }); + it('getEnv reads from the seeded env bag', () => { const host = createScopedTestHost(bootstrapSeed({ env: { FOO: 'bar' } })); const svc = host.app.accessor.get(IBootstrapService); diff --git a/packages/agent-core-v2/test/app/config/configFileMutex.test.ts b/packages/agent-core-v2/test/app/config/configFileMutex.test.ts new file mode 100644 index 0000000000..5ec2f68572 --- /dev/null +++ b/packages/agent-core-v2/test/app/config/configFileMutex.test.ts @@ -0,0 +1,159 @@ +/** + * Scenario: config.toml atomic read-modify-write. + * + * Two independent `ConfigService` instances share one storage root on the real + * filesystem. The atomic-document Store owns cross-process exclusion, so + * interleaved writes merge without lost updates and `ConfigService` never + * handles lock paths or lock services itself. Watch-based reloads ride real + * chokidar (150ms debounce), so assertions poll with real timers. + */ + +import { existsSync, readFileSync } from 'node:fs'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, relative } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { SyncDescriptor } from '#/_base/di/descriptors'; +import { DisposableStore } from '#/_base/di/lifecycle'; +import { TestInstantiationService } from '#/_base/di/test'; +import { ILogService } from '#/_base/log/log'; +import { IBootstrapService } from '#/app/bootstrap/bootstrap'; +import { IConfigRegistry, IConfigService } from '#/app/config/config'; +import { ConfigRegistry, ConfigService } from '#/app/config/configService'; +import { CrossProcessLockService } from '#/os/backends/node-local/crossProcessLockService'; +import { + CrossProcessLockErrorCode, + type ICrossProcessLockService, + type ICrossProcessLockHandle, +} from '#/os/interface/crossProcessLock'; +import { TomlAtomicDocumentStore } from '#/persistence/backends/node-fs/atomicDocumentStore'; +import { FileStorageService } from '#/persistence/backends/node-fs/fileStorageService'; +import { IAtomicTomlDocumentStore } from '#/persistence/interface/atomicDocumentStore'; +import { + IFileSystemStorageService, + StorageErrors, +} from '#/persistence/interface/storage'; + +import { stubLog } from '../../_base/log/stubs'; +import { stubBootstrap } from '../bootstrap/stubs'; + +const realSleep = (ms: number): Promise => + new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); + +async function waitFor(cond: () => boolean, timeoutMs = 5_000): Promise { + const start = Date.now(); + for (;;) { + if (cond()) return; + if (Date.now() - start > timeoutMs) throw new Error('waitFor timed out'); + await realSleep(25); + } +} + +describe('ConfigService config.toml lock-in-RMW', () => { + let homeDir: string; + let disposables: DisposableStore; + + beforeEach(async () => { + homeDir = await mkdtemp(join(tmpdir(), 'config-rmw-')); + disposables = new DisposableStore(); + }); + + afterEach(async () => { + disposables.dispose(); + await rm(homeDir, { recursive: true, force: true }); + }); + + function createContainer( + lock: ICrossProcessLockService = new CrossProcessLockService(), + configPath = join(homeDir, 'config.toml'), + ): IConfigService { + const ix = disposables.add(new TestInstantiationService()); + ix.stub(ILogService, stubLog()); + ix.stub(IBootstrapService, { + ...stubBootstrap(homeDir, {}), + configPath, + configKey: relative(homeDir, configPath), + }); + ix.stub( + IFileSystemStorageService, + new FileStorageService(homeDir, undefined, undefined, lock), + ); + ix.set(IAtomicTomlDocumentStore, new SyncDescriptor(TomlAtomicDocumentStore)); + ix.set(IConfigRegistry, new SyncDescriptor(ConfigRegistry)); + ix.set(IConfigService, new SyncDescriptor(ConfigService)); + return ix.get(IConfigService); + } + + it('two containers interleave set()s without losing section updates', async () => { + await writeFile(join(homeDir, 'config.toml'), '[hand_written]\nkeep = "me"\n'); + const a = createContainer(); + const b = createContainer(); + await a.ready; + await b.ready; + + const rounds = 4; + for (let i = 0; i < rounds; i++) { + await Promise.all([a.set(`alphaSide${i}`, { v: i }), b.set(`betaSide${i}`, { v: i })]); + } + + const toml = readFileSync(join(homeDir, 'config.toml'), 'utf8'); + expect(toml).toContain('[hand_written]'); + for (let i = 0; i < rounds; i++) { + expect(toml).toContain(`[alpha_side${i}]`); + expect(toml).toContain(`[beta_side${i}]`); + } + + await a.reload(); + await b.reload(); + expect(b.get('alphaSide0')).toEqual({ v: 0 }); + expect(a.get('betaSide3')).toEqual({ v: 3 }); + }); + + it('writes and locks a custom config path at its actual location', async () => { + const configPath = join(homeDir, 'nested', 'custom.toml'); + const config = createContainer(new CrossProcessLockService(), configPath); + await config.set('alphaSection', { one: 1 }); + + expect(readFileSync(configPath, 'utf8')).toContain('[alpha_section]'); + expect(existsSync(join(homeDir, 'custom.toml'))).toBe(false); + expect(existsSync(`${configPath}.lock`)).toBe(true); + }); + + it('fails set() with storage.locked while another holder is stuck, leaving config.toml intact', async () => { + let nowValue = 1_000_000; + let lockSeq = 0; + const victim = new CrossProcessLockService({ + selfPid: 1001, + instanceId: 'victim', + now: () => nowValue, + newLockId: () => `victim-${++lockSeq}`, + sleep: (ms) => { + nowValue += ms; + return Promise.resolve(); + }, + }); + const config = createContainer(victim); + await config.set('seedSection', { ok: true }); + const before = readFileSync(join(homeDir, 'config.toml'), 'utf8'); + + const attacker = new CrossProcessLockService({ + selfPid: 2002, + instanceId: 'attacker', + newLockId: () => 'attacker-lock', + }); + const lockPath = join(homeDir, 'config.toml.lock'); + const handle: ICrossProcessLockHandle = await attacker.acquire(lockPath); + try { + await expect(config.set('blockedSection', { no: true })).rejects.toMatchObject({ + code: StorageErrors.codes.STORAGE_LOCKED, + cause: { code: CrossProcessLockErrorCode.WaitTimeout }, + }); + expect(readFileSync(join(homeDir, 'config.toml'), 'utf8')).toBe(before); + } finally { + handle.release(); + } + expect(existsSync(lockPath)).toBe(true); + }); +}); diff --git a/packages/agent-core-v2/test/app/edit/tools/edit.test.ts b/packages/agent-core-v2/test/app/edit/tools/edit.test.ts index 5bbaf6853b..75327364b7 100644 --- a/packages/agent-core-v2/test/app/edit/tools/edit.test.ts +++ b/packages/agent-core-v2/test/app/edit/tools/edit.test.ts @@ -27,7 +27,12 @@ import { HostFileSystem } from '#/os/backends/node-local/hostFsService'; import { HostFsError, OsFsErrors } from '#/os/interface/hostFsErrors'; import { IHostFileSystem } from '#/os/interface/hostFileSystem'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; -import type { ExecutableToolContext, ExecutableToolResult, ToolExecution } from '#/tool/toolContract'; +import { + type ExecutableToolContext, + type ExecutableToolResult, + type ToolExecution, + toolFileRevision, +} from '#/tool/toolContract'; const signal = new AbortController().signal; const PERMISSIVE_WORKSPACE = stubWorkspaceContext('/'); @@ -52,13 +57,25 @@ function createSpiedEditFs( options: { readText?: ReturnType; writeText?: ReturnType; + stat?: ReturnType; } = {}, ) { const readText = options.readText ?? vi.fn(async () => ''); const writeText = options.writeText ?? vi.fn(async () => undefined); - const stat = vi.fn(async () => ({ isFile: true, isDirectory: false, size: 0 })); - const fs = { readText, writeText, stat } as unknown as IHostFileSystem; - return { fs, readText, writeText }; + const writeTextWithStat = async (path: string, data: string) => { + await (writeText as (path: string, data: string) => Promise)(path, data); + return { + isFile: true, + isDirectory: false, + size: Buffer.byteLength(data), + mtimeMs: 1000, + ino: 1, + }; + }; + const stat = + options.stat ?? vi.fn(async () => ({ isFile: true, isDirectory: false, size: 0 })); + const fs = { readText, writeText: writeTextWithStat, stat } as unknown as IHostFileSystem; + return { fs, readText, writeText, stat }; } function buildTool( @@ -211,6 +228,45 @@ describe('EditTool', () => { expect(result.output).toContain('Replaced 1 occurrence'); expect(writeText).toHaveBeenCalledWith('/tmp/a.txt', 'alpha gamma'); + expect(result[toolFileRevision]).toEqual({ + path: '/tmp/a.txt', + size: 11, + mtimeMs: 1000, + ino: 1, + }); + }); + + it('fails closed when the file revision changes between reading and committing the edit', async () => { + const originalStat = { + isFile: true, + isDirectory: false, + size: 10, + mtimeMs: 1000, + ino: 1, + }; + const changedStat = { ...originalStat, size: 11, mtimeMs: 1001 }; + const stat = vi + .fn() + .mockResolvedValueOnce(originalStat) + .mockResolvedValueOnce(originalStat) + .mockResolvedValueOnce(changedStat); + const writeText = vi.fn().mockResolvedValue(undefined); + const { fs } = createSpiedEditFs({ + readText: vi.fn().mockResolvedValue('alpha beta'), + writeText, + stat, + }); + const tool = buildTool(fs, createTestEnv(), PERMISSIVE_WORKSPACE); + + const result = await execute(tool, { + path: '/tmp/a.txt', + old_string: 'beta', + new_string: 'gamma', + }); + + expect(result.isError).toBe(true); + expect(result.output).toContain('changed on disk'); + expect(writeText).not.toHaveBeenCalled(); }); it('expands leading tilde paths using the kaos home directory', async () => { diff --git a/packages/agent-core-v2/test/app/externalHooksRunner/integration.test.ts b/packages/agent-core-v2/test/app/externalHooksRunner/integration.test.ts index 43bcbcabbb..794886fac0 100644 --- a/packages/agent-core-v2/test/app/externalHooksRunner/integration.test.ts +++ b/packages/agent-core-v2/test/app/externalHooksRunner/integration.test.ts @@ -187,11 +187,13 @@ function stubSessionLifecycle(): ISessionLifecycleService { hooks: createHooks([ 'onDidCreateSession', 'onWillCloseSession', + 'onWillReleaseSession', ]), onDidCreateSession: Event.None as ISessionLifecycleService['onDidCreateSession'], onDidCloseSession: Event.None as ISessionLifecycleService['onDidCloseSession'], onDidArchiveSession: Event.None as ISessionLifecycleService['onDidArchiveSession'], onDidForkSession: Event.None as ISessionLifecycleService['onDidForkSession'], + beginClose: async () => {}, create: async () => { throw new Error('not implemented'); }, @@ -199,6 +201,7 @@ function stubSessionLifecycle(): ISessionLifecycleService { list: () => [], resume: async () => undefined, close: async () => {}, + closeAll: async () => {}, archive: async () => {}, restore: async () => undefined, fork: async () => { diff --git a/packages/agent-core-v2/test/app/flag/flag.test.ts b/packages/agent-core-v2/test/app/flag/flag.test.ts index 2749afcb05..658b8de601 100644 --- a/packages/agent-core-v2/test/app/flag/flag.test.ts +++ b/packages/agent-core-v2/test/app/flag/flag.test.ts @@ -18,9 +18,11 @@ import { IAtomicTomlDocumentStore } from '#/persistence/interface/atomicDocument import { TomlAtomicDocumentStore } from '#/persistence/backends/node-fs/atomicDocumentStore'; import { InMemoryStorageService } from '#/persistence/backends/memory/inMemoryStorageService'; import { IFileSystemStorageService } from '#/persistence/interface/storage'; +import { ICrossProcessLockService } from '#/os/interface/crossProcessLock'; import { stubBootstrap } from '../bootstrap/stubs'; import { stubLog } from '../../_base/log/stubs'; +import { stubCrossProcessLock } from '../../os/stubs'; const exampleFlag: FlagDefinitionInput = { id: 'example_flag', @@ -75,6 +77,7 @@ describe('FlagService', () => { ix.stub(IFileSystemStorageService, new InMemoryStorageService()); ix.set(IAtomicTomlDocumentStore, new SyncDescriptor(TomlAtomicDocumentStore)); ix.set(IConfigRegistry, new SyncDescriptor(ConfigRegistry)); + ix.stub(ICrossProcessLockService, stubCrossProcessLock()); ix.set(IConfigService, new SyncDescriptor(ConfigService)); ix.set(IFlagRegistry, new SyncDescriptor(FlagRegistryService)); ix.set(IFlagService, new SyncDescriptor(FlagService)); diff --git a/packages/agent-core-v2/test/app/sessionExport/sessionExport.test.ts b/packages/agent-core-v2/test/app/sessionExport/sessionExport.test.ts index 2d78765bb6..83b6c20150 100644 --- a/packages/agent-core-v2/test/app/sessionExport/sessionExport.test.ts +++ b/packages/agent-core-v2/test/app/sessionExport/sessionExport.test.ts @@ -852,9 +852,11 @@ function registerSessionExportServices( onDidCloseSession: noopEvent, onDidArchiveSession: noopEvent, onDidForkSession: noopEvent, + beginClose: async () => {}, hooks: createHooks([ 'onDidCreateSession', 'onWillCloseSession', + 'onWillReleaseSession', ]), create: async () => { throw new Error('create should not be called by session export'); @@ -863,6 +865,7 @@ function registerSessionExportServices( list: () => (options.lifecycleHandle === undefined ? [] : [options.lifecycleHandle]), resume: async () => options.lifecycleHandle, close: async () => {}, + closeAll: async () => {}, archive: async () => {}, restore: async () => options.lifecycleHandle, fork: async () => { diff --git a/packages/agent-core-v2/test/app/sessionLifecycle/sessionLifecycle.test.ts b/packages/agent-core-v2/test/app/sessionLifecycle/sessionLifecycle.test.ts index ced4de63cc..6ebba27119 100644 --- a/packages/agent-core-v2/test/app/sessionLifecycle/sessionLifecycle.test.ts +++ b/packages/agent-core-v2/test/app/sessionLifecycle/sessionLifecycle.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { tryAcquireKernelFileLock } from '@moonshot-ai/kernel-file-lock'; import { mkdtemp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -8,18 +9,21 @@ import { InstantiationType } from '#/_base/di/extensions'; import { Disposable } from '#/_base/di/lifecycle'; import { type IAgentScopeHandle, + type ISessionScopeHandle, LifecycleScope, _clearScopedRegistryForTests, registerScopedService, } from '#/_base/di/scope'; import { type ScopedTestHost, createScopedTestHost, stubPair } from '#/_base/di/test'; import { Event } from '#/_base/event'; +import { ILogService } from '#/_base/log/log'; import { IBootstrapService } from '#/app/bootstrap/bootstrap'; import { IConfigService } from '#/app/config/config'; import { IHostEnvironment } from '#/os/interface/hostEnvironment'; import { IHostFileSystem } from '#/os/interface/hostFileSystem'; import { HostFileSystem } from '#/os/backends/node-local/hostFsService'; import { IEventService } from '#/app/event/event'; +import { IFlagService } from '#/app/flag/flag'; import { IAgentLifecycleService, MAIN_AGENT_ID, @@ -40,14 +44,33 @@ import { ISessionAgentProfileCatalog } from '#/session/sessionAgentProfileCatalo import { ISessionIndex, type SessionSummary } from '#/app/sessionIndex/sessionIndex'; import { IAppendLogStore } from '#/persistence/interface/appendLogStore'; import { IAtomicDocumentStore } from '#/persistence/interface/atomicDocumentStore'; +import { JsonAtomicDocumentStore } from '#/persistence/backends/node-fs/atomicDocumentStore'; +import { ISessionWriteAdmission } from '#/persistence/interface/sessionWriteAdmission'; +import { IStorageWriteAdmission } from '#/persistence/interface/storageWriteAdmission'; +import { AppendLogStore } from '#/persistence/backends/node-fs/appendLogStore'; +import { FileStorageService } from '#/persistence/backends/node-fs/fileStorageService'; +import { StorageWriteAdmissionService } from '#/persistence/backends/node-fs/storageWriteAdmissionService'; import { IWorkspaceLocalConfigService } from '#/app/workspaceLocalConfig/workspaceLocalConfig'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; import { SessionWorkspaceContextService } from '#/session/workspaceContext/workspaceContextService'; import { IWorkspaceRegistry, type Workspace } from '#/app/workspaceRegistry/workspaceRegistry'; import { encodeWorkDirKey } from '#/_base/utils/workdir-slug'; import { ISessionContext } from '#/session/sessionContext/sessionContext'; +import { ISessionLeaseService } from '#/session/sessionLease/sessionLease'; +import { + ISessionLeaseContactProvider, + SessionLeaseContactProvider, +} from '#/session/sessionLease/sessionLeaseContactProvider'; import { ITelemetryService } from '#/app/telemetry/telemetry'; import { Error2, ErrorCodes } from '#/errors'; +import { + CrossProcessLockErrorCode, + ICrossProcessLockService, +} from '#/os/interface/crossProcessLock'; +import { CrossProcessLockService } from '#/os/backends/node-local/crossProcessLockService'; +import { stubCrossProcessLock } from '../../os/stubs'; +import { stubFlag } from '../flag/stubs'; +import { stubLog } from '../../_base/log/stubs'; import { recordingTelemetry, type TelemetryRecord } from '../../app/telemetry/stubs'; function bootstrapStub(): IBootstrapService { @@ -279,6 +302,8 @@ function atomicDocumentStoreStub(): IAtomicDocumentStore { _serviceBrand: undefined, get: () => Promise.resolve(undefined), set: () => Promise.resolve(), + update: (_scope, _key, mutate) => Promise.resolve(mutate(undefined)), + withExclusiveKeyMutation: (_scope, _key, mutation) => mutation(), delete: () => Promise.resolve(), list: () => Promise.resolve([]), watch: () => (_listener) => ({ dispose: () => {} }), @@ -389,8 +414,18 @@ function tick(): Promise { return new Promise((resolve) => setTimeout(resolve, 0)); } -class NoopSessionExternalHooksService implements ISessionExternalHooksService { +let disposedSessionServices = 0; + +class NoopSessionExternalHooksService + extends Disposable + implements ISessionExternalHooksService +{ declare readonly _serviceBrand: undefined; + + override dispose(): void { + disposedSessionServices++; + super.dispose(); + } } let recordedSessionHookEvents: string[] = []; @@ -424,6 +459,7 @@ describe('SessionLifecycleService', () => { let tmpRoots: string[]; beforeEach(() => { + disposedSessionServices = 0; recordedSessionHookEvents = []; telemetryRecords = []; tmpRoots = []; @@ -477,6 +513,11 @@ describe('SessionLifecycleService', () => { stubPair(IWorkspaceLocalConfigService, workspaceLocalConfigStub()), stubPair(ITelemetryService, recordingTelemetry(telemetryRecords)), stubPair(ICronTaskPersistence, cronStoreStub()), + stubPair(ILogService, stubLog()), + stubPair(IFlagService, stubFlag(false)), + stubPair(ICrossProcessLockService, stubCrossProcessLock()), + stubPair(IStorageWriteAdmission, new StorageWriteAdmissionService()), + stubPair(ISessionLeaseContactProvider, new SessionLeaseContactProvider()), ...extra, ]); return host.app.accessor.get(ISessionLifecycleService); @@ -499,6 +540,22 @@ describe('SessionLifecycleService', () => { expect(svc.get('s1')).toBeUndefined(); }); + it('rejects new session materialization after close admission begins', async () => { + const svc = build(); + await svc.beginClose(); + + await expect(svc.create({ sessionId: 's1', workDir: '/tmp/proj' })).rejects.toMatchObject({ + code: ErrorCodes.SESSION_CLOSED, + }); + await expect(svc.resume('s1')).rejects.toMatchObject({ code: ErrorCodes.SESSION_CLOSED }); + await expect( + svc.fork({ sourceSessionId: 's1', newSessionId: 's2' }), + ).rejects.toMatchObject({ code: ErrorCodes.SESSION_CLOSED }); + await expect( + svc.createChild({ sourceSessionId: 's1', newSessionId: 's3' }), + ).rejects.toMatchObject({ code: ErrorCodes.SESSION_CLOSED }); + }); + it('create seeds identity and materializes metadata', async () => { const svc = build(); const h = await svc.create({ sessionId: 's1', workDir: '/tmp/proj' }); @@ -962,6 +1019,21 @@ describe('SessionLifecycleService', () => { expect(closed).toEqual(['s1']); }); + it('logs and continues when a release hook fails', async () => { + const svc = build(); + const closed: string[] = []; + svc.onDidCloseSession((e) => closed.push(e.sessionId)); + svc.hooks.onWillReleaseSession.register('failing-hook', async () => { + throw new Error('hook failed'); + }); + await svc.create({ sessionId: 's1', workDir: '/tmp/proj' }); + + await expect(svc.close('s1')).resolves.toBeUndefined(); + + expect(closed).toEqual(['s1']); + expect(svc.get('s1')).toBeUndefined(); + }); + it('fires onDidArchiveSession when a session is archived', async () => { const svc = build([ stubPair(IAgentLifecycleService, { @@ -972,10 +1044,16 @@ describe('SessionLifecycleService', () => { } as unknown as IAgentLifecycleService), ]); const archived: string[] = []; + const releasing: string[] = []; svc.onDidArchiveSession((e) => archived.push(e.sessionId)); + svc.hooks.onWillReleaseSession.register('test', async (event, next) => { + releasing.push(`${event.reason}:${event.sessionId}`); + await next(); + }); await svc.create({ sessionId: 's1', workDir: '/tmp/proj' }); await svc.archive('s1'); expect(archived).toEqual(['s1']); + expect(releasing).toEqual(['archive:s1']); }); describe('additional dirs', () => { @@ -1288,6 +1366,377 @@ describe('SessionLifecycleService', () => { }); }); + describe('session lease', () => { + function realInstanceSeeds( + root: string, + over: ReturnType[] = [], + ): ReturnType[] { + return [ + stubPair(IBootstrapService, tmpBootstrapStub(root)), + stubPair(ICrossProcessLockService, new CrossProcessLockService()), + stubPair(IStorageWriteAdmission, new StorageWriteAdmissionService()), + ...over, + ]; + } + + function realAlsSeeds(root: string): { + seeds: ReturnType[]; + appendLog: AppendLogStore; + registry: StorageWriteAdmissionService; + storage: FileStorageService; + docs: JsonAtomicDocumentStore; + } { + const registry = new StorageWriteAdmissionService(); + const locks = new CrossProcessLockService(); + const storage = new FileStorageService(root, undefined, undefined, locks, registry); + const appendLog = new AppendLogStore(storage); + const docs = new JsonAtomicDocumentStore(storage); + return { + seeds: [ + stubPair(IBootstrapService, tmpBootstrapStub(root)), + stubPair(ICrossProcessLockService, locks), + stubPair(IStorageWriteAdmission, registry), + stubPair(IAppendLogStore, appendLog), + stubPair(IAtomicDocumentStore, docs), + ], + appendLog, + registry, + storage, + docs, + }; + } + + function leaseFile(root: string, sessionId: string): string { + return join(root, 'session-leases', `${sessionId}.lock`); + } + + function leaseOwnerFile(root: string, sessionId: string): string { + return `${leaseFile(root, sessionId)}.owner.json`; + } + + async function createError( + svc: ISessionLifecycleService, + sessionId: string, + ): Promise { + return svc + .create({ sessionId, workDir: '/tmp/proj' }) + .then(() => { + throw new Error('expected create to fail'); + }, (error: unknown) => error as Error2); + } + + function writeStateDoc(docs: JsonAtomicDocumentStore, sessionId: string): Promise { + return docs.set(`sessions/wd_stub/${sessionId}`, 'state.json', { + id: sessionId, + version: 2, + cwd: '/tmp/proj', + createdAt: 1, + updatedAt: 1, + archived: false, + agents: {}, + custom: {}, + }); + } + + function poisonSessionAppend(storage: FileStorageService, sessionId: string, failure: Error): void { + const originalAppend = storage.append.bind(storage); + storage.append = async (...args) => { + if (args[0].startsWith(`sessions/wd_stub/${sessionId}`)) throw failure; + return originalAppend(...args); + }; + } + + async function expectLeaseFree(root: string, sessionId: string): Promise { + const successor = await new CrossProcessLockService().acquire(leaseFile(root, sessionId)); + successor.release(); + } + + async function expectLeaseReleased(root: string, sessionId: string): Promise { + await expect(stat(leaseFile(root, sessionId))).resolves.toBeDefined(); + await expect(stat(leaseOwnerFile(root, sessionId))).rejects.toThrow(); + } + + it('refuses to materialize a session live-held by another instance', async () => { + const root = await makeTmpRoot(); + const first = build(realInstanceSeeds(root)); + const firstHost = host!; + await first.create({ sessionId: 's1', workDir: '/tmp/proj' }); + try { + const second = build(realInstanceSeeds(root)); + const error = await createError(second, 's1'); + expect(error.code).toBe(ErrorCodes.SESSION_HELD_BY_PEER); + expect(error.details).toEqual({ kind: 'held-by-peer', phase: 'held-by-local-instance' }); + expect(telemetryRecords).toContainEqual({ + event: 'session_held_by_peer_returned', + properties: { session_id: 's1', phase: 'held-by-local-instance' }, + }); + } finally { + firstHost.dispose(); + } + }); + + it('reports the routable phase with the holder address', async () => { + const root = await makeTmpRoot(); + const first = build( + realInstanceSeeds(root, [ + stubPair( + ISessionLeaseContactProvider, + new SessionLeaseContactProvider(() => ({ + type: 'address', + address: 'http://127.0.0.1:5555', + })), + ), + ]), + ); + const firstHost = host!; + await first.create({ sessionId: 's1', workDir: '/tmp/proj' }); + try { + const second = build(realInstanceSeeds(root)); + const error = await createError(second, 's1'); + expect(error.code).toBe(ErrorCodes.SESSION_HELD_BY_PEER); + expect(error.details).toEqual({ + kind: 'held-by-peer', + phase: 'routable', + address: 'http://127.0.0.1:5555', + }); + } finally { + firstHost.dispose(); + } + }); + + it('reports the creating phase while the kernel holder publishes owner metadata', async () => { + const root = await makeTmpRoot(); + const handle = tryAcquireKernelFileLock(leaseFile(root, 's1'))!; + try { + const svc = build(realInstanceSeeds(root)); + const error = await createError(svc, 's1'); + expect(error.code).toBe(ErrorCodes.SESSION_HELD_BY_PEER); + expect(error.details).toEqual({ + kind: 'held-by-peer', + phase: 'creating', + retry_after_ms: 1000, + }); + } finally { + handle.release(); + } + }); + + it('close returns with the just-appended journal tail durable and the lease released', async () => { + const root = await makeTmpRoot(); + const { seeds, appendLog } = realAlsSeeds(root); + const svc = build(seeds); + await svc.create({ sessionId: 's1', workDir: '/tmp/proj' }); + const agentScopeStr = 'sessions/wd_stub/s1/agents/main'; + // Mirror the wire handle lifecycle: release retires the buffer and its + // final flush is fire-and-forget — close must still await it. + const released = appendLog.acquire(agentScopeStr, 'wire.jsonl'); + appendLog.append(agentScopeStr, 'wire.jsonl', { tail: true }); + released.dispose(); + + await svc.close('s1'); + + const disk = await readFile(join(root, agentScopeStr, 'wire.jsonl'), 'utf8'); + expect(disk).toBe('{"tail":true}\n'); + await expectLeaseReleased(root, 's1'); + }); + + it('keeps the session lease until release hooks settle', async () => { + const root = await makeTmpRoot(); + const svc = build(realInstanceSeeds(root)); + await svc.create({ sessionId: 's1', workDir: '/tmp/proj' }); + let enterHook!: () => void; + const hookEntered = new Promise((resolve) => { + enterHook = resolve; + }); + let releaseHook!: () => void; + const hookGate = new Promise((resolve) => { + releaseHook = resolve; + }); + svc.hooks.onWillReleaseSession.register('lease-test', async (event, next) => { + expect(event).toEqual({ sessionId: 's1', reason: 'close' }); + expect(disposedSessionServices).toBe(1); + enterHook(); + await hookGate; + await next(); + }); + + const closing = svc.close('s1'); + await hookEntered; + + try { + await expect( + new CrossProcessLockService().acquire(leaseFile(root, 's1')), + ).rejects.toMatchObject({ code: CrossProcessLockErrorCode.Held }); + } finally { + releaseHook(); + await closing; + } + await expectLeaseFree(root, 's1'); + }); + + it('seals new writes and waits for an admitted write before releasing the lease', async () => { + const root = await makeTmpRoot(); + const svc = build(realInstanceSeeds(root)); + const handle = await svc.create({ sessionId: 's1', workDir: '/tmp/proj' }); + const writeAdmission = handle.accessor.get(ISessionWriteAdmission); + let finishWrite!: () => void; + const writeBlocked = new Promise((resolve) => { + finishWrite = resolve; + }); + let enterWrite!: () => void; + const writeEntered = new Promise((resolve) => { + enterWrite = resolve; + }); + const write = writeAdmission.withPhysicalWrite(async () => { + enterWrite(); + await writeBlocked; + }); + await writeEntered; + + let closeSettled = false; + const closing = svc.close('s1').finally(() => { + closeSettled = true; + }); + let sealed = false; + for (let attempt = 0; attempt < 10 && !sealed; attempt++) { + try { + writeAdmission.assertCanWriteNow(); + } catch (error) { + expect(error).toMatchObject({ code: ErrorCodes.SESSION_LEASE_LOST }); + sealed = true; + } + if (!sealed) await tick(); + } + + expect(sealed).toBe(true); + expect(closeSettled).toBe(false); + finishWrite(); + await write; + await closing; + await expectLeaseFree(root, 's1'); + }); + + it('abandons and releases the lease when the session durability barrier fails', async () => { + const root = await makeTmpRoot(); + const { seeds, appendLog, registry, storage, docs } = realAlsSeeds(root); + const svc = build(seeds); + await svc.create({ sessionId: 's1', workDir: '/tmp/proj' }); + await writeStateDoc(docs, 's1'); + const failure = new Error('durable append failed'); + poisonSessionAppend(storage, 's1', failure); + appendLog.append('sessions/wd_stub/s1/agents/main', 'wire.jsonl', { tail: true }); + + await expect(svc.close('s1')).rejects.toMatchObject({ + code: ErrorCodes.SESSION_DURABILITY_FAILED, + cause: failure, + details: { sessionId: 's1', stage: 'flush' }, + }); + expect(svc.get('s1')).toBeUndefined(); + await expect( + registry.withPhysicalWrite('sessions/wd_stub/s1/agents/main', async () => {}), + ).rejects.toMatchObject({ code: ErrorCodes.SESSION_LEASE_LOST }); + expect(await docs.get<{ custom?: { dirtyAbort?: { reason?: string } } }>( + 'sessions/wd_stub/s1', + 'state.json', + )).toMatchObject({ custom: { dirtyAbort: { reason: 'flush-failed' } } }); + await expectLeaseReleased(root, 's1'); + await expectLeaseFree(root, 's1'); + await expect(svc.close('s1')).resolves.toBeUndefined(); + }); + + it('reports an automatic dirty abort after a durability failure', async () => { + const root = await makeTmpRoot(); + const { seeds, appendLog, storage } = realAlsSeeds(root); + const svc = build(seeds); + await svc.create({ sessionId: 's1', workDir: '/tmp/proj' }); + const failure = new Error('durable append failed'); + poisonSessionAppend(storage, 's1', failure); + appendLog.append('sessions/wd_stub/s1/agents/main', 'wire.jsonl', { tail: true }); + + await expect(svc.close('s1')).rejects.toMatchObject({ + code: ErrorCodes.SESSION_DURABILITY_FAILED, + cause: failure, + }); + + await expectLeaseReleased(root, 's1'); + await expectLeaseFree(root, 's1'); + expect(telemetryRecords).toContainEqual({ + event: 'session_dirty_abort', + properties: { session_id: 's1', reason: 'flush-failed', sessionId: 's1' }, + }); + }); + + it('dirty-aborts an active session when closeAll hits a durability failure', async () => { + const root = await makeTmpRoot(); + const { seeds, appendLog, registry, storage, docs } = realAlsSeeds(root); + const svc = build(seeds); + await svc.create({ sessionId: 's1', workDir: '/tmp/proj' }); + await writeStateDoc(docs, 's1'); + poisonSessionAppend(storage, 's1', new Error('durable append failed')); + appendLog.append('sessions/wd_stub/s1/agents/main', 'wire.jsonl', { tail: true }); + + await svc.closeAll(); + + await expect( + registry.withPhysicalWrite('sessions/wd_stub/s1', async () => {}), + ).rejects.toMatchObject({ code: ErrorCodes.SESSION_LEASE_LOST }); + expect(await docs.get<{ custom?: { dirtyAbort?: { reason?: string } } }>( + 'sessions/wd_stub/s1', + 'state.json', + )).toMatchObject({ custom: { dirtyAbort: { reason: 'flush-failed' } } }); + await expectLeaseFree(root, 's1'); + }); + + it('shares one release when close and closeAll overlap', async () => { + const root = await makeTmpRoot(); + const { seeds, registry } = realAlsSeeds(root); + const svc = build(seeds); + await svc.create({ sessionId: 's1', workDir: '/tmp/proj' }); + + await Promise.all([svc.close('s1'), svc.closeAll()]); + + await expect( + registry.withPhysicalWrite('sessions/wd_stub/s1', async () => {}), + ).rejects.toMatchObject({ code: ErrorCodes.SESSION_LEASE_LOST }); + await expectLeaseFree(root, 's1'); + }); + + it('closing one session does not wait for another session append', async () => { + const root = await makeTmpRoot(); + const { seeds, appendLog, storage } = realAlsSeeds(root); + const svc = build(seeds); + await svc.create({ sessionId: 's1', workDir: '/tmp/proj' }); + await svc.create({ sessionId: 's10', workDir: '/tmp/proj' }); + let markBlockedStarted!: () => void; + const blockedStarted = new Promise((resolvePromise) => { + markBlockedStarted = resolvePromise; + }); + let releaseBlocked!: () => void; + const blockedGate = new Promise((resolvePromise) => { + releaseBlocked = resolvePromise; + }); + const originalAppend = storage.append.bind(storage); + storage.append = async (...args) => { + if (args[0].startsWith('sessions/wd_stub/s10')) { + markBlockedStarted(); + await blockedGate; + } + return originalAppend(...args); + }; + appendLog.append('sessions/wd_stub/s10/agents/main', 'wire.jsonl', { n: 2 }); + appendLog.append('sessions/wd_stub/s1/agents/main', 'wire.jsonl', { n: 1 }); + await blockedStarted; + + await svc.close('s1'); + await expectLeaseReleased(root, 's1'); + await expect(stat(leaseFile(root, 's10'))).resolves.toBeDefined(); + + releaseBlocked(); + await appendLog.flush('sessions/wd_stub/s10'); + await svc.close('s10'); + }); + }); + describe('defaultPlanMode bootstrap', () => { it('enters plan mode on a fresh session when config.defaultPlanMode is true', async () => { const { lifecycle, enter, create } = agentLifecycleCapturingPlanSpy(); diff --git a/packages/agent-core-v2/test/app/skillCatalog/skillRoots.test.ts b/packages/agent-core-v2/test/app/skillCatalog/skillRoots.test.ts index ae4e7735f6..62c1749b5d 100644 --- a/packages/agent-core-v2/test/app/skillCatalog/skillRoots.test.ts +++ b/packages/agent-core-v2/test/app/skillCatalog/skillRoots.test.ts @@ -5,8 +5,10 @@ import { join } from 'pathe'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { configuredRoots, projectRoots, userRoots } from '#/app/skillCatalog/skillRoots'; +import { HostFileSystem } from '#/os/backends/node-local/hostFsService'; describe('skillRoots', () => { + const fs = new HostFileSystem(); let root: string; beforeEach(async () => { @@ -26,7 +28,7 @@ describe('skillRoots', () => { await markGitRoot(); await mkdir(join(root, '.kimi-code/skills/commit'), { recursive: true }); - const roots = await projectRoots(root); + const roots = await projectRoots(fs, root); expect(roots.some((r) => r.path.endsWith('.kimi-code/skills') && r.source === 'project')).toBe( true, @@ -37,7 +39,7 @@ describe('skillRoots', () => { await markGitRoot(); await mkdir(join(root, '.agents/skills/review'), { recursive: true }); - const roots = await projectRoots(root); + const roots = await projectRoots(fs, root); expect(roots.some((r) => r.path.endsWith('.agents/skills') && r.source === 'project')).toBe( true, @@ -51,7 +53,7 @@ describe('skillRoots', () => { const child = join(root, 'src/pkg'); await mkdir(child, { recursive: true }); - const roots = await projectRoots(child); + const roots = await projectRoots(fs, child); expect(roots.some((r) => r.path.endsWith('.kimi-code/skills'))).toBe(true); }); @@ -61,7 +63,7 @@ describe('skillRoots', () => { await mkdir(join(root, '.kimi-code/skills'), { recursive: true }); await mkdir(join(root, '.agents/skills'), { recursive: true }); - const roots = await projectRoots(root); + const roots = await projectRoots(fs, root); const brandIdx = roots.findIndex((r) => r.path.endsWith('.kimi-code/skills')); const genericIdx = roots.findIndex((r) => r.path.endsWith('.agents/skills')); @@ -74,7 +76,7 @@ describe('skillRoots', () => { it('resolves the brand skills directory under homeDir', async () => { await mkdir(join(root, 'skills/notes'), { recursive: true }); - const roots = await userRoots(root, root); + const roots = await userRoots(fs, root, root); expect(roots.some((r) => r.path.endsWith('/skills') && r.source === 'user')).toBe(true); }); @@ -85,7 +87,7 @@ describe('skillRoots', () => { await mkdir(homeDir, { recursive: true }); await mkdir(join(osHomeDir, '.agents/skills/notes'), { recursive: true }); - const roots = await userRoots(homeDir, osHomeDir); + const roots = await userRoots(fs, homeDir, osHomeDir); expect(roots.some((r) => r.path.endsWith('.agents/skills') && r.source === 'user')).toBe( true, @@ -103,7 +105,13 @@ describe('skillRoots', () => { await mkdir(absDir, { recursive: true }); await mkdir(join(root, 'relative'), { recursive: true }); - const roots = await configuredRoots(['~', '~/notes', absDir, 'relative'], root, homeDir, 'extra'); + const roots = await configuredRoots( + fs, + ['~', '~/notes', absDir, 'relative'], + root, + homeDir, + 'extra', + ); const paths = roots.map((root) => root.path); expect(roots.every((root) => root.source === 'extra')).toBe(true); diff --git a/packages/agent-core-v2/test/app/workspaceRegistry/workspaceRegistryService.test.ts b/packages/agent-core-v2/test/app/workspaceRegistry/workspaceRegistryService.test.ts index 8532e9b589..43a5e9228c 100644 --- a/packages/agent-core-v2/test/app/workspaceRegistry/workspaceRegistryService.test.ts +++ b/packages/agent-core-v2/test/app/workspaceRegistry/workspaceRegistryService.test.ts @@ -13,7 +13,10 @@ import { import { createScopedTestHost, stubPair } from '#/_base/di/test'; import { encodeWorkDirKey, workspaceRootKey } from '#/_base/utils/workdir-slug'; import { ErrorCodes, Error2 } from '#/errors'; +import { IBootstrapService } from '#/app/bootstrap/bootstrap'; +import { CrossProcessLockService } from '#/os/backends/node-local/crossProcessLockService'; import { HostFileSystem } from '#/os/backends/node-local/hostFsService'; +import { ICrossProcessLockService } from '#/os/interface/crossProcessLock'; import { IHostFileSystem } from '#/os/interface/hostFileSystem'; import { JsonAtomicDocumentStore } from '#/persistence/backends/node-fs/atomicDocumentStore'; import { FileStorageService } from '#/persistence/backends/node-fs/fileStorageService'; @@ -23,6 +26,7 @@ import { IWorkspaceRegistry } from '#/app/workspaceRegistry/workspaceRegistry'; import { WorkspaceRegistryService } from '#/app/workspaceRegistry/workspaceRegistryService'; import { FileWorkspacePersistence } from '#/app/workspaceRegistry/fileWorkspacePersistence'; import { IWorkspacePersistence, type PersistedWorkspaceEntry } from '#/app/workspaceRegistry/workspacePersistence'; +import { stubBootstrap } from '../bootstrap/stubs'; interface SessionIndexLine { readonly sessionId: string; @@ -32,7 +36,7 @@ interface SessionIndexLine { describe('WorkspaceRegistryService (file-backed)', () => { let homeDir: string; - let currentHost: ReturnType | undefined; + const hosts: ReturnType[] = []; beforeEach(async () => { _clearScopedRegistryForTests(); @@ -54,25 +58,26 @@ describe('WorkspaceRegistryService (file-backed)', () => { }); afterEach(async () => { - currentHost?.dispose(); - currentHost = undefined; + for (const host of hosts.splice(0)) host.dispose(); await fsp.rm(homeDir, { recursive: true, force: true }); }); function build(hostFs: IHostFileSystem = new HostFileSystem()): IWorkspaceRegistry { - const fileStorage = new FileStorageService(homeDir); + const locks = new CrossProcessLockService(); + const fileStorage = new FileStorageService(homeDir, undefined, undefined, locks); const host = createScopedTestHost([ stubPair(IFileSystemStorageService, fileStorage), stubPair(IAtomicDocumentStore, new JsonAtomicDocumentStore(fileStorage)), stubPair(IHostFileSystem, hostFs), + stubPair(IBootstrapService, stubBootstrap(homeDir)), + stubPair(ICrossProcessLockService, locks), ]); - currentHost = host; + hosts.push(host); return host.app.accessor.get(IWorkspaceRegistry); } function restart(): IWorkspaceRegistry { - currentHost?.dispose(); - currentHost = undefined; + hosts.pop()?.dispose(); return build(); } @@ -637,6 +642,59 @@ describe('WorkspaceRegistryService (file-backed)', () => { const afterMerge = await readWorkspacesJson(); expect(Object.keys(afterMerge.workspaces)).toEqual([unrelatedId]); }); + + it('two registries interleave createOrTouch without losing entries', async () => { + const a = build(); + const b = build(); + const dirsA: string[] = []; + const dirsB: string[] = []; + for (let i = 0; i < 4; i++) { + const da = join(homeDir, `side-a-${i}`); + const db = join(homeDir, `side-b-${i}`); + await fsp.mkdir(da); + await fsp.mkdir(db); + dirsA.push(da); + dirsB.push(db); + } + for (let i = 0; i < 4; i++) { + await Promise.all([a.createOrTouch(dirsA[i]!), b.createOrTouch(dirsB[i]!)]); + } + + const onDisk = await readWorkspacesJson(); + const expectedIds = [...dirsA, ...dirsB].map((dir) => encodeWorkDirKey(dir)).toSorted(); + expect(Object.keys(onDisk.workspaces).toSorted()).toEqual(expectedIds); + expect(onDisk.deleted_workspace_ids).toEqual([]); + for (const entry of Object.values(onDisk.workspaces)) { + expect(Object.keys(entry).toSorted()).toEqual([ + 'created_at', + 'last_opened_at', + 'name', + 'root', + ]); + } + }); + + it('interleaved delete and createOrTouch keep both the new entry and the tombstone', async () => { + const dirKeep = join(homeDir, 'keep'); + const dirDrop = join(homeDir, 'drop'); + const dirNew = join(homeDir, 'new'); + await fsp.mkdir(dirKeep); + await fsp.mkdir(dirDrop); + await fsp.mkdir(dirNew); + const a = build(); + await a.createOrTouch(dirKeep); + const drop = await a.createOrTouch(dirDrop); + const b = build(); + + await Promise.all([a.delete(drop.id), b.createOrTouch(dirNew)]); + + const onDisk = await readWorkspacesJson(); + expect(Object.keys(onDisk.workspaces).toSorted()).toEqual( + [encodeWorkDirKey(dirKeep), encodeWorkDirKey(dirNew)].toSorted(), + ); + expect(onDisk.deleted_workspace_ids).toEqual([drop.id]); + }); + }); describe('workspaceRootKey', () => { diff --git a/packages/agent-core-v2/test/harness/agent.ts b/packages/agent-core-v2/test/harness/agent.ts index 8d93b027df..32b1df1cc3 100644 --- a/packages/agent-core-v2/test/harness/agent.ts +++ b/packages/agent-core-v2/test/harness/agent.ts @@ -86,6 +86,8 @@ import { IAgentPermissionRulesService, IHostFileSystem, ISessionContext, + ISessionLeaseService, + ISessionWriteAdmission, ISessionProcessRunner, IAgentScopeContext, IAgentStepRetryService, @@ -98,6 +100,7 @@ import { IAgentBuiltinToolsRegistrar, IAgentUserToolService, IAgentUsageService, + IStorageWriteAdmission, ISessionWorkspaceContext, AgentLLMRequesterService, LifecycleScope, @@ -139,9 +142,12 @@ import { ISessionQuestionService, type QuestionResult } from '#/session/question import { ISessionSkillCatalog } from '#/session/sessionSkillCatalog/skillCatalog'; import { ISessionSwarmService } from '#/session/swarm/sessionSwarm'; import type { PathAccessOperation } from '#/session/workspaceContext/workspaceContext'; +import { IHostFsWatchService } from '#/os/interface/hostFsWatch'; import { recordAgentEvents, type RecordedEventEntry } from '../snapshot/events'; import { createFakeHostFs, createFakeProcessRunner } from '../tools/fixtures/fake-exec'; +import { fakeHostFsWatch } from '../session/sessionFs/stubs'; +import { stubSessionLeaseService } from '../session/sessionLease/stubs'; import { createScriptedGenerate } from './scripted-generate'; import { DEFAULT_TEST_SYSTEM_PROMPT, @@ -457,13 +463,14 @@ export function homeDirServices(homeDir: string | undefined): TestAgentServiceOv if (homeDir !== undefined) { for (const [id, value] of bootstrapSeed({ homeDir, + osHomeDir: homeDir, cwd: process.cwd(), env: process.env, })) { reg.defineInstance(id, value); } const file = (): SyncDescriptor => - new SyncDescriptor(FileStorageService, [homeDir], true); + new SyncDescriptor(FileStorageService, [homeDir, undefined, undefined], true); reg.defineDescriptor(IFileSystemStorageService, file()); reg.define(IBlobStore, BlobStoreService); } @@ -887,6 +894,7 @@ export class AgentTestContext { private readonly root: Scope; private readonly session: Scope; private readonly agent: Scope; + private readonly sessionWriteAdmissionRegistration: IDisposable; private readonly disposables: IDisposable[] = []; private suppressWireSnapshot = false; private pluginSessionStartRegistered = false; @@ -926,6 +934,7 @@ export class AgentTestContext { })) { reg.defineInstance(id, value); } + reg.defineInstance(IHostFsWatchService, fakeHostFsWatch().service); const memoryStorage = (): SyncDescriptor => new SyncDescriptor(InMemoryStorageService, [], true); reg.defineDescriptor(IFileSystemStorageService, memoryStorage()); @@ -1002,6 +1011,10 @@ export class AgentTestContext { .get(ITelemetryService) .withContext({ agent_id: agentId }); const sessionScope = bootstrap.sessionScope(workspaceId, sessionId); + const sessionLease = stubSessionLeaseService(); + this.sessionWriteAdmissionRegistration = this.root.accessor + .get(IStorageWriteAdmission) + .registerSession(sessionScope, sessionLease); this.session = this.root.createChild(LifecycleScope.Session, sessionId, { extra: collectScopeSeed( [ @@ -1016,6 +1029,8 @@ export class AgentTestContext { scope: (subKey?: string): string => subKey === undefined || subKey === '' ? sessionScope : `${sessionScope}/${subKey}`, }); + reg.defineInstance(ISessionLeaseService, sessionLease); + reg.defineInstance(ISessionWriteAdmission, sessionLease); reg.defineInstance(ISessionInteractionService, this.createInteractionService()); reg.defineInstance(ISessionApprovalService, this.createApprovalService()); reg.defineInstance(ISessionQuestionService, this.createQuestionService()); @@ -1631,6 +1646,7 @@ export class AgentTestContext { disposable.dispose(); } await this.closeWire(); + this.sessionWriteAdmissionRegistration.dispose(); this.root.dispose(); } diff --git a/packages/agent-core-v2/test/os/backends/node-local/crossProcessLockService.test.ts b/packages/agent-core-v2/test/os/backends/node-local/crossProcessLockService.test.ts new file mode 100644 index 0000000000..b215c9956c --- /dev/null +++ b/packages/agent-core-v2/test/os/backends/node-local/crossProcessLockService.test.ts @@ -0,0 +1,103 @@ +/** + * `crossProcessLock` domain — node-local kernel-lock integration tests. + * + * Exercises permanent sentinels, diagnostic owner metadata, fail-fast and + * waiting acquisition, and release behavior against a real temporary + * directory. + */ + +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { CrossProcessLockService } from '#/os/backends/node-local/crossProcessLockService'; +import { + CrossProcessLockErrorCode, + type CrossProcessLockServiceDeps, + type ICrossProcessLockHandle, +} from '#/os/interface/crossProcessLock'; + +let tmpDir: string; +let lockPath: string; +const handles: ICrossProcessLockHandle[] = []; + +function service( + instanceId: string, + pid: number, + deps: Pick = {}, +): CrossProcessLockService { + let sequence = 0; + return new CrossProcessLockService({ + ...deps, + instanceId, + selfPid: pid, + newLockId: () => `${instanceId}-${++sequence}`, + }); +} + +function ownerPath(): string { + return `${lockPath}.owner.json`; +} + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'kimi-kernel-lock-')); + lockPath = join(tmpDir, 'resource.lock'); +}); + +afterEach(() => { + for (const handle of handles.splice(0)) handle.release(); + rmSync(tmpDir, { recursive: true, force: true }); +}); + +describe('CrossProcessLockService', () => { + it('rejects a second holder in the same process', async () => { + const first = await service('alpha', 1001).acquire(lockPath); + handles.push(first); + + await expect(service('beta', 2002).acquire(lockPath)).rejects.toMatchObject({ + code: CrossProcessLockErrorCode.Held, + details: { path: lockPath, reason: 'held' }, + }); + }); + + it('reports creating when a holder has no readable owner metadata', async () => { + const lock = service('alpha', 1001); + const handle = await lock.acquire(lockPath); + handles.push(handle); + writeFileSync(ownerPath(), '{'); + + expect(lock.inspect(lockPath)).toEqual({ state: 'creating' }); + }); + + it('times out waiting for a held lock', async () => { + const first = await service('alpha', 1001).acquire(lockPath); + handles.push(first); + + await expect( + service('beta', 2002).withLock(lockPath, { wait: { timeoutMs: 15, retryIntervalMs: 5 } }, () => {}), + ).rejects.toMatchObject({ code: CrossProcessLockErrorCode.WaitTimeout }); + }); + + it('withLock releases after the callback throws', async () => { + const lock = service('alpha', 1001); + await expect( + lock.withLock(lockPath, { wait: { timeoutMs: 100 } }, () => { + throw new Error('boom'); + }), + ).rejects.toThrow('boom'); + + const next = await service('beta', 2002).acquire(lockPath); + handles.push(next); + expect(next.checkHeld()).toBe(true); + }); + + it('release is idempotent and checkHeld fails closed afterwards', async () => { + const handle = await service('alpha', 1001).acquire(lockPath); + handle.release(); + handle.release(); + + expect(handle.checkHeld()).toBe(false); + }); +}); diff --git a/packages/agent-core-v2/test/os/backends/node-local/hostFsWatchService.test.ts b/packages/agent-core-v2/test/os/backends/node-local/hostFsWatchService.test.ts index d4cc51e68e..5b3299bd6d 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/hostFsWatchService.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/hostFsWatchService.test.ts @@ -4,6 +4,7 @@ */ import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -86,4 +87,23 @@ describe('HostFsWatchService', () => { expect(events).toHaveLength(0); }); + + it('ignores unix sockets and other special files instead of erroring', async () => { + root = await mkdtemp(join(tmpdir(), 'hostfswatch-')); + // A unix socket inside a watched dir makes chokidar's per-entry fs.watch + // call throw UNKNOWN — special files must be filtered, never watched. + const sockPath = join(root, 'special.sock'); + const server = createServer(); + await new Promise((resolve) => server.listen(sockPath, resolve)); + + const events = await start(); + await writeFile(join(root, 'real.txt'), 'x'); + await wait(300); + server.close(); + + expect(events.some((e) => e.path === sockPath)).toBe(false); + expect(events.some((e) => e.path === join(root, 'real.txt') && e.action === 'created')).toBe( + true, + ); + }); }); diff --git a/packages/agent-core-v2/test/os/backends/node-local/tools/bash.test.ts b/packages/agent-core-v2/test/os/backends/node-local/tools/bash.test.ts index 49d4409bd2..5b6b5b9154 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/tools/bash.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/tools/bash.test.ts @@ -474,6 +474,8 @@ function createFakeTaskService(options: { maxRunningTasks?: number } = {}): { const service: IAgentTaskService = { _serviceBrand: undefined, + beginClose: () => {}, + flushPersistence: async () => {}, track(): never { throw new Error('fake IAgentTaskService.track is not implemented'); }, diff --git a/packages/agent-core-v2/test/os/backends/node-local/tools/read.test.ts b/packages/agent-core-v2/test/os/backends/node-local/tools/read.test.ts index 292861964a..29814d3f41 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/tools/read.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/tools/read.test.ts @@ -23,7 +23,7 @@ import { PathSecurityError } from '#/tool/path-access'; import { MEDIA_SNIFF_BYTES } from '#/agent/media/file-type'; import type { ISessionSkillCatalog } from '#/session/sessionSkillCatalog/skillCatalog'; import { stubWorkspaceContext } from '../../../../session/workspaceContext/stub-workspace-context'; -import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import type { HostFileStat, IHostFileSystem } from '#/os/interface/hostFileSystem'; import { MAX_BYTES, MAX_LINE_LENGTH, @@ -33,7 +33,12 @@ import { ReadTool, } from '#/os/backends/node-local/tools/read'; import type { IHostEnvironment } from '#/os/interface/hostEnvironment'; -import type { ExecutableToolContext, ExecutableToolResult, ToolExecution } from '#/tool/toolContract'; +import { + type ExecutableToolContext, + type ExecutableToolResult, + type ToolExecution, + toolFileRevision, +} from '#/tool/toolContract'; const signal = new AbortController().signal; const PERMISSIVE_WORKSPACE = stubWorkspaceContext('/'); @@ -220,6 +225,44 @@ describe('ReadTool', () => { '2 lines read from file starting from line 1. Total lines in file: 2. End of file reached.', ), }); + expect(result[toolFileRevision]).toEqual({ + path: '/tmp/a.txt', + size: 11, + mtimeMs: undefined, + ino: undefined, + }); + }); + + it('rejects a file whose revision changes while it is being read', async () => { + const initial: HostFileStat = { + isFile: true, + isDirectory: false, + size: 6, + mtimeMs: 1000, + ino: 1, + }; + const changed: HostFileStat = { ...initial, size: 7, mtimeMs: 1001 }; + const readLines = vi.fn().mockImplementation(async function* ( + _path: string, + options?: { onFileStat?: (stat: HostFileStat) => void }, + ): AsyncGenerator { + yield 'alpha\n'; + options?.onFileStat?.(changed); + }); + const fs = { + readBytes: vi.fn(async () => Buffer.from('alpha\n')), + readLines, + stat: vi.fn(async () => initial), + } as unknown as IHostFileSystem; + const tool = new ReadTool(fs, createTestEnv(), PERMISSIVE_WORKSPACE); + + const result = await execute(tool, { path: '/tmp/a.txt' }); + + expect(result).toEqual({ + isError: true, + output: '"/tmp/a.txt" changed while it was being read. Retry the read.', + }); + expect(result[toolFileRevision]).toBeUndefined(); }); it('stats the resolved target so symlinked files stay readable', async () => { @@ -258,7 +301,7 @@ describe('ReadTool', () => { ); }); - it('respects one-based line_offset and positive n_lines', async () => { + it('returns the requested line range without a complete-file revision', async () => { const tool = toolWithContent('a\nb\nc\nd\ne'); const result = await execute(tool, { path: '/tmp/a.txt', line_offset: 2, n_lines: 2 }); @@ -267,6 +310,7 @@ describe('ReadTool', () => { output: '2\tb\n3\tc', note: readNote('2 lines read from file starting from line 2. Total lines in file: 5.'), }); + expect(result[toolFileRevision]).toBeUndefined(); }); it('returns an empty successful output when line_offset is beyond EOF', async () => { @@ -347,7 +391,10 @@ describe('ReadTool', () => { ), ); expect(readBytes).toHaveBeenCalledWith('/tmp/external.txt', MEDIA_SNIFF_BYTES); - expect(readLines).toHaveBeenCalledWith('/tmp/external.txt', { errors: 'strict' }); + expect(readLines).toHaveBeenCalledWith('/tmp/external.txt', { + errors: 'strict', + onFileStat: expect.any(Function), + }); }); it('returns a friendly error for missing files before sniffing bytes', async () => { @@ -393,7 +440,10 @@ describe('ReadTool', () => { ), ); expect(readBytes).toHaveBeenCalledWith('/home/test/notes/today.txt', MEDIA_SNIFF_BYTES); - expect(readLines).toHaveBeenCalledWith('/home/test/notes/today.txt', { errors: 'strict' }); + expect(readLines).toHaveBeenCalledWith('/home/test/notes/today.txt', { + errors: 'strict', + onFileStat: expect.any(Function), + }); }); it('blocks sensitive files independently from workspace access', async () => { @@ -548,7 +598,7 @@ describe('ReadTool', () => { expect(output).not.toContain('encoded data was not valid'); }); - it('truncates long lines and surfaces the affected line numbers', async () => { + it('returns a revision when a default read truncates long lines', async () => { const long = 'x'.repeat(MAX_LINE_LENGTH + 10); const tool = toolWithContent([long, 'short', long].join('\n')); @@ -556,9 +606,10 @@ describe('ReadTool', () => { expect(result.note).toContain('Lines [1, 3] were truncated.'); expect(result.output).toContain('...'); + expect(result[toolFileRevision]).toBeDefined(); }); - it('checks the byte cap before adding the next rendered line', async () => { + it('returns a revision when default output reaches the byte cap', async () => { const line = 'x'.repeat(MAX_LINE_LENGTH); const content = Array.from({ length: 80 }, () => line).join('\n'); const tool = toolWithContent(content); @@ -568,6 +619,7 @@ describe('ReadTool', () => { expect(Buffer.byteLength(output, 'utf8')).toBeLessThanOrEqual(MAX_BYTES); expect(result.note).toContain(`Max ${String(MAX_BYTES)} bytes reached.`); + expect(result[toolFileRevision]).toBeDefined(); }); it('reads through bounded byte preflight and streams line iteration without full readText', async () => { @@ -605,7 +657,7 @@ describe('ReadTool', () => { expect(readText).not.toHaveBeenCalled(); }); - it('caps default reads at MAX_LINES', async () => { + it('returns a revision when default output reaches the line cap', async () => { const content = Array.from({ length: MAX_LINES + 1 }, (_, i) => `line ${String(i + 1)}`).join( '\n', ); @@ -616,6 +668,7 @@ describe('ReadTool', () => { expect(result.note).toContain(`Max ${String(MAX_LINES)} lines reached.`); expect(result.output).toContain(`${String(MAX_LINES)}\tline ${String(MAX_LINES)}`); expect(result.output).not.toContain(`${String(MAX_LINES + 1)}\tline ${String(MAX_LINES + 1)}`); + expect(result[toolFileRevision]).toBeDefined(); }); it('tail byte truncation keeps the newest lines closest to EOF', async () => { diff --git a/packages/agent-core-v2/test/os/backends/node-local/tools/write.test.ts b/packages/agent-core-v2/test/os/backends/node-local/tools/write.test.ts index 894d792e54..5c4983f5eb 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/tools/write.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/tools/write.test.ts @@ -18,7 +18,12 @@ import type { HostFileStat, IHostFileSystem } from '#/os/interface/hostFileSyste import { stubWorkspaceContext } from '../../../../session/workspaceContext/stub-workspace-context'; import { type WriteInput, WriteInputSchema, WriteTool } from '#/os/backends/node-local/tools/write'; import type { IHostEnvironment } from '#/os/interface/hostEnvironment'; -import type { ExecutableToolContext, ExecutableToolResult, ToolExecution } from '#/tool/toolContract'; +import { + type ExecutableToolContext, + type ExecutableToolResult, + type ToolExecution, + toolFileRevision, +} from '#/tool/toolContract'; const signal = new AbortController().signal; const PERMISSIVE_WORKSPACE = stubWorkspaceContext('/'); @@ -47,8 +52,8 @@ function createTestEnv(home = '/home'): IHostEnvironment { interface WriteFsOptions { readText?: (path: string) => Promise; - writeText?: (path: string, data: string) => Promise; - appendText?: (path: string, data: string) => Promise; + writeText?: (path: string, data: string) => Promise; + appendText?: (path: string, data: string) => Promise; stat?: (path: string) => Promise; mkdir?: (path: string) => Promise; } @@ -60,8 +65,26 @@ function createWriteFs(options: WriteFsOptions = {}) { throw Object.assign(new Error('ENOENT: no such file or directory'), { code: 'ENOENT' }); }), ); - const writeText = vi.fn(options.writeText ?? (async () => {})); - const appendText = vi.fn(options.appendText ?? (async () => {})); + const writeText = vi.fn( + options.writeText ?? + (async (_path: string, data: string) => ({ + isFile: true, + isDirectory: false, + size: Buffer.byteLength(data), + mtimeMs: 1000, + ino: 1, + })), + ); + const appendText = vi.fn( + options.appendText ?? + (async (_path: string, data: string) => ({ + isFile: true, + isDirectory: false, + size: Buffer.byteLength(data), + mtimeMs: 1000, + ino: 1, + })), + ); const stat = vi.fn( options.stat ?? (async () => ({ isFile: false, isDirectory: true, size: 0 })), ); @@ -192,6 +215,12 @@ describe('WriteTool', () => { expect(writeText).toHaveBeenCalledWith('/tmp/new.txt', 'hello'); expect(result.output).toContain('Wrote 5 bytes'); + expect(result[toolFileRevision]).toEqual({ + path: '/tmp/new.txt', + size: 5, + mtimeMs: 1000, + ino: 1, + }); }); it('expands leading tilde paths using the kaos home directory', async () => { diff --git a/packages/agent-core-v2/test/os/stubs.ts b/packages/agent-core-v2/test/os/stubs.ts new file mode 100644 index 0000000000..92b0234c7b --- /dev/null +++ b/packages/agent-core-v2/test/os/stubs.ts @@ -0,0 +1,60 @@ +/** + * `os` test stubs — shared cross-process-lock pass-through for unit tests. + * + * `stubCrossProcessLock()` mirrors the real service's mutual-exclusion + * semantics (a held path rejects further acquisitions) without touching the + * filesystem, for tests whose suite is otherwise fully in-memory. Lives under + * `test/` (not `src/`) so test-support code stays out of the production tree. + * Import from a relative path (`./stubs` or `../os/stubs`). + */ + +import { + CrossProcessLockError, + CrossProcessLockErrorCode, + type CrossProcessLockInspection, + type ICrossProcessLockHandle, + type ICrossProcessLockService, +} from '#/os/interface/crossProcessLock'; + +export function stubCrossProcessLock(): ICrossProcessLockService { + const held = new Set(); + const acquireHandle = (lockPath: string): ICrossProcessLockHandle => { + if (held.has(lockPath)) { + throw new CrossProcessLockError( + CrossProcessLockErrorCode.Held, + `cross-process lock unavailable (held): ${lockPath}`, + { details: { path: lockPath, reason: 'held' } }, + ); + } + held.add(lockPath); + let released = false; + return { + lockPath, + lockId: 'stub-lock', + checkHeld: () => !released, + release: () => { + if (released) return; + released = true; + held.delete(lockPath); + }, + }; + }; + return { + _serviceBrand: undefined, + acquire: (lockPath) => Promise.resolve(acquireHandle(lockPath)), + withLock: async ( + lockPath: string, + _options: Parameters[1], + fn: (handle: ICrossProcessLockHandle) => T | Promise, + ): Promise => { + const handle = acquireHandle(lockPath); + try { + return await fn(handle); + } finally { + handle.release(); + } + }, + inspect: (lockPath): CrossProcessLockInspection => + held.has(lockPath) ? { state: 'held' } : { state: 'free' }, + }; +} diff --git a/packages/agent-core-v2/test/persistence/backends/node-fs/appendLogStore.test.ts b/packages/agent-core-v2/test/persistence/backends/node-fs/appendLogStore.test.ts index a6b24eb762..284a19e08c 100644 --- a/packages/agent-core-v2/test/persistence/backends/node-fs/appendLogStore.test.ts +++ b/packages/agent-core-v2/test/persistence/backends/node-fs/appendLogStore.test.ts @@ -7,6 +7,8 @@ * test/persistence/backends/node-fs/appendLogStore.test.ts`. */ +import { join } from 'node:path'; + import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { SyncDescriptor } from '#/_base/di/descriptors'; @@ -130,7 +132,7 @@ describe('AppendLogStore', () => { ); }); - it('reacquire after final release waits for retiring storage before fresh I/O', async () => { + it('keeps a failed retired generation reachable and blocks replacement I/O', async () => { const failure = new Error('retiring append failed'); let markAppendStarted!: () => void; const appendStarted = new Promise((resolve) => { @@ -140,32 +142,16 @@ describe('AppendLogStore', () => { const appendGate = new Promise((resolve) => { releaseAppend = resolve; }); - let markReplacementAppendStarted!: () => void; - const replacementAppendStarted = new Promise((resolve) => { - markReplacementAppendStarted = resolve; - }); - let releaseReplacementAppend!: () => void; - const replacementAppendGate = new Promise((resolve) => { - releaseReplacementAppend = resolve; - }); let reportFailure!: (error: unknown) => void; const reportedFailure = new Promise((resolve) => { reportFailure = resolve; }); let appendAttempts = 0; - let replacementStarted = false; - const originalAppend = storage.append.bind(storage); storage.append = async (...args) => { appendAttempts++; - if (appendAttempts === 1) { - markAppendStarted(); - await appendGate; - throw failure; - } - replacementStarted = true; - markReplacementAppendStarted(); - await replacementAppendGate; - return originalAppend(...args); + markAppendStarted(); + await appendGate; + throw failure; }; const retiringOwner = record.acquire(SCOPE, KEY); @@ -174,27 +160,13 @@ describe('AppendLogStore', () => { retiringOwner.dispose(); const replacementOwner = record.acquire(SCOPE, KEY); record.append(SCOPE, KEY, { n: 2 }); - const orderedFlush = record.flush(); - await Promise.resolve(); - await Promise.resolve(); - expect(replacementStarted).toBe(false); releaseAppend(); expect(await reportedFailure).toBe(failure); - await replacementAppendStarted; - - const currentFlush = record.flush(); - let flushSettled = false; - void currentFlush.then(() => { - flushSettled = true; - }); - await Promise.resolve(); - await Promise.resolve(); - expect(flushSettled).toBe(false); - - releaseReplacementAppend(); - await Promise.all([orderedFlush, currentFlush]); - expect(await collect(SCOPE, KEY)).toEqual([{ n: 2 }]); + await expect(record.flush(SCOPE)).rejects.toBe(failure); + await expect(record.flush(SCOPE)).rejects.toBe(failure); + expect(appendAttempts).toBe(1); + expect(await storage.read(SCOPE, KEY)).toBeUndefined(); replacementOwner.dispose(); }); @@ -223,8 +195,8 @@ describe('AppendLogStore', () => { finalOwner.dispose(); const replacementOwner = record.acquire(SCOPE, KEY); record.append(SCOPE, KEY, { n: 2 }); - await record.flush(); - expect(await collect(SCOPE, KEY)).toEqual([{ n: 2 }]); + await expect(record.flush()).rejects.toBe(failure); + expect(appendAttempts).toBe(1); replacementOwner.dispose(); }); @@ -680,4 +652,5 @@ describe('AppendLogStore', () => { { n: 2, s: '日本語' }, ]); }); + }); diff --git a/packages/agent-core-v2/test/persistence/backends/node-fs/atomicDocumentStore.test.ts b/packages/agent-core-v2/test/persistence/backends/node-fs/atomicDocumentStore.test.ts index ae427f496a..2d59f9a197 100644 --- a/packages/agent-core-v2/test/persistence/backends/node-fs/atomicDocumentStore.test.ts +++ b/packages/agent-core-v2/test/persistence/backends/node-fs/atomicDocumentStore.test.ts @@ -45,6 +45,32 @@ describe('JsonAtomicDocumentStore', () => { expect(await config.get('session', 'state.json')).toEqual({ title: 'new', count: 2 }); }); + it('serializes concurrent read-modify-write updates for one key', async () => { + await config.set('session', 'state.json', { count: 0 }); + let releaseFirst!: () => void; + const firstMayFinish = new Promise((resolve) => { + releaseFirst = resolve; + }); + let firstEntered!: () => void; + const firstDidEnter = new Promise((resolve) => { + firstEntered = resolve; + }); + + const first = config.update('session', 'state.json', async (current) => { + firstEntered(); + await firstMayFinish; + return { count: (current?.count ?? 0) + 1 }; + }); + await firstDidEnter; + const second = config.update('session', 'state.json', (current) => ({ + count: (current?.count ?? 0) + 1, + })); + releaseFirst(); + + await expect(Promise.all([first, second])).resolves.toEqual([{ count: 1 }, { count: 2 }]); + expect(await config.get('session', 'state.json')).toEqual({ count: 2 }); + }); + it('keys are independent', async () => { await config.set('session', 'a.json', { title: 'A' }); await config.set('session', 'b.json', { title: 'B' }); diff --git a/packages/agent-core-v2/test/persistence/backends/node-fs/fileStorageService.test.ts b/packages/agent-core-v2/test/persistence/backends/node-fs/fileStorageService.test.ts index c173164776..d53f1e6ecb 100644 --- a/packages/agent-core-v2/test/persistence/backends/node-fs/fileStorageService.test.ts +++ b/packages/agent-core-v2/test/persistence/backends/node-fs/fileStorageService.test.ts @@ -1,10 +1,14 @@ -import { mkdtemp, mkdir, rm, stat, writeFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'pathe'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { Error2, ErrorCodes } from '#/errors'; +import { CrossProcessLockService } from '#/os/backends/node-local/crossProcessLockService'; import { FileStorageService } from '#/persistence/backends/node-fs/fileStorageService'; +import { StorageWriteAdmissionService } from '#/persistence/backends/node-fs/storageWriteAdmissionService'; +import type { ISessionWriteAdmission } from '#/persistence/interface/sessionWriteAdmission'; const isWin = process.platform === 'win32'; const encoder = new TextEncoder(); @@ -87,3 +91,109 @@ describe('FileStorageService — error translation', () => { }); }); }); + +describe('FileStorageService — session write admission', () => { + let dir: string; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'fss-fence-')); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('runs write, append, and delete through session write admission', async () => { + const registry = new StorageWriteAdmissionService(); + let writable = true; + const assertCanWriteNow = (): void => { + if (!writable) { + throw new Error2(ErrorCodes.SESSION_LEASE_LOST, 'session lease lost'); + } + }; + const admission: ISessionWriteAdmission = { + _serviceBrand: undefined, + assertCanWriteNow, + withPhysicalWrite: async (io) => { + assertCanWriteNow(); + return io(); + }, + sealAndDrain: async () => {}, + }; + const registration = registry.registerSession('sessions/workspace/session', admission); + expect(() => registry.registerSession('sessions/workspace/session', admission)).toThrow( + /already registered/, + ); + const svc = new FileStorageService(dir, undefined, undefined, undefined, registry); + const scope = 'sessions/workspace/session/agents/main/tool-results'; + + await svc.write(scope, 'result.txt', encoder.encode('a')); + await svc.append(scope, 'result.txt', encoder.encode('b')); + expect(await readFile(join(dir, scope, 'result.txt'), 'utf8')).toBe('ab'); + + writable = false; + await expect(svc.write(scope, 'result.txt', encoder.encode('c'))).rejects.toMatchObject({ + code: ErrorCodes.SESSION_LEASE_LOST, + }); + await expect(svc.append(scope, 'result.txt', encoder.encode('c'))).rejects.toMatchObject({ + code: ErrorCodes.SESSION_LEASE_LOST, + }); + await expect(svc.delete(scope, 'result.txt')).rejects.toMatchObject({ + code: ErrorCodes.SESSION_LEASE_LOST, + }); + expect(await readFile(join(dir, scope, 'result.txt'), 'utf8')).toBe('ab'); + registration.dispose(); + await expect(svc.append(scope, 'result.txt', encoder.encode('c'))).rejects.toMatchObject({ + code: ErrorCodes.SESSION_LEASE_LOST, + }); + }); + + it('fails closed without session admission and leaves non-session scopes unrestricted', async () => { + const registry = new StorageWriteAdmissionService(); + const svc = new FileStorageService(dir, undefined, undefined, undefined, registry); + + await expect( + svc.write('sessions/workspace/session/agents/main/blobs', 'blob', encoder.encode('x')), + ).rejects.toMatchObject({ code: ErrorCodes.SESSION_LEASE_LOST }); + await expect(svc.write('cron/workspace', 'task.json', encoder.encode('{}'))).resolves.toBeUndefined(); + }); + + it('revalidates session admission after acquiring the exclusive key mutation', async () => { + const registry = new StorageWriteAdmissionService(); + let checks = 0; + const admission: ISessionWriteAdmission = { + _serviceBrand: undefined, + assertCanWriteNow: () => { + checks++; + if (checks > 1) { + throw new Error2( + ErrorCodes.SESSION_LEASE_LOST, + 'session admission sealed while waiting', + ); + } + }, + withPhysicalWrite: (io) => io(), + sealAndDrain: async () => {}, + }; + registry.registerSession('sessions/workspace/session', admission); + const svc = new FileStorageService( + dir, + undefined, + undefined, + new CrossProcessLockService(), + registry, + ); + let mutationRan = false; + + await expect( + svc.withExclusiveKeyMutation( + 'sessions/workspace/session', + 'state.json', + async () => { + mutationRan = true; + }, + ), + ).rejects.toMatchObject({ code: ErrorCodes.SESSION_LEASE_LOST }); + expect(mutationRan).toBe(false); + }); +}); diff --git a/packages/agent-core-v2/test/session/agentLifecycle/agentLifecycle.test.ts b/packages/agent-core-v2/test/session/agentLifecycle/agentLifecycle.test.ts index ca067f15df..0915d64688 100644 --- a/packages/agent-core-v2/test/session/agentLifecycle/agentLifecycle.test.ts +++ b/packages/agent-core-v2/test/session/agentLifecycle/agentLifecycle.test.ts @@ -150,7 +150,9 @@ describe('AgentLifecycleService', () => { let registerAgent: ReturnType>; let atomicDocs: Map; let permissionModeSetMode: ReturnType; + let beginTaskClose: ReturnType; let stopAllOnExit: ReturnType; + let flushPersistence: ReturnType; let loopActiveTurnId: number | undefined; let loopPendingTurnIds: number[]; let loopCancel: ReturnType>; @@ -210,6 +212,20 @@ describe('AgentLifecycleService', () => { set: async (scope: string, key: string, value: T): Promise => { atomicDocs.set(`${scope}/${key}`, value); }, + update: async ( + scope: string, + key: string, + mutate: (current: T | undefined) => T | Promise, + ): Promise => { + const next = await mutate(atomicDocs.get(`${scope}/${key}`) as T | undefined); + atomicDocs.set(`${scope}/${key}`, next); + return next; + }, + withExclusiveKeyMutation: async ( + _scope: string, + _key: string, + mutation: () => Promise, + ): Promise => mutation(), delete: async (scope: string, key: string): Promise => { atomicDocs.delete(`${scope}/${key}`); }, @@ -333,10 +349,14 @@ describe('AgentLifecycleService', () => { onDidChangeMode: Event.None, } as unknown as IAgentPermissionModeService); ix.set(ISessionMcpService, new SyncDescriptor(SessionMcpService)); + beginTaskClose = vi.fn(); stopAllOnExit = vi.fn(async () => []); + flushPersistence = vi.fn(async () => {}); ix.stub(IAgentTaskService, { _serviceBrand: undefined, + beginClose: beginTaskClose, stopAllOnExit, + flushPersistence, } as unknown as IAgentTaskService); ix.stub(IAgentFullCompactionService, { _serviceBrand: undefined, diff --git a/packages/agent-core-v2/test/session/sessionFileLedger/fileLedger.test.ts b/packages/agent-core-v2/test/session/sessionFileLedger/fileLedger.test.ts new file mode 100644 index 0000000000..b45824ad49 --- /dev/null +++ b/packages/agent-core-v2/test/session/sessionFileLedger/fileLedger.test.ts @@ -0,0 +1,115 @@ +/** + * `sessionFileLedger` domain (L2) — verifies the optimistic-concurrency + * verdicts (clean / stale / no-baseline) against a real tmpdir and a real + * `HostFileSystem` with stat calls counted. Baselines are compared only + * against fresh stat tuples, and stat failures fail closed. + */ + +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { LifecycleScope } from '#/_base/di/scope'; +import { createScopedTestHost, stubPair, type ScopedTestHost } from '#/_base/di/test'; +import { unwrapErrorCause } from '#/_base/errors/errors'; +import { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { ISessionFileLedger } from '#/session/sessionFileLedger/fileLedger'; +import { SessionFileLedger } from '#/session/sessionFileLedger/fileLedgerService'; + +import { countingHostFs } from '../sessionFs/stubs'; + +void SessionFileLedger; + +interface World { + readonly workDir: string; + readonly fs: IHostFileSystem; + readonly ledger: ISessionFileLedger; + readonly statCalls: () => number; + readonly poisonedPaths: Set; +} + +function makeWorld(): World { + const workDir = mkdtempSync(join(tmpdir(), 'kimi-ledger-work-')); + cleanupPaths.push(workDir); + const poisonedPaths = new Set(); + const { fs, statCalls } = countingHostFs(poisonedPaths); + const host = createScopedTestHost([stubPair(IHostFileSystem, fs)]); + const session = host.child(LifecycleScope.Session, 's1'); + hosts.push(host); + return { + workDir, + fs, + ledger: session.accessor.get(ISessionFileLedger), + statCalls, + poisonedPaths, + }; +} + +async function recordCurrentBaseline(world: World, path: string): Promise { + try { + const stat = await world.fs.stat(path); + world.ledger.recordBaseline(path, { + exists: true, + ino: stat.ino, + mtimeMs: stat.mtimeMs, + size: stat.size, + }); + } catch (error) { + const code = (unwrapErrorCause(error) as NodeJS.ErrnoException).code; + if (code !== 'ENOENT' && code !== 'ENOTDIR') throw error; + world.ledger.recordBaseline(path, { exists: false }); + } +} + +const hosts: ScopedTestHost[] = []; +const cleanupPaths: string[] = []; + +describe('SessionFileLedger', () => { + afterEach(() => { + for (const host of hosts.splice(0)) host.dispose(); + for (const path of cleanupPaths.splice(0)) rmSync(path, { recursive: true, force: true }); + }); + + it('returns clean for a missing file (new-file creation is exempt)', async () => { + const world = makeWorld(); + expect(await world.ledger.compare(join(world.workDir, 'new.txt'))).toBe('clean'); + }); + + it('returns stale when a baselined file is modified outside the session', async () => { + const world = makeWorld(); + const file = join(world.workDir, 'a.txt'); + writeFileSync(file, 'hello'); + await recordCurrentBaseline(world, file); + + writeFileSync(file, 'hello world'); + + expect(await world.ledger.compare(file)).toBe('stale'); + }); + + it('performs a fresh stat for every comparison', async () => { + const world = makeWorld(); + const file = join(world.workDir, 'a.txt'); + writeFileSync(file, 'hello'); + await recordCurrentBaseline(world, file); + expect(world.statCalls()).toBe(1); + + expect(await world.ledger.compare(file)).toBe('clean'); + expect(world.statCalls()).toBe(2); + + expect(await world.ledger.compare(file)).toBe('clean'); + expect(world.statCalls()).toBe(3); + }); + + it('fails closed when stat fails for reasons other than not-found', async () => { + const world = makeWorld(); + const file = join(world.workDir, 'a.txt'); + writeFileSync(file, 'hello'); + await recordCurrentBaseline(world, file); + world.poisonedPaths.add(file); + + expect(await world.ledger.compare(file)).toBe('stale'); + expect(world.statCalls()).toBeGreaterThan(0); + }); +}); diff --git a/packages/agent-core-v2/test/session/sessionFs/fsService.test.ts b/packages/agent-core-v2/test/session/sessionFs/fsService.test.ts index 32de5cef24..396ef412af 100644 --- a/packages/agent-core-v2/test/session/sessionFs/fsService.test.ts +++ b/packages/agent-core-v2/test/session/sessionFs/fsService.test.ts @@ -99,8 +99,8 @@ function fakeFs( if (c === undefined) throw enoent(p); return c; }, - writeText: async () => {}, - appendText: async () => {}, + writeText: async () => ({ isFile: true, isDirectory: false, size: 0 }), + appendText: async () => ({ isFile: true, isDirectory: false, size: 0 }), readBytes: async (p, n) => { const c = fileMap.get(p); if (c === undefined) throw enoent(p); @@ -336,11 +336,12 @@ function makeSession( symlinks: readonly string[] = [], runner?: ISessionProcessRunner, symlinkTargets: Record = {}, + hostFs?: IHostFileSystem, ): ISessionFsService { host = createScopedTestHost(); const session = host.child(LifecycleScope.Session, 's1', [ stubPair(ISessionWorkspaceContext, stubWorkspace()), - stubPair(IHostFileSystem, fakeFs(files, symlinks, symlinkTargets)), + stubPair(IHostFileSystem, hostFs ?? fakeFs(files, symlinks, symlinkTargets)), stubPair(ISessionProcessRunner, runner ?? fakeRunner(handler)), stubPair(ITelemetryService, telemetryStub(events)), stubPair(IGitService, git), diff --git a/packages/agent-core-v2/test/session/sessionFs/fsWatchService.test.ts b/packages/agent-core-v2/test/session/sessionFs/fsWatchService.test.ts index 08868dd713..8630bd4e11 100644 --- a/packages/agent-core-v2/test/session/sessionFs/fsWatchService.test.ts +++ b/packages/agent-core-v2/test/session/sessionFs/fsWatchService.test.ts @@ -11,17 +11,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { LifecycleScope } from '#/_base/di/scope'; import { createScopedTestHost, stubPair } from '#/_base/di/test'; import { IHostFileSystem } from '#/os/interface/hostFileSystem'; -import { - type HostFsChange, - type IHostFsWatchHandle, - IHostFsWatchService, -} from '#/os/interface/hostFsWatch'; +import { IHostFsWatchService } from '#/os/interface/hostFsWatch'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; import type { FsChangeEvent } from '#/session/sessionFs/fsWatch'; -import { ISessionFsWatchService } from '#/session/sessionFs/fsWatch'; +import { ISessionFsWatchService, normalizeFsWatchKey } from '#/session/sessionFs/fsWatch'; import { SessionFsWatchService } from '#/session/sessionFs/fsWatchService'; +import { fakeHostFsWatch, type FakeWatch } from './stubs'; + const WORK_DIR = '/repo'; void SessionFsWatchService; @@ -44,44 +42,6 @@ function stubWorkspace(): ISessionWorkspaceContext { }; } -interface FakeWatch { - readonly service: IHostFsWatchService; - readonly watchCalls: string[]; - fire: (rel: string, action: HostFsChange['action'], kind?: HostFsChange['kind']) => void; - readonly disposed: () => boolean; -} - -function fakeHostFsWatch(): FakeWatch { - const watchCalls: string[] = []; - let listener: ((e: HostFsChange) => void) | undefined; - let disposed = false; - const handle: IHostFsWatchHandle = { - onDidChange: (l) => { - listener = l; - return { dispose: () => (listener = undefined) }; - }, - dispose: () => { - disposed = true; - listener = undefined; - }, - }; - const service: IHostFsWatchService = { - _serviceBrand: undefined, - watch: (path) => { - watchCalls.push(path); - disposed = false; - return handle; - }, - }; - return { - service, - watchCalls, - fire: (rel, action, kind = 'file') => - listener?.({ path: join(WORK_DIR, rel), action, kind }), - disposed: () => disposed, - }; -} - function fakeHostFs(gitignore?: string): IHostFileSystem { return { _serviceBrand: undefined, @@ -100,13 +60,13 @@ interface Harness { readonly events: FsChangeEvent[]; } -function makeSession(gitignore?: string): Harness { +function makeSession(gitignore?: string, hostFs?: IHostFileSystem): Harness { const watch = fakeHostFsWatch(); const host = createScopedTestHost(); const session = host.child(LifecycleScope.Session, 's1', [ stubPair(ISessionWorkspaceContext, stubWorkspace()), stubPair(IHostFsWatchService, watch.service), - stubPair(IHostFileSystem, fakeHostFs(gitignore)), + stubPair(IHostFileSystem, hostFs ?? fakeHostFs(gitignore)), ]); const svc = session.accessor.get(ISessionFsWatchService); const events: FsChangeEvent[] = []; @@ -186,6 +146,43 @@ describe('SessionFsWatchService', () => { expect(events[0]?.changes.map((c) => c.path)).toEqual(['src/keep.ts']); }); + it('rebuilds the matcher when the workspace `.gitignore` changes', async () => { + let gitignore = 'dist/\n'; + const hostFs = { + _serviceBrand: undefined, + readText: async (p: string) => { + if (p === join(WORK_DIR, '.gitignore')) return gitignore; + const err = new Error(`ENOENT: ${p}`) as NodeJS.ErrnoException; + err.code = 'ENOENT'; + throw err; + }, + } as unknown as IHostFileSystem; + const { svc, watch, events } = makeSession(undefined, hostFs); + svc.setWatchedPaths(['.']); + await Promise.resolve(); + await Promise.resolve(); + + watch.fire('dist/a.js', 'created'); + vi.advanceTimersByTime(200); + expect(events).toHaveLength(0); + + gitignore = 'build/\n'; + watch.fire('.gitignore', 'modified'); + await Promise.resolve(); + await Promise.resolve(); + vi.advanceTimersByTime(200); + + watch.fire('dist/b.js', 'created'); + watch.fire('build/c.js', 'created'); + vi.advanceTimersByTime(200); + + // The `.gitignore` change itself is a delivered event; afterwards + // `dist/` passes (no longer ignored) and `build/` is filtered. + expect(events).toHaveLength(2); + expect(events[0]?.changes.map((c) => c.path)).toEqual(['.gitignore']); + expect(events[1]?.changes.map((c) => c.path)).toEqual(['dist/b.js']); + }); + it('rejects paths that escape the workspace', () => { const { svc } = makeSession(); expect(() => svc.setWatchedPaths(['../x'])).toThrowError(/escapes workspace|rejected/); @@ -209,3 +206,10 @@ describe('SessionFsWatchService', () => { expect(events).toHaveLength(0); }); }); + +describe('fsWatch key helpers', () => { + it('normalizes keys lexically and folds case on macOS/Windows', () => { + const folded = process.platform === 'darwin' || process.platform === 'win32'; + expect(normalizeFsWatchKey('/A//B/../C')).toBe(folded ? '/a/c' : '/A/C'); + }); +}); diff --git a/packages/agent-core-v2/test/session/sessionFs/stubs.ts b/packages/agent-core-v2/test/session/sessionFs/stubs.ts new file mode 100644 index 0000000000..bf361d0fc8 --- /dev/null +++ b/packages/agent-core-v2/test/session/sessionFs/stubs.ts @@ -0,0 +1,92 @@ +/** + * `sessionFs` test stubs — controllable fake host watcher and stat-counting + * host filesystem. + * + * `fakeHostFsWatch()` mirrors `IHostFsWatchService.watch()` semantics (one + * independent handle per call) without touching the real filesystem: tests + * fire synthetic changes at the most recent handle and advance the debounce + * window with fake timers. `countingHostFs()` wraps a real `HostFileSystem` + * and counts `stat` calls, optionally failing chosen paths with `EACCES`. + * Import from a relative path (`./stubs` or `../sessionFs/stubs`). + */ + +import { join } from 'node:path'; + +import { HostFileSystem } from '#/os/backends/node-local/hostFsService'; +import { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { + type HostFsChange, + type IHostFsWatchHandle, + IHostFsWatchService, +} from '#/os/interface/hostFsWatch'; + +export interface FakeWatch { + readonly service: IHostFsWatchService; + readonly watchCalls: string[]; + fire: (rel: string, action: HostFsChange['action'], kind?: HostFsChange['kind']) => void; + readonly disposed: () => boolean; +} + +export function fakeHostFsWatch(): FakeWatch { + const watchCalls: string[] = []; + const handles: Array<{ + fire: (rel: string, action: HostFsChange['action'], kind?: HostFsChange['kind']) => void; + disposed: () => boolean; + }> = []; + const service: IHostFsWatchService = { + _serviceBrand: undefined, + watch: (path) => { + watchCalls.push(path); + let listener: ((e: HostFsChange) => void) | undefined; + let disposed = false; + const handle: IHostFsWatchHandle = { + ready: Promise.resolve(), + onDidChange: (l) => { + listener = l; + return { dispose: () => (listener = undefined) }; + }, + dispose: () => { + disposed = true; + listener = undefined; + }, + }; + handles.push({ + fire: (rel, action, kind = 'file') => + listener?.({ path: join(path, rel), action, kind }), + disposed: () => disposed, + }); + return handle; + }, + }; + return { + service, + watchCalls, + fire: (rel, action, kind = 'file') => handles.at(-1)?.fire(rel, action, kind), + disposed: () => handles.every((h) => h.disposed()), + }; +} + +export function countingHostFs(poisonedPaths?: Set): { + fs: IHostFileSystem; + statCalls: () => number; +} { + const real = new HostFileSystem(); + let count = 0; + const fs = new Proxy(real, { + get(target, prop, receiver) { + if (prop === 'stat') { + return async (path: string) => { + count += 1; + if (poisonedPaths?.has(path)) { + const err = new Error(`EACCES: permission denied, stat '${path}'`) as NodeJS.ErrnoException; + err.code = 'EACCES'; + throw err; + } + return target.stat(path); + }; + } + return Reflect.get(target, prop, receiver); + }, + }) as IHostFileSystem; + return { fs, statCalls: () => count }; +} diff --git a/packages/agent-core-v2/test/session/sessionLease/sessionLease.test.ts b/packages/agent-core-v2/test/session/sessionLease/sessionLease.test.ts new file mode 100644 index 0000000000..df3f106952 --- /dev/null +++ b/packages/agent-core-v2/test/session/sessionLease/sessionLease.test.ts @@ -0,0 +1,105 @@ +/** + * `sessionLease` domain — unit tests for the per-session write lease. + * + * Runs against the real node-local kernel-lock service rooted at a mkdtemp + * home, asserting loss notification, write admission/draining, and release. + */ + +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { Error2, ErrorCodes } from '#/errors'; +import { CrossProcessLockService } from '#/os/backends/node-local/crossProcessLockService'; +import { SessionLease, sessionLeasePath } from '#/session/sessionLease/sessionLease'; + +let tmpDir: string; +let locks: CrossProcessLockService; + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'kimi-session-lease-')); + locks = new CrossProcessLockService(); +}); + +afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); +}); + +async function acquire( + sessionId = 's1', + onLost: (sessionId: string) => void = () => {}, +): Promise { + return new SessionLease( + sessionId, + await locks.acquire(sessionLeasePath(tmpDir, sessionId)), + onLost, + ); +} + +function thrownError(fn: () => void): Error2 { + try { + fn(); + } catch (error) { + return error as Error2; + } + throw new Error('expected the call to throw'); +} + +describe('SessionLease', () => { + it('fires the loss notification once and then fails closed', async () => { + const onLost = vi.fn(); + const lease = await acquire('s1', onLost); + // Replacing the sentinel fails the kernel handle's dev/ino identity + // check, driving the loss path through the real gate. + rmSync(sessionLeasePath(tmpDir, 's1')); + writeFileSync(sessionLeasePath(tmpDir, 's1'), ''); + expect(thrownError(() => lease.assertWritable()).code).toBe(ErrorCodes.SESSION_LEASE_LOST); + expect(onLost).toHaveBeenCalledTimes(1); + expect(onLost).toHaveBeenCalledWith('s1'); + expect(thrownError(() => lease.assertWritable()).code).toBe(ErrorCodes.SESSION_LEASE_LOST); + expect(onLost).toHaveBeenCalledTimes(1); + }); + + it('release is idempotent and later assertions throw', async () => { + const lease = await acquire(); + lease.release(); + lease.release(); + + expect(lease.info).toBeUndefined(); + expect(thrownError(() => lease.assertWritable()).code).toBe(ErrorCodes.SESSION_LEASE_LOST); + }); + + it('sealAndDrain rejects new writes while waiting for an admitted physical write', async () => { + const lease = await acquire(); + let enterWrite!: () => void; + const writeEntered = new Promise((resolve) => { + enterWrite = resolve; + }); + let finishWrite!: () => void; + const physicalWriteBlocked = new Promise((resolve) => { + finishWrite = resolve; + }); + const write = lease.withPhysicalWrite(async () => { + enterWrite(); + await physicalWriteBlocked; + }); + await writeEntered; + + let drained = false; + const drain = lease.sealAndDrain().then(() => { + drained = true; + }); + await expect(lease.withPhysicalWrite(async () => {})).rejects.toMatchObject({ + code: ErrorCodes.SESSION_LEASE_LOST, + }); + expect(drained).toBe(false); + + finishWrite(); + await write; + await drain; + expect(drained).toBe(true); + lease.release(); + }); +}); diff --git a/packages/agent-core-v2/test/session/sessionLease/stubs.ts b/packages/agent-core-v2/test/session/sessionLease/stubs.ts new file mode 100644 index 0000000000..da90cfe4c8 --- /dev/null +++ b/packages/agent-core-v2/test/session/sessionLease/stubs.ts @@ -0,0 +1,28 @@ +/** + * `sessionLease` test stubs — no-op session lease and write admission. + * + * Lives under `test/` (not `src/`). The default stub reports no lease info + * and admits every write; tests that exercise fencing override the relevant + * gate. Import from a relative path. + */ + +import type { ISessionWriteAdmission } from '#/persistence/interface/sessionWriteAdmission'; +import { ISessionLeaseService } from '#/session/sessionLease/sessionLease'; + +export function stubSessionLeaseService( + overrides: Partial = {}, +): ISessionLeaseService & ISessionWriteAdmission { + const lease: ISessionLeaseService & ISessionWriteAdmission = { + _serviceBrand: undefined, + info: undefined, + assertWritable: () => {}, + assertCanWriteNow: () => lease.assertWritable(), + withPhysicalWrite: async (io) => { + lease.assertCanWriteNow(); + return io(); + }, + sealAndDrain: async () => {}, + ...overrides, + }; + return lease; +} diff --git a/packages/agent-core-v2/test/session/sessionMetadata/sessionMetadata.test.ts b/packages/agent-core-v2/test/session/sessionMetadata/sessionMetadata.test.ts index b98e6b7c80..96f3b08f8f 100644 --- a/packages/agent-core-v2/test/session/sessionMetadata/sessionMetadata.test.ts +++ b/packages/agent-core-v2/test/session/sessionMetadata/sessionMetadata.test.ts @@ -247,4 +247,5 @@ describe('SessionMetadata', () => { expect(next.agents?.['main']?.labels).toEqual({ swarmItem: 'src/a.ts' }); expect(next.updatedAt).toBeGreaterThan(before); }); + }); diff --git a/packages/agent-core-v2/test/session/sessionSkillCatalog/skillCatalog.test.ts b/packages/agent-core-v2/test/session/sessionSkillCatalog/skillCatalog.test.ts index 104d2727d2..276b090cf1 100644 --- a/packages/agent-core-v2/test/session/sessionSkillCatalog/skillCatalog.test.ts +++ b/packages/agent-core-v2/test/session/sessionSkillCatalog/skillCatalog.test.ts @@ -34,107 +34,16 @@ import { ISessionSkillCatalog } from '#/session/sessionSkillCatalog/skillCatalog import { IPluginSkillSource } from '#/session/sessionSkillCatalog/pluginSkillSource'; import { ISkillDiscovery } from '#/app/skillCatalog/skillDiscovery'; import type { SkillRoot } from '#/app/skillCatalog/types'; +import { IHostFsWatchService } from '#/os/interface/hostFsWatch'; import { stubBootstrap } from '../../app/bootstrap/stubs'; import { stubSkill } from '../../app/skillCatalog/stubs'; import { stubProviderService } from '../../app/provider/stubs'; +import { fakeHostFsWatch } from '../sessionFs/stubs'; +import { configStub, pluginStub, workspaceStub } from './stubs'; const bootstrapStub = stubBootstrap('/home'); -function configStub(): IConfigService & { - setExtraSkillDirs(dirs: readonly string[]): void; - setMergeAllAvailableSkills(value: boolean): void; - fireSectionChange(domain: string): void; -} { - let extraSkillDirs: readonly string[] = []; - let mergeAllAvailableSkills = true; - const sectionChangeListeners: Array<(event: unknown) => void> = []; - return { - _serviceBrand: undefined, - ready: Promise.resolve(), - onDidChangeConfiguration: () => ({ dispose: () => {} }), - onDidSectionChange: (listener: (event: unknown) => void) => { - sectionChangeListeners.push(listener); - return { dispose: () => {} }; - }, - get: (domain: string) => { - if (domain === EXTRA_SKILL_DIRS_SECTION) return [...extraSkillDirs]; - if (domain === MERGE_ALL_AVAILABLE_SKILLS_SECTION) return mergeAllAvailableSkills; - return undefined; - }, - inspect: () => ({ value: undefined, defaultValue: undefined, userValue: undefined, memoryValue: undefined }), - getAll: () => ({}), - set: async () => {}, - replace: async () => {}, - reload: async () => {}, - diagnostics: () => [], - setExtraSkillDirs: (dirs: readonly string[]) => { - extraSkillDirs = [...dirs]; - }, - setMergeAllAvailableSkills: (value: boolean) => { - mergeAllAvailableSkills = value; - }, - fireSectionChange: (domain: string) => { - for (const listener of sectionChangeListeners) { - listener({ domain, source: 'set', value: undefined, previousValue: undefined }); - } - }, - } as unknown as IConfigService & { - setExtraSkillDirs(dirs: readonly string[]): void; - setMergeAllAvailableSkills(value: boolean): void; - fireSectionChange(domain: string): void; - }; -} - -function pluginStub( - skillRoots: readonly SkillRoot[] = [], - reloadEmitter?: Emitter, -): IPluginService { - return { - _serviceBrand: undefined, - onDidReload: reloadEmitter !== undefined ? reloadEmitter.event : () => ({ dispose: () => {} }), - listPlugins: async () => [], - installPlugin: async () => ({ id: '' }) as never, - setPluginEnabled: async () => {}, - setPluginMcpServerEnabled: async () => {}, - removePlugin: async () => {}, - reloadPlugins: async () => ({ added: [], removed: [], errors: [] }), - getPluginInfo: async () => { - throw new Error('getPluginInfo is not used by these tests'); - }, - listPluginCommands: async () => [], - checkUpdates: async () => [], - pluginSkillRoots: async () => skillRoots, - enabledSessionStarts: async () => [], - enabledMcpServers: async () => ({}), - enabledHooks: async () => [], - }; -} - -function workspaceStub(workDir: string): { - readonly stub: ISessionWorkspaceContext; - setWorkDir(dir: string): void; -} { - let current = workDir; - const stub = { - _serviceBrand: undefined, - get workDir() { - return current; - }, - additionalDirs: [] as readonly string[], - setWorkDir: (dir: string) => { - current = dir; - }, - setAdditionalDirs: () => {}, - resolve: (rel: string) => rel, - isWithin: () => true, - assertAllowed: (p: string) => p, - addAdditionalDir: () => {}, - removeAdditionalDir: () => {}, - } satisfies ISessionWorkspaceContext; - return { stub, setWorkDir: (dir) => { current = dir; } }; -} - function makeHost( store: ISkillDiscovery, ws: ISessionWorkspaceContext, @@ -153,6 +62,7 @@ function makeHost( stubPair(IConfigService, config), stubPair(ISkillCatalogRuntimeOptions, runtimeOptions), stubPair(IPluginService, pluginStub(pluginRoots, pluginReloadEmitter)), + stubPair(IHostFsWatchService, fakeHostFsWatch().service), ]); const session = host.child(LifecycleScope.Session, 's1', [stubPair(ISessionWorkspaceContext, ws)]); return { host, session, config }; @@ -328,6 +238,7 @@ describe('SessionSkillCatalogService', () => { stubPair(IConfigService, config), stubPair(ISkillCatalogRuntimeOptions, runtimeOptions), stubPair(IPluginService, pluginStub()), + stubPair(IHostFsWatchService, fakeHostFsWatch().service), ]); const session = host.child(LifecycleScope.Session, 's1', [stubPair(ISessionWorkspaceContext, ws)]); @@ -367,6 +278,7 @@ describe('SessionSkillCatalogService', () => { stubPair(IConfigService, config), stubPair(ISkillCatalogRuntimeOptions, runtimeOptions), stubPair(IPluginService, pluginStub()), + stubPair(IHostFsWatchService, fakeHostFsWatch().service), ]); const session = host.child(LifecycleScope.Session, 's1', [stubPair(ISessionWorkspaceContext, ws)]); @@ -584,6 +496,7 @@ describe('SessionSkillCatalogService', () => { _serviceBrand: undefined, } as unknown as ISkillCatalogRuntimeOptions), stubPair(IPluginService, pluginStub()), + stubPair(IHostFsWatchService, fakeHostFsWatch().service), ]); const session = host.child(LifecycleScope.Session, 's1', [ stubPair(ISessionWorkspaceContext, ws), @@ -643,6 +556,7 @@ describe('SessionSkillCatalogService', () => { _serviceBrand: undefined, } as unknown as ISkillCatalogRuntimeOptions), stubPair(IPluginService, pluginService), + stubPair(IHostFsWatchService, fakeHostFsWatch().service), ]); const session = host.child(LifecycleScope.Session, 's1', [ stubPair(ISessionWorkspaceContext, ws), @@ -701,6 +615,7 @@ describe('SessionSkillCatalogService', () => { _serviceBrand: undefined, } as unknown as ISkillCatalogRuntimeOptions), stubPair(IProviderService, stubProviderService()), + stubPair(IHostFsWatchService, fakeHostFsWatch().service), ]); const { stub: ws } = workspaceStub('/work'); const session = host.child(LifecycleScope.Session, 's1', [ diff --git a/packages/agent-core-v2/test/session/sessionSkillCatalog/skillHotReload.test.ts b/packages/agent-core-v2/test/session/sessionSkillCatalog/skillHotReload.test.ts new file mode 100644 index 0000000000..489c7e9a10 --- /dev/null +++ b/packages/agent-core-v2/test/session/sessionSkillCatalog/skillHotReload.test.ts @@ -0,0 +1,230 @@ +/** + * Scenario: skill-catalog hot reload — filesystem changes under watched skill + * roots drive each file-backed source's `onDidChange`, the serialized catalog + * remerge, and expose the new skills through `ISessionSkillCatalog`. + * + * Exercises the real chokidar watcher (`HostFsWatchService`) and real + * `FileSkillDiscovery` over temporary directories, wired through the real + * DI scope tree. Run: `pnpm --filter @moonshot-ai/agent-core-v2 exec vitest + * run test/session/sessionSkillCatalog/skillHotReload.test.ts`. + */ + +import { realpathSync } from 'node:fs'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; + +import { join } from 'pathe'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { createScopedTestHost, stubPair, type ScopedTestHost } from '#/_base/di/test'; +import { SyncDescriptor } from '#/_base/di/descriptors'; +import { LifecycleScope, type Scope } from '#/_base/di/scope'; +import '#/index'; +import { IBootstrapService } from '#/app/bootstrap/bootstrap'; +import { IConfigService } from '#/app/config/config'; +import { ILogService } from '#/_base/log/log'; +import { IPluginService } from '#/app/plugin/plugin'; +import { FileSkillDiscovery } from '#/app/skillCatalog/fileSkillDiscovery'; +import { ISkillDiscovery } from '#/app/skillCatalog/skillDiscovery'; +import { ISkillCatalogRuntimeOptions } from '#/app/skillCatalog/skillCatalogRuntimeOptions'; +import { IHostFsWatchService } from '#/os/interface/hostFsWatch'; +import { HostFsWatchService } from '#/os/backends/node-local/hostFsWatchService'; +import { ISessionSkillCatalog } from '#/session/sessionSkillCatalog/skillCatalog'; +import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; + +import { stubLog } from '../../_base/log/stubs'; +import { stubBootstrap } from '../../app/bootstrap/stubs'; +import { configStub, pluginStub, workspaceStub } from './stubs'; + +const wait = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); + +interface HotReloadFixture { + readonly host: ScopedTestHost; + readonly session: Scope; + readonly catalog: ISessionSkillCatalog; + readonly config: ReturnType; + readonly changes: string[]; +} + +async function makeBase(): Promise { + return realpathSync(await mkdtemp(join(tmpdir(), 'skill-hot-reload-'))); +} + +async function writeSkill(root: string, name: string, description?: string): Promise { + const descriptionText = description ?? `desc for ${name}`; + await mkdir(join(root, name), { recursive: true }); + await writeFile(join(root, name, 'SKILL.md'), `---\nname: ${name}\ndescription: ${descriptionText}\n---\nbody`); +} + +function makeHost( + base: string, + opts: { readonly extraSkillDirs?: readonly string[]; readonly explicitDirs?: readonly string[] } = {}, +): HotReloadFixture { + const homeDir = join(base, 'home'); + const osHomeDir = join(base, 'os-home'); + const workDir = join(base, 'project'); + const bootstrap = { ...stubBootstrap(homeDir), osHomeDir }; + const config = configStub(); + const runtimeOptions = { + _serviceBrand: undefined, + explicitDirs: opts.explicitDirs, + } as unknown as ISkillCatalogRuntimeOptions; + const host = createScopedTestHost([ + [ISkillDiscovery, new SyncDescriptor(FileSkillDiscovery) as unknown], + stubPair(ILogService, stubLog()), + stubPair(IBootstrapService, bootstrap), + stubPair(IConfigService, config), + stubPair(ISkillCatalogRuntimeOptions, runtimeOptions), + stubPair(IPluginService, pluginStub()), + stubPair(IHostFsWatchService, new HostFsWatchService()), + ]); + const session = host.child(LifecycleScope.Session, 's1', [ + stubPair(ISessionWorkspaceContext, workspaceStub(workDir).stub), + stubPair(ILogService, stubLog()), + ]); + const catalog = session.accessor.get(ISessionSkillCatalog); + const changes: string[] = []; + catalog.onDidChange((sourceId) => changes.push(sourceId)); + if (opts.extraSkillDirs !== undefined) config.setExtraSkillDirs(opts.extraSkillDirs); + return { host, session, catalog, config, changes }; +} + +async function waitFor(cond: () => boolean, label: string, timeoutMs = 20000): Promise { + const start = Date.now(); + while (!cond()) { + if (Date.now() - start > timeoutMs) throw new Error(`timed out waiting for: ${label}`); + await wait(100); + } +} + +describe('skill hot reload', () => { + const tmpdirs: string[] = []; + const fixtures: HotReloadFixture[] = []; + + async function fixture( + opts: { readonly extraSkillDirs?: readonly string[]; readonly explicitDirs?: readonly string[] } = {}, + ): Promise { + const base = await makeBase(); + tmpdirs.push(base); + const f = makeHost(base, opts); + fixtures.push(f); + return { ...f, base }; + } + + afterEach(async () => { + for (const f of fixtures.splice(0)) f.host.dispose(); + for (const dir of tmpdirs.splice(0)) await rm(dir, { recursive: true, force: true }); + }); + + it('reloads the user source on add / modify / remove under /skills', async () => { + const { catalog, changes, base } = await fixture(); + const userRoot = join(base, 'home', 'skills'); + + await catalog.load(); + expect(catalog.catalog.getSkill('hot-one')).toBeUndefined(); + + await writeSkill(userRoot, 'hot-one'); + await waitFor(() => catalog.catalog.getSkill('hot-one') !== undefined, 'hot-one appears'); + + await writeSkill(userRoot, 'hot-one', 'updated description'); + await waitFor( + () => catalog.catalog.getSkill('hot-one')?.description === 'updated description', + 'hot-one description updates', + ); + + await rm(join(userRoot, 'hot-one'), { recursive: true, force: true }); + await waitFor(() => catalog.catalog.getSkill('hot-one') === undefined, 'hot-one disappears'); + + expect(changes.filter((id) => id === 'user').length).toBeGreaterThanOrEqual(3); + }, 30000); + + it('a burst of writes collapses into a bounded number of catalog reloads', async () => { + const { catalog, changes, base } = await fixture(); + const userRoot = join(base, 'home', 'skills'); + + await catalog.load(); + await wait(400); + changes.length = 0; + + for (let i = 0; i < 4; i += 1) { + await writeSkill(userRoot, `burst-${i}`); + } + await waitFor(() => catalog.catalog.getSkill('burst-3') !== undefined, 'burst skills appear'); + await wait(600); + + expect(catalog.catalog.getSkill('burst-0')).toBeDefined(); + expect(changes.length).toBeGreaterThanOrEqual(1); + expect(changes.length).toBeLessThanOrEqual(2); + }, 30000); + + it('reloads the workspace source for both project brand and generic roots', async () => { + const { catalog, changes, base } = await fixture(); + const projectDir = join(base, 'project'); + await mkdir(join(projectDir, '.git'), { recursive: true }); + const brandRoot = join(projectDir, '.kimi-code', 'skills'); + + await catalog.load(); + await wait(400); + + await writeSkill(brandRoot, 'workspace-one'); + await waitFor(() => catalog.catalog.getSkill('workspace-one') !== undefined, 'workspace-one appears'); + + await rm(join(brandRoot, 'workspace-one'), { recursive: true, force: true }); + await waitFor(() => catalog.catalog.getSkill('workspace-one') === undefined, 'workspace-one disappears'); + + await writeSkill(join(projectDir, '.agents', 'skills'), 'workspace-generic'); + await waitFor( + () => catalog.catalog.getSkill('workspace-generic') !== undefined, + 'workspace-generic appears', + ); + + expect(changes).toContain('workspace'); + }, 30000); + + it('reloads the extra source under a configured directory created after load', async () => { + const probe = await makeBase(); + tmpdirs.push(probe); + const extraDir = join(probe, 'extra-skills'); + const { catalog, changes } = await fixture({ extraSkillDirs: [extraDir] }); + + await catalog.load(); + await wait(400); + expect(catalog.catalog.getSkill('extra-one')).toBeUndefined(); + + await writeSkill(extraDir, 'extra-one'); + await waitFor(() => catalog.catalog.getSkill('extra-one') !== undefined, 'extra-one appears'); + + expect(changes).toContain('extra'); + }, 30000); + + it('reloads the explicit source and keeps default user discovery replaced', async () => { + const probe = await makeBase(); + tmpdirs.push(probe); + const explicitDir = join(probe, 'explicit-skills'); + const { catalog, base } = await fixture({ explicitDirs: [explicitDir] }); + + await writeSkill(join(base, 'home', 'skills'), 'user-hidden'); + await catalog.load(); + await wait(400); + expect(catalog.catalog.getSkill('user-hidden')).toBeUndefined(); + + await writeSkill(explicitDir, 'explicit-one'); + await waitFor(() => catalog.catalog.getSkill('explicit-one') !== undefined, 'explicit-one appears'); + }, 30000); + + it('session dispose stops its watchers', async () => { + const { catalog, session, changes, base } = await fixture(); + await catalog.load(); + await wait(400); + + session.dispose(); + const firedBeforeWrites = changes.length; + + await writeSkill(join(base, 'home', 'skills'), 'after-dispose'); + await writeSkill(join(base, 'project', '.kimi-code', 'skills'), 'after-dispose-ws'); + await wait(1500); + + expect(changes.length).toBe(firedBeforeWrites); + expect(catalog.catalog.getSkill('after-dispose')).toBeUndefined(); + }, 30000); +}); diff --git a/packages/agent-core-v2/test/session/sessionSkillCatalog/stubs.ts b/packages/agent-core-v2/test/session/sessionSkillCatalog/stubs.ts new file mode 100644 index 0000000000..8eff0796c9 --- /dev/null +++ b/packages/agent-core-v2/test/session/sessionSkillCatalog/stubs.ts @@ -0,0 +1,64 @@ +/** `sessionSkillCatalog` test stubs — config / plugin / workspace fakes for catalog tests. */ + +import type { Emitter } from '#/_base/event'; +import { IConfigService } from '#/app/config/config'; +import { IPluginService } from '#/app/plugin/plugin'; +import type { ReloadSummary } from '#/app/plugin/types'; +import { + EXTRA_SKILL_DIRS_SECTION, + MERGE_ALL_AVAILABLE_SKILLS_SECTION, +} from '#/app/skillCatalog/configSection'; +import type { SkillRoot } from '#/app/skillCatalog/types'; +import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; + +type ConfigStub = IConfigService & { + setExtraSkillDirs(dirs: readonly string[]): void; + fireSectionChange(domain: string): void; +}; + +export function configStub(): ConfigStub { + let extraSkillDirs: readonly string[] = []; + const listeners: Array<(event: unknown) => void> = []; + return { + _serviceBrand: undefined, + ready: Promise.resolve(), + onDidChangeConfiguration: () => ({ dispose: () => {} }), + onDidSectionChange: (listener: (event: unknown) => void) => { + listeners.push(listener); + return { dispose: () => {} }; + }, + get: (domain: string) => { + if (domain === EXTRA_SKILL_DIRS_SECTION) return [...extraSkillDirs]; + if (domain === MERGE_ALL_AVAILABLE_SKILLS_SECTION) return true; + return undefined; + }, + setExtraSkillDirs: (dirs: readonly string[]) => (extraSkillDirs = [...dirs]), + fireSectionChange: (domain: string) => listeners.forEach((l) => l({ domain, source: 'set' })), + } as unknown as ConfigStub; +} + +export function pluginStub( + skillRoots: readonly SkillRoot[] = [], + reloadEmitter?: Emitter, +): IPluginService { + return { + _serviceBrand: undefined, + onDidReload: reloadEmitter !== undefined ? reloadEmitter.event : () => ({ dispose: () => {} }), + pluginSkillRoots: async () => skillRoots, + } as unknown as IPluginService; +} + +export function workspaceStub(workDir: string): { + readonly stub: ISessionWorkspaceContext; + setWorkDir(dir: string): void; +} { + let current = workDir; + const stub = { + _serviceBrand: undefined, + get workDir() { + return current; + }, + additionalDirs: [] as readonly string[], + } as unknown as ISessionWorkspaceContext; + return { stub, setWorkDir: (dir: string) => (current = dir) }; +} diff --git a/packages/agent-core-v2/test/session/workspaceCommand/workspaceCommand.test.ts b/packages/agent-core-v2/test/session/workspaceCommand/workspaceCommand.test.ts index f43d2763fc..98ca932ba9 100644 --- a/packages/agent-core-v2/test/session/workspaceCommand/workspaceCommand.test.ts +++ b/packages/agent-core-v2/test/session/workspaceCommand/workspaceCommand.test.ts @@ -63,7 +63,7 @@ class MemoryHostFs implements IHostFileSystem { return text; } - async writeText(path: string, data: string): Promise { + async writeText(path: string, data: string): Promise { const pause = this.nextWritePause; if (pause !== undefined) { this.nextWritePause = undefined; @@ -76,10 +76,13 @@ class MemoryHostFs implements IHostFileSystem { } } this.files.set(path, data); + return { isFile: true, isDirectory: false, size: Buffer.byteLength(data) }; } - async appendText(path: string, data: string): Promise { - this.files.set(path, (this.files.get(path) ?? '') + data); + async appendText(path: string, data: string): Promise { + const content = (this.files.get(path) ?? '') + data; + this.files.set(path, content); + return { isFile: true, isDirectory: false, size: Buffer.byteLength(content) }; } pauseNextWrite(): { readonly started: Promise; readonly release: () => void } { diff --git a/packages/kap-server/src/envelope.ts b/packages/kap-server/src/envelope.ts index 88f626cca3..2611cb6727 100644 --- a/packages/kap-server/src/envelope.ts +++ b/packages/kap-server/src/envelope.ts @@ -4,4 +4,9 @@ * Keep this file as a re-export shim so downstream `from './envelope'` * imports inside the server stay stable and don't all need to be touched. */ -export { okEnvelope, errEnvelope, type Envelope } from './protocol/envelope'; +export { + okEnvelope, + errEnvelope, + ownershipRedirectEnvelope, + type Envelope, +} from './protocol/envelope'; diff --git a/packages/kap-server/src/error-handler.ts b/packages/kap-server/src/error-handler.ts index 010eb6a05d..5225d40678 100644 --- a/packages/kap-server/src/error-handler.ts +++ b/packages/kap-server/src/error-handler.ts @@ -4,6 +4,11 @@ * Wraps unhandled exceptions in the Feishu-style envelope: * - HTTP status ALWAYS 200 (business outcome lives in `code`); * - `code: 50001` (`internal.error`) for unknown exceptions; + * - `code: 40921` (`session.held_by_peer`) when the session's write lease is + * held by a sibling instance — the structured ownership details ride the + * envelope so the client can redirect or retry (routes without a local + * error switch — approvals, questions, skills — resume sessions + * unguarded and land here); * - `request_id` echoes the inbound request id (set by Fastify's * `genReqId` via `resolveRequestId`); * - `data: null`. @@ -17,7 +22,8 @@ * we never bleed it into the JSON response. */ -import { errEnvelope } from './envelope'; +import { ErrorCodes, isError2 } from '@moonshot-ai/agent-core-v2'; +import { errEnvelope, ownershipRedirectEnvelope } from './envelope'; import { ErrorCode } from './protocol/error-codes'; import type { FastifyError } from 'fastify'; @@ -40,6 +46,13 @@ interface ErrorHandlerHost { export function installErrorHandler(app: ErrorHandlerHost): void { app.setErrorHandler((err, req, reply) => { const requestId = req.id; + // Session-ownership contention is an expected multi-server outcome, not a + // server failure: surface 40921 with the structured details (phase / + // redirect address) and keep the stack in the log only. + if (isError2(err) && err.code === ErrorCodes.SESSION_HELD_BY_PEER) { + reply.status(200).send(ownershipRedirectEnvelope(err, requestId)); + return; + } req.log.error({ err, request_id: requestId }, 'unhandled error'); reply.status(200).send( errEnvelope( diff --git a/packages/kap-server/src/index.ts b/packages/kap-server/src/index.ts index 999cb62a56..fed1a9457d 100644 --- a/packages/kap-server/src/index.ts +++ b/packages/kap-server/src/index.ts @@ -5,6 +5,7 @@ export { startServer } from './start'; export type { ServerStartOptions, RunningServer } from './start'; +export { formatServerOrigin } from './serverOrigin'; export { okEnvelope, errEnvelope } from './envelope'; export type { Envelope } from './envelope'; export { classify } from './security/bindClassify'; diff --git a/packages/kap-server/src/instanceRegistry.ts b/packages/kap-server/src/instanceRegistry.ts index 96c9305faa..5d044fc78f 100644 --- a/packages/kap-server/src/instanceRegistry.ts +++ b/packages/kap-server/src/instanceRegistry.ts @@ -16,9 +16,10 @@ import { randomBytes } from 'node:crypto'; import { mkdir, open, readdir, readFile, rename, unlink } from 'node:fs/promises'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import { resolveKimiHome } from '@moonshot-ai/agent-core-v2'; +import { syncDir } from '@moonshot-ai/agent-core-v2/_base/utils/fs'; import { ulid } from 'ulid'; /** Default cadence for refreshing `heartbeat_at`. */ @@ -147,7 +148,7 @@ async function readInstanceFile(filePath: string): Promise { const tmpPath = `${filePath}.tmp.${process.pid}.${randomBytes(4).toString('hex')}`; let renamed = false; @@ -155,11 +156,15 @@ async function writeFileAtomic(filePath: string, content: string): Promise const fh = await open(tmpPath, 'w'); try { await fh.writeFile(content); + // Flush the payload before the rename, so a crash can't resurrect a 0-byte entry. + await fh.sync(); } finally { await fh.close(); } await rename(tmpPath, filePath); renamed = true; + // The rename itself is not durable — fsync the directory so the new entry survives a crash. + await syncDir(dirname(filePath)); } finally { if (!renamed) { try { diff --git a/packages/kap-server/src/protocol/envelope.ts b/packages/kap-server/src/protocol/envelope.ts index 620a4ae62e..96361eec35 100644 --- a/packages/kap-server/src/protocol/envelope.ts +++ b/packages/kap-server/src/protocol/envelope.ts @@ -6,6 +6,8 @@ import { z } from 'zod'; +import { ErrorCode } from './error-codes'; + export const envelopeSchema = (data: T) => z.object({ code: z.number().int(), @@ -35,12 +37,36 @@ export function okEnvelope(data: T, requestId: string): Envelope { * (or `undefined`) the field is absent and the wire shape stays byte-identical * to the original `{ code, msg, data: null, request_id }` — `JSON.stringify` * drops `undefined` properties, so callers that have no stack are unaffected. + * `details` behaves the same way: when provided it carries a structured, + * client-consumable payload (e.g. `SessionOwnershipDetails` under + * `SESSION_HELD_BY_PEER`); when omitted the field is absent from the wire. */ export function errEnvelope( code: number, msg: string, requestId: string, stack?: string, + details?: unknown, +): Envelope { + return { code, msg, data: null, request_id: requestId, stack, details }; +} + +/** + * Build the `40921 session.held_by_peer` ownership-redirect envelope. The + * structured `details` payload (`held-by-peer` phase / redirect address) is + * the actionable part, so it rides the envelope while the stack stays + * server-side. Accepts the `Error2` shape structurally so this module stays + * free of the engine dependency. + */ +export function ownershipRedirectEnvelope( + err: { readonly message: string; readonly details?: unknown }, + requestId: string, ): Envelope { - return { code, msg, data: null, request_id: requestId, stack }; + return errEnvelope( + ErrorCode.SESSION_HELD_BY_PEER, + err.message, + requestId, + undefined, + err.details, + ); } diff --git a/packages/kap-server/src/protocol/error-codes.ts b/packages/kap-server/src/protocol/error-codes.ts index 775f895925..164bb82fc7 100644 --- a/packages/kap-server/src/protocol/error-codes.ts +++ b/packages/kap-server/src/protocol/error-codes.ts @@ -104,6 +104,8 @@ export const ErrorCode = { FS_ALREADY_EXISTS: 40919, /** goal 只允许主 agent 使用 */ GOAL_UNSUPPORTED_AGENT: 40920, + /** session 的 lease 被其他实例持有；`details` 为 SessionOwnershipDetails */ + SESSION_HELD_BY_PEER: 40921, /** approval 60s 超时 */ APPROVAL_EXPIRED: 41001, diff --git a/packages/kap-server/src/protocol/rest-snapshot.ts b/packages/kap-server/src/protocol/rest-snapshot.ts index 9b445d6158..d3691561aa 100644 --- a/packages/kap-server/src/protocol/rest-snapshot.ts +++ b/packages/kap-server/src/protocol/rest-snapshot.ts @@ -71,8 +71,13 @@ export type SnapshotSubagent = z.infer; export const sessionSnapshotResponseSchema = z.object({ /** Durable event watermark this snapshot is consistent with. */ as_of_seq: z.number().int().nonnegative(), - /** Journal epoch — pass back via the WS cursor for invalidation detection. */ - epoch: z.string().min(1), + /** + * Journal epoch — pass back via the WS cursor for invalidation detection. + * Absent when the journal has no baseline yet (no durable event written): + * "no baseline" is not "baseline changed" — treat the cursor as fresh, + * not as invalidated. + */ + epoch: z.string().min(1).optional(), session: sessionSchema, /** Most recent messages (chronological ascending), bounded page. */ messages: z.object({ diff --git a/packages/kap-server/src/protocol/session.ts b/packages/kap-server/src/protocol/session.ts index 9fa454244a..b3126a5bcc 100644 --- a/packages/kap-server/src/protocol/session.ts +++ b/packages/kap-server/src/protocol/session.ts @@ -39,6 +39,22 @@ export function emptySessionUsage(): SessionUsage { export const sessionPendingInteractionSchema = z.enum(['none', 'approval', 'question']); export type SessionPendingInteraction = z.infer; +/** + * Per-row holder annotation joined onto `GET /sessions` items (multi-instance, + * design §3.8): `self` = this instance holds the write lease (the session is + * materialized here), `peer` = another instance holds it (`address` is its + * reachable base URL when the holder advertised one — the redirect target), + * `none` = no lease on disk (the session is materialized nowhere). Purely + * display/redirect metadata: the lease file stays the authority and is + * re-checked on every materialization. + */ +export const sessionOwnershipSchema = z.object({ + held_by: z.enum(['self', 'peer', 'none']), + address: z.string().min(1).optional(), +}); + +export type SessionOwnership = z.infer; + export const sessionSchema = z.object({ id: z.string().min(1), workspace_id: workspaceIdSchema, @@ -54,6 +70,7 @@ export const sessionSchema = z.object({ /** Outcome of the main agent's most recent turn. */ last_turn_reason: z.enum(['completed', 'cancelled', 'failed']).optional(), archived: z.boolean().optional(), + ownership: sessionOwnershipSchema.optional(), current_prompt_id: z.string().min(1).optional(), /** Text of the most recent user prompt, for search/preview. Absent for empty sessions. */ last_prompt: z.string().optional(), diff --git a/packages/kap-server/src/routes/fs.ts b/packages/kap-server/src/routes/fs.ts index bd3433217f..3dc5a6962f 100644 --- a/packages/kap-server/src/routes/fs.ts +++ b/packages/kap-server/src/routes/fs.ts @@ -33,7 +33,7 @@ import { } from '@moonshot-ai/agent-core-v2/session/sessionFs/fs'; import { z } from 'zod'; -import { errEnvelope, okEnvelope } from '../envelope'; +import { errEnvelope, okEnvelope, ownershipRedirectEnvelope } from '../envelope'; import { launchDetached, openFileCommandFor, @@ -530,6 +530,9 @@ function sendMappedError(reply: Reply, req: { id: string }, err: unknown): void case ErrorCodes.SESSION_NOT_FOUND: reply.send(errEnvelope(ErrorCode.SESSION_NOT_FOUND, err.message, requestId, err.stack)); return; + case ErrorCodes.SESSION_HELD_BY_PEER: + reply.send(ownershipRedirectEnvelope(err, requestId)); + return; // hostFs errors that escaped the sessionFs layer keep their `os.fs.*` // code; map them onto the closest v1 wire code (ENOTDIR collapses into // path-not-found, matching `mapFsError`). diff --git a/packages/kap-server/src/routes/prompts.ts b/packages/kap-server/src/routes/prompts.ts index 1b5fa9b330..008f39f036 100644 --- a/packages/kap-server/src/routes/prompts.ts +++ b/packages/kap-server/src/routes/prompts.ts @@ -62,7 +62,7 @@ import { } from '../protocol/rest-prompt'; import { z } from 'zod'; -import { errEnvelope, okEnvelope } from '../envelope'; +import { errEnvelope, okEnvelope, ownershipRedirectEnvelope } from '../envelope'; import { requestLog } from '../lib/requestLog'; import { defineRoute } from '../middleware/defineRoute'; import { ensureMainAgent, MAIN_AGENT_ID } from '../transport/mainAgent'; @@ -778,6 +778,9 @@ function sendMappedError( case 'session.busy': reply.send(errEnvelope(ErrorCode.SESSION_BUSY, err.message, requestId, err.stack)); return; + case ErrorCodes.SESSION_HELD_BY_PEER: + reply.send(ownershipRedirectEnvelope(err, requestId)); + return; case 'prompt.already_completed': reply.send({ code: ErrorCode.PROMPT_ALREADY_COMPLETED, diff --git a/packages/kap-server/src/routes/sessions.ts b/packages/kap-server/src/routes/sessions.ts index e47a0f7637..288b0e35c1 100644 --- a/packages/kap-server/src/routes/sessions.ts +++ b/packages/kap-server/src/routes/sessions.ts @@ -69,6 +69,13 @@ * their original cwd and stay listed / gettable (matching v1, which stores * `workDir` on the session). `IWorkspaceRegistry` is consulted only as a * back-compat fallback for sessions written before `cwd` was persisted. + * + * **Ownership join (multi-instance, design §3.8)**: `GET /sessions` rows also + * carry an additive `ownership` field joined from the shared + * `session-leases/` tree — `held_by` is 'self' when this instance holds the + * session's write lease, 'peer' (with the holder's `address`, when + * advertised) when another instance does, and 'none' when no instance does. + * See {@link resolveSessionOwnership}. */ import { @@ -81,10 +88,13 @@ import { IAgentRPCService, IAgentActivityView, IAuthSummaryService, + IBootstrapService, + ICrossProcessLockService, ISessionBtwService, ISessionContext, ISessionIndex, ISessionInteractionService, + ISessionLeaseService, ISessionLifecycleService, ISessionMetadata, ISessionLegacyService, @@ -92,6 +102,8 @@ import { IWorkspaceRegistry, isError2, Error2, + heldByPeerDetailsFromInspection, + sessionLeasePath, toProtocolMessage, type ContextMessage, type IAgentScopeHandle, @@ -120,12 +132,13 @@ import { emptySessionUsage, sessionSchema, type Session, + type SessionOwnership, type SessionPendingInteraction, } from '../protocol/session'; import { workspaceIdSchema } from '../protocol/workspace'; import { z } from 'zod'; -import { errEnvelope, okEnvelope } from '../envelope'; +import { errEnvelope, okEnvelope, ownershipRedirectEnvelope } from '../envelope'; import { requestLog } from '../lib/requestLog'; import { defineRoute } from '../middleware/defineRoute'; import { ensureMainAgent, MAIN_AGENT_ID } from '../transport/mainAgent'; @@ -442,11 +455,16 @@ export function registerSessionsRoutes(app: SessionRouteHost, core: Scope): void ? (pageSize ?? DEFAULT_SESSION_LIST_PAGE_SIZE) : (pageSize ?? visible.length); const hasMore = visible.length > limit; - const projected: Session[] = visible - .slice(0, limit) - .map(({ summary, cwd, facts }) => - toWireSession(summary, cwd, facts ?? resolveSessionFacts(core, summary.id)), - ); + // Owner annotation is joined from the shared `session-leases/` tree per + // row (design §3.8): 'self' when this instance materialized the session, + // the holder's address for a routable 'peer', 'none' for a session + // materialized nowhere. Read-only per row; a vanished holder races at + // worst a 'peer' that resolves to 'none' on the next list. + const homeDir = core.accessor.get(IBootstrapService).homeDir; + const projected: Session[] = visible.slice(0, limit).map(({ summary, cwd, facts }) => ({ + ...toWireSession(summary, cwd, facts ?? resolveSessionFacts(core, summary.id)), + ownership: resolveSessionOwnership(core, homeDir, summary.id), + })); // v1 filters ordinary lists by the busy fact post-page; `archived_only` // already applied it before pagination above so it can drain to a full page. const items = @@ -1123,6 +1141,43 @@ function resolvePendingInteraction( return 'none'; } +/** + * Join one session's holder annotation from the shared `session-leases/` tree + * (design §3.8 layer 2). Classification: + * + * - 'self': this instance materialized the session and still holds its write + * lease (`ISessionLeaseService.info` survives only while held). + * - 'peer': the lease file is held (or mid-creation) under a live kernel + * lock by someone else; `address` rides along when the holder advertised + * one — the redirect target. + * - 'none': no live kernel lock on the lease file — the session is + * materialized nowhere and any instance may acquire it. A dead holder's + * lock is released by the kernel, so leftover owner metadata alone does + * not count as held. + * + * Advisory only: the sync `inspect` read can observe a holder that died the + * next millisecond; the lease's own acquisition protocol stays the authority. + */ +function resolveSessionOwnership( + core: Scope, + homeDir: string, + sessionId: string, +): SessionOwnership { + const handle = core.accessor.get(ISessionLifecycleService).get(sessionId); + if (handle !== undefined) { + const lease = handle.accessor.get(ISessionLeaseService); + if (lease.info !== undefined) return { held_by: 'self' }; + } + const inspection = core + .accessor.get(ICrossProcessLockService) + .inspect(sessionLeasePath(homeDir, sessionId)); + const heldByPeer = heldByPeerDetailsFromInspection(inspection); + if (heldByPeer === undefined) return { held_by: 'none' }; + return heldByPeer.address !== undefined + ? { held_by: 'peer', address: heldByPeer.address } + : { held_by: 'peer' }; +} + /** * Resume the session (cold-load if needed) and resolve its main agent, throwing * `session.not_found` when the session is unknown or its workspace is gone. @@ -1241,6 +1296,9 @@ function sendMappedError( stack: err.stack, }); return; + case ErrorCodes.SESSION_HELD_BY_PEER: + reply.send(ownershipRedirectEnvelope(err, requestId)); + return; case ErrorCodes.GOAL_ALREADY_EXISTS: reply.send(errEnvelope(ErrorCode.GOAL_ALREADY_EXISTS, err.message, requestId, err.stack)); return; diff --git a/packages/kap-server/src/routes/skills.ts b/packages/kap-server/src/routes/skills.ts index 861056a38d..bda8dbf080 100644 --- a/packages/kap-server/src/routes/skills.ts +++ b/packages/kap-server/src/routes/skills.ts @@ -76,6 +76,7 @@ import { IBootstrapService, IConfigService, IEventService, + IHostFileSystem, IPluginService, ISessionIndex, ISessionLifecycleService, @@ -338,6 +339,7 @@ async function listWorkspaceSkillsForRoot( ): Promise { const discovery = core.accessor.get(ISkillDiscovery); const bootstrap = core.accessor.get(IBootstrapService); + const hostFs = core.accessor.get(IHostFileSystem); const plugins = core.accessor.get(IPluginService); const config = core.accessor.get(IConfigService); await config.ready; @@ -350,12 +352,12 @@ async function listWorkspaceSkillsForRoot( const rootOptions = { mergeAllAvailableSkills }; const [userRootList, projectRootList, explicitRootList, extraRootList, pluginRootList] = await Promise.all([ - useExplicitDirs ? Promise.resolve([]) : userRoots(bootstrap.homeDir, bootstrap.osHomeDir, rootOptions), - useExplicitDirs ? Promise.resolve([]) : projectRoots(workDir, rootOptions), + useExplicitDirs ? Promise.resolve([]) : userRoots(hostFs, bootstrap.homeDir, bootstrap.osHomeDir, rootOptions), + useExplicitDirs ? Promise.resolve([]) : projectRoots(hostFs, workDir, rootOptions), useExplicitDirs - ? configuredRoots(explicitDirs, workDir, bootstrap.osHomeDir, 'user') + ? configuredRoots(hostFs, explicitDirs, workDir, bootstrap.osHomeDir, 'user') : Promise.resolve([]), - configuredRoots(extraSkillDirs, workDir, bootstrap.osHomeDir, 'extra'), + configuredRoots(hostFs, extraSkillDirs, workDir, bootstrap.osHomeDir, 'extra'), plugins.pluginSkillRoots(), ]); const [user, project, explicit, extra, plugin] = await Promise.all([ diff --git a/packages/kap-server/src/routes/snapshot.ts b/packages/kap-server/src/routes/snapshot.ts index c3e7b3eb7c..ca5c1c03f1 100644 --- a/packages/kap-server/src/routes/snapshot.ts +++ b/packages/kap-server/src/routes/snapshot.ts @@ -196,6 +196,8 @@ async function readViaLegacyAssembly( return { as_of_seq: snapState.seq, + // `epoch` is absent when the journal has no baseline (never fabricated) — + // see services/snapshot/snapshotReader.ts. epoch: snapState.epoch, session, messages: { items, has_more: hasMore }, diff --git a/packages/kap-server/src/routes/terminals.ts b/packages/kap-server/src/routes/terminals.ts index 8a0ce8b5c1..4bc551c8e1 100644 --- a/packages/kap-server/src/routes/terminals.ts +++ b/packages/kap-server/src/routes/terminals.ts @@ -32,7 +32,7 @@ import { import { createTerminalRequestSchema } from '@moonshot-ai/agent-core-v2/os/interface/terminal'; import { z } from 'zod'; -import { errEnvelope, okEnvelope } from '../envelope'; +import { errEnvelope, okEnvelope, ownershipRedirectEnvelope } from '../envelope'; import { requestLog } from '../lib/requestLog'; import { defineRoute } from '../middleware/defineRoute'; import { ErrorCode } from '../protocol/error-codes'; @@ -241,6 +241,9 @@ function sendMappedError( case ErrorCodes.TERMINAL_NOT_FOUND: reply.send(errEnvelope(ErrorCode.TERMINAL_NOT_FOUND, err.message, requestId, err.stack)); return; + case ErrorCodes.SESSION_HELD_BY_PEER: + reply.send(ownershipRedirectEnvelope(err, requestId)); + return; } } // `ISessionWorkspaceContext.assertAllowed` throws a plain (uncoded) Error when a cwd diff --git a/packages/kap-server/src/serverOrigin.ts b/packages/kap-server/src/serverOrigin.ts new file mode 100644 index 0000000000..bf57b3d951 --- /dev/null +++ b/packages/kap-server/src/serverOrigin.ts @@ -0,0 +1,11 @@ +/** + * Server origin formatting — builds parseable HTTP origins from bind hosts. + * + * Bracket-wraps IPv6 literals per RFC 3986 while leaving DNS names and IPv4 + * addresses unchanged. + */ + +export function formatServerOrigin(host: string, port: number): string { + const urlHost = host.includes(':') && !host.startsWith('[') ? `[${host}]` : host; + return `http://${urlHost}:${String(port)}`; +} diff --git a/packages/kap-server/src/services/sessionListWatch/sessionListWatchService.ts b/packages/kap-server/src/services/sessionListWatch/sessionListWatchService.ts new file mode 100644 index 0000000000..f68c86b597 --- /dev/null +++ b/packages/kap-server/src/services/sessionListWatch/sessionListWatchService.ts @@ -0,0 +1,230 @@ +/** + * `SessionListWatchService` — the event plane of multi-instance session-list + * sync. + * + * Several kap-server instances can share one home directory. The session list + * itself needs no synchronization: `ISessionIndex.list()` re-enumerates the + * shared `/sessions` tree on every request, so a peer's sessions are + * visible on the next pull. What a peer can never produce is the *event* — + * core events are process-local. This service closes that gap locally: it + * watches the shared sessions tree and, on any workspace/session directory + * appearing or disappearing, publishes ONE debounced `session.list_changed` + * hint on this instance's core `IEventService`, which the + * `SessionEventBroadcaster` fans out live (volatile, never journaled) to + * every connected WS client. Clients then re-pull the list — the directory + * scan stays the single authority, the hint is pure "go refetch" advice and + * deliberately carries no payload. + * + * Two-layer topology (a root recursive watch was rejected as an event flood): + * - root `/sessions` at depth 0: workspace directories appearing / + * disappearing — per-workspace watchers are added and removed here; + * - one depth-0 watcher per workspace directory: session directories + * appearing / disappearing. + * Existing workspaces are scanned at `start()`; every watcher runs with + * `ignoreInitial` so boot produces no hint flood. + * + * This is transport state (like `FsWatchBridge` / `SessionEventBroadcaster`): + * constructed in `start.ts` on every boot — never DI-registered — and + * disposed during server close before the core scope goes down. + */ + +import { mkdirSync } from 'node:fs'; +import { readdir } from 'node:fs/promises'; +import { join, relative, sep } from 'node:path'; + +import type { + HostFsChange, + IEventService, + IHostFsWatchHandle, + IHostFsWatchService, +} from '@moonshot-ai/agent-core-v2'; + +import type { JournalLogger } from '../../transport/ws/v1/sessionEventJournal'; + +/** Same debounce window as the storage-byte watch layer (`fileStorageService`). */ +export const SESSION_LIST_WATCH_DEBOUNCE_MS = 150; + +export class SessionListWatchService { + private readonly sessionsDir: string; + private readonly fsWatch: IHostFsWatchService; + private readonly events: IEventService; + private readonly logger: JournalLogger | undefined; + private readonly debounceMs: number; + + private rootHandle: IHostFsWatchHandle | undefined; + /** workspaceId → handle, added/removed as workspace directories come and go. */ + private readonly workspaceHandles = new Map(); + private readonly pendingWorkspaceWatchers = new Set(); + private readonly removedWorkspaces = new Set(); + private timer: ReturnType | undefined; + private started = false; + + constructor(opts: { + readonly sessionsDir: string; + readonly fsWatch: IHostFsWatchService; + readonly events: IEventService; + readonly logger?: JournalLogger; + /** Test seam: defaults to {@link SESSION_LIST_WATCH_DEBOUNCE_MS}. */ + readonly debounceMs?: number; + }) { + this.sessionsDir = opts.sessionsDir; + this.fsWatch = opts.fsWatch; + this.events = opts.events; + this.logger = opts.logger; + this.debounceMs = opts.debounceMs ?? SESSION_LIST_WATCH_DEBOUNCE_MS; + } + + /** Resolves when the initial workspace scan is done (never rejects). */ + start(): Promise { + if (this.started) return Promise.resolve(); + this.started = true; + // Watching a not-yet-existing root would leave later first-session + // creation (which mkdirs the whole chain) up to chokidar's nonexistent- + // path timing; creating the dir up front removes that race. The server + // owns this tree anyway — the first session write would create it. + try { + mkdirSync(this.sessionsDir, { recursive: true }); + } catch (error) { + this.logger?.warn( + { err: String(error) }, + 'session list watch: failed to ensure sessions dir; watching anyway', + ); + } + this.rootHandle = this.fsWatch.watch(this.sessionsDir, { recursive: false }); + this.rootHandle.onDidChange((change) => this.onRootChange(change)); + return this.rootHandle.ready + .catch((error: unknown) => { + this.logger?.warn({ err: String(error) }, 'session list watch: root watcher failed to ready'); + }) + .then(() => this.scanWorkspaces()); + } + + dispose(): void { + if (this.timer !== undefined) { + clearTimeout(this.timer); + this.timer = undefined; + } + for (const handle of this.workspaceHandles.values()) handle.dispose(); + this.workspaceHandles.clear(); + this.pendingWorkspaceWatchers.clear(); + this.removedWorkspaces.clear(); + this.rootHandle?.dispose(); + this.rootHandle = undefined; + this.started = false; + } + + private async scanWorkspaces(): Promise { + let entries; + try { + entries = await readdir(this.sessionsDir, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + this.logger?.warn({ err: String(error) }, 'session list watch: workspace scan failed'); + } + return; + } + for (const entry of entries) { + if (!this.started) return; + if (entry.isDirectory()) await this.addWorkspaceWatcher(entry.name); + } + } + + /** Root depth-0 events: only directories matter — workspace ids. */ + private onRootChange(change: HostFsChange): void { + if (change.kind !== 'directory') return; + // depth-0 watching reports direct children only; check defensively anyway. + const workspaceId = relative(this.sessionsDir, change.path); + if (workspaceId === '' || workspaceId.includes(sep)) return; + this.scheduleHint(); + if (change.action === 'created') { + this.removedWorkspaces.delete(workspaceId); + void this.addWorkspaceWatcher(workspaceId); + } else if (change.action === 'deleted') { + this.removedWorkspaces.add(workspaceId); + this.removeWorkspaceWatcher(workspaceId); + } + // The hint for the root event itself already covers a workspace that + // appeared WITH its first session inside (created between the fs event and + // our watcher attach): one debounced hint, no missed window. + } + + /** Per-workspace depth-0 events: only directories matter — session ids. */ + private onWorkspaceChange(change: HostFsChange): void { + if (change.kind !== 'directory') return; + if (change.action === 'modified') return; + this.scheduleHint(); + } + + private async addWorkspaceWatcher(workspaceId: string): Promise { + if (this.workspaceHandles.has(workspaceId) || this.pendingWorkspaceWatchers.has(workspaceId)) return; + this.pendingWorkspaceWatchers.add(workspaceId); + try { + const workspacePath = join(this.sessionsDir, workspaceId); + const before = await this.readDirectoryNames(workspacePath); + let handle: IHostFsWatchHandle; + try { + handle = this.fsWatch.watch(workspacePath, { + recursive: false, + }); + } catch (error) { + this.logger?.warn( + { workspaceId, err: String(error) }, + 'session list watch: failed to watch workspace dir', + ); + return; + } + if (!this.started || this.removedWorkspaces.has(workspaceId)) { + handle.dispose(); + return; + } + this.workspaceHandles.set(workspaceId, handle); + handle.onDidChange((change) => this.onWorkspaceChange(change)); + await handle.ready.catch((error: unknown) => { + this.logger?.warn({ workspaceId, err: String(error) }, 'session list watch: workspace watcher failed to ready'); + }); + if (!this.started || this.workspaceHandles.get(workspaceId) !== handle) return; + const after = await this.readDirectoryNames(workspacePath); + if (!sameNames(before, after)) this.scheduleHint(); + } finally { + this.pendingWorkspaceWatchers.delete(workspaceId); + } + } + + private async readDirectoryNames(path: string): Promise> { + try { + const entries = await readdir(path, { withFileTypes: true }); + return new Set(entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name)); + } catch { + return new Set(); + } + } + + private removeWorkspaceWatcher(workspaceId: string): void { + const handle = this.workspaceHandles.get(workspaceId); + if (handle === undefined) return; + this.workspaceHandles.delete(workspaceId); + handle.dispose(); + } + + private scheduleHint(): void { + if (this.timer !== undefined) clearTimeout(this.timer); + this.timer = setTimeout(() => { + this.timer = undefined; + this.publishHint(); + }, this.debounceMs); + } + + private publishHint(): void { + try { + this.events.publish({ type: 'session.list_changed', payload: {} }); + } catch (error) { + this.logger?.warn({ err: String(error) }, 'session list watch: hint publish failed'); + } + } +} + +function sameNames(left: Set, right: Set): boolean { + if (left.size !== right.size) return false; + for (const name of left) if (!right.has(name)) return false; + return true; +} diff --git a/packages/kap-server/src/services/snapshot/snapshotReader.ts b/packages/kap-server/src/services/snapshot/snapshotReader.ts index 0b2f14758e..b3add7783f 100644 --- a/packages/kap-server/src/services/snapshot/snapshotReader.ts +++ b/packages/kap-server/src/services/snapshot/snapshotReader.ts @@ -138,6 +138,8 @@ export class SnapshotReader implements ISnapshotReader { return { as_of_seq: snapState.seq, + // A journal with no baseline yet has `epoch: undefined`; on the wire the + // field is simply absent (never fabricated). epoch: snapState.epoch, session, messages: { items, has_more: hasMore }, diff --git a/packages/kap-server/src/start.ts b/packages/kap-server/src/start.ts index e5729f4373..9e11a44580 100644 --- a/packages/kap-server/src/start.ts +++ b/packages/kap-server/src/start.ts @@ -11,21 +11,27 @@ import { bootstrap, hostRequestHeadersSeed, IConfigService, + IEventService, + IHostFsWatchService, IProviderDiscoveryService, + ISessionLifecycleService, IWorkspaceRegistry, logSeed, resolveConfigPath, resolveKimiHome, resolveLoggingConfig, + sessionLeaseContactSeed, skillCatalogRuntimeOptionsSeed, type Scope, type ScopeSeed, + type SessionLeaseContact, } from '@moonshot-ai/agent-core-v2'; import { createAsyncApiDocument } from './protocol/asyncapi'; import Fastify, { type FastifyInstance } from 'fastify'; import { installErrorHandler } from './error-handler'; import { createInstanceRegistry, type InstanceRegistration } from './instanceRegistry'; +import { formatServerOrigin } from './serverOrigin'; import { transformOpenApiDocument } from './openapi/transforms'; import { registerRequestLogging } from './requestLogging'; import { resolveRequestId } from './request-id'; @@ -48,6 +54,7 @@ import { import { extractWsBearerToken } from './transport/ws/bearerProtocol'; import { SessionEventBroadcaster } from './transport/ws/v1/sessionEventBroadcaster'; import { FsWatchBridge } from './transport/ws/v1/fsWatchBridge'; +import { SkillCatalogBridge } from './transport/ws/v1/skillCatalogBridge'; import { registerWsV1, WS_PATH as WS_PATH_V1 } from './transport/ws/v1/registerWsV1'; import { getServerVersion } from './version'; import { classify } from './security/bindClassify'; @@ -60,6 +67,7 @@ import { createOriginHook, isOriginAllowed, parseCorsOrigins } from './middlewar import { createSecurityHeadersHook } from './middleware/securityHeaders'; import { createAuthHook } from './middleware/auth'; import { GuiStoreService } from './services/guiStore/guiStoreService'; +import { SessionListWatchService } from './services/sessionListWatch/sessionListWatchService'; import { loadSnapshotConfig, SnapshotReader } from './services/snapshot'; import { TranscriptService } from './services/transcript/transcriptService'; import { ModelCatalogRefreshScheduler } from './services/modelCatalog/modelCatalogRefreshScheduler'; @@ -119,7 +127,7 @@ export interface ServerStartOptions { readonly webAssetsDir?: string; /** * Host product version, reported as `server_version` (GET /api/v1/meta), in - * the OpenAPI document, session exports, the lock / instance registry, and + * the OpenAPI document, session exports, the instance registry, and * the default User-Agent. Defaults to kap-server's own package version; * embedding hosts (the CLI) should pass their own version. */ @@ -198,6 +206,10 @@ export async function startServer(opts: ServerStartOptions = {}): Promise leaseContact.current), ...(opts.seeds ?? []), ]); @@ -288,7 +301,36 @@ export async function startServer(opts: ServerStartOptions = {}): Promise => { + // Shutdown is re-entrant at two points: the remote /api/v1/shutdown route + // and the CLI signal handler both converge on this `close`, and the boot + // failure path below uses it as cleanup. The in-flight promise guard makes + // it idempotent — concurrent and repeat callers share one execution. + let closePromise: Promise | undefined; + const close = (): Promise => { + closePromise ??= doClose().catch((error) => { + closePromise = undefined; + throw error; + }); + return closePromise; + }; + const doClose = async (): Promise => { + const lifecycle = core.accessor.get(ISessionLifecycleService); + const lifecycleDrain = lifecycle.beginClose(); + // Stop the sessions-tree watcher first: a debounced hint firing mid-close + // would publish into an event bus whose subscribers are already unwinding. + sessionListWatch.dispose(); + // Release-order contract: sessions close FIRST while the transport is + // still alive, so teardown events still fan out; each session's release + // hook drains and closes its event journal before the write lease is + // released. The transport/app closes next, followed by the core scope, the + // failure limiter/scheduler, and the registration handle. + // + // Before any session scope is disposed, settle the broadcaster's in-flight + // dispatches: an `ensureState` resumed after its session's scope died + // would abandon its event (e.g. `event.session.created` for a session + // created right before shutdown), and its journal entry with it. + await Promise.all([lifecycleDrain, broadcaster.drainDispatches()]); + await lifecycle.closeAll(); await app.close(); authFailureLimiter?.dispose(); modelCatalogRefreshScheduler.dispose(); @@ -300,11 +342,26 @@ export async function startServer(opts: ServerStartOptions = {}): Promise + logger.error({ err: error }, 'session list watch failed to start'), + ); const snapshotReader = new SnapshotReader({ homeDir, @@ -379,6 +436,7 @@ export async function startServer(opts: ServerStartOptions = {}): Promise { connectionRegistry.closeAll('server shutting down'); wssV1.close(); + fsWatchBridge.dispose(); + skillCatalogBridge.dispose(); await broadcaster.close(); }); @@ -528,6 +588,10 @@ export async function startServer(opts: ServerStartOptions = {}): Promise { logger.warn( diff --git a/packages/kap-server/src/transport/errors.ts b/packages/kap-server/src/transport/errors.ts index f20c63e97e..38100b543f 100644 --- a/packages/kap-server/src/transport/errors.ts +++ b/packages/kap-server/src/transport/errors.ts @@ -47,6 +47,7 @@ const KIMI_TO_PROTOCOL: Record = { [ErrorCodes.GOAL_OBJECTIVE_EMPTY]: ErrorCode.GOAL_OBJECTIVE_EMPTY, [ErrorCodes.GOAL_OBJECTIVE_TOO_LONG]: ErrorCode.GOAL_OBJECTIVE_TOO_LONG, [ErrorCodes.GOAL_UNSUPPORTED_AGENT]: ErrorCode.GOAL_UNSUPPORTED_AGENT, + [ErrorCodes.SESSION_HELD_BY_PEER]: ErrorCode.SESSION_HELD_BY_PEER, // hostFs / storage codes → closest v1 wire equivalent (ENOTDIR collapses // into path-not-found); codes without an equivalent fall back to 50001. [ErrorCodes.OS_FS_NOT_FOUND]: ErrorCode.FS_PATH_NOT_FOUND, @@ -61,12 +62,15 @@ const KIMI_TO_PROTOCOL: Record = { /** * Map an internal error to the project envelope. `Error2` keeps its coded * mapping; everything else becomes `50001`. Stack traces are intentionally not - * surfaced. + * surfaced. Structured `Error2.details` always ride the mapped envelope + * verbatim (omitted when absent), so newly-mapped codes such as + * `session.held_by_peer` carry their ownership payload without a per-code + * branch. */ export function mapError(err: unknown, requestId: string): ReturnType { if (err instanceof Error2) { const code = KIMI_TO_PROTOCOL[err.code] ?? ErrorCode.INTERNAL_ERROR; - return errEnvelope(code, err.message, requestId, err.stack); + return errEnvelope(code, err.message, requestId, err.stack, err.details); } if (err instanceof TimeoutError) { return errEnvelope(ErrorCode.INTERNAL_ERROR, err.message, requestId, err.stack); diff --git a/packages/kap-server/src/transport/ws/v1/events.ts b/packages/kap-server/src/transport/ws/v1/events.ts index 22efbc7c8f..94db717092 100644 --- a/packages/kap-server/src/transport/ws/v1/events.ts +++ b/packages/kap-server/src/transport/ws/v1/events.ts @@ -14,6 +14,10 @@ import type { DomainEvent } from '@moonshot-ai/agent-core-v2/app/event/eventBus' import type { MessageContent } from '@moonshot-ai/agent-core-v2/agent/contextMemory/protocolMessage'; import type { PermissionMode } from '@moonshot-ai/agent-core-v2/agent/permissionPolicy/types'; import type { UsageStatus } from '@moonshot-ai/agent-core-v2/agent/usage/usage'; +import type { + ProviderRefreshChange, + ProviderRefreshFailure, +} from '@moonshot-ai/agent-core-v2/kosong/model/discovery'; import type { AgentPhase } from '../../../services/legacyStatus/legacyStatus'; import type { ConfigResponse } from '../../../protocol/rest-config'; import type { Session, SessionPendingInteraction } from '../../../protocol/session'; @@ -52,6 +56,32 @@ export interface SessionCreatedEvent { readonly session: Session; } +/** + * Volatile, payload-less hint that the set of sessions on disk changed + * (design §3.8): a workspace or session directory appeared or vanished under + * the shared `/sessions` tree, possibly created by ANOTHER server + * instance sharing the home. Clients should re-pull `GET /sessions` instead + * of reading anything into the event itself — it is fanned out live only + * (never journaled, never replayed). + */ +export interface SessionListChangedEvent { + readonly type: 'session.list_changed'; +} + +/** + * Volatile per-session hint that the session's skill catalog changed: a skill + * file or directory under one of the watched sources appeared, changed, or + * vanished (mirrors the core `ISessionSkillCatalog.onDidChange` feed, whose + * payload is the changed source id). Fanned out live only (never journaled, + * never replayed) to connections subscribed to that session — clients should + * re-pull `GET /sessions/{sid}/skills` instead of reading anything into the + * hint itself. + */ +export interface SkillCatalogChangedEvent { + readonly type: 'skill_catalog.changed'; + readonly sourceId: string; +} + export interface WorkspaceCreatedEvent { readonly type: 'event.workspace.created'; readonly workspace: Workspace; @@ -96,6 +126,20 @@ export interface ConfigChangedEvent { readonly config: ConfigResponse; } +/** + * Published (core event bus) when the provider/model catalog refreshes (auth + * change, config edit, scheduled) and the effective catalog changed. Carries + * the per-provider diff so clients can both refresh their model/provider + * caches and surface a summary ("3 models added") without re-diffing the + * whole config. + */ +export interface ModelCatalogChangedEvent { + readonly type: 'event.model_catalog.changed'; + readonly changed: readonly ProviderRefreshChange[]; + readonly unchanged: readonly string[]; + readonly failed: readonly ProviderRefreshFailure[]; +} + export interface PromptSubmittedEvent { readonly type: 'prompt.submitted'; readonly promptId: string; @@ -172,12 +216,15 @@ export type AgentEvent = | AgentDisposedEvent | SessionMetaUpdatedEvent | SessionCreatedEvent + | SessionListChangedEvent + | SkillCatalogChangedEvent | WorkspaceCreatedEvent | WorkspaceUpdatedEvent | WorkspaceDeletedEvent | SessionWorkChangedEvent | SessionStatusChangedEvent | ConfigChangedEvent + | ModelCatalogChangedEvent | PromptSubmittedEvent | BackgroundTaskStartedEvent | BackgroundTaskTerminatedEvent; diff --git a/packages/kap-server/src/transport/ws/v1/fsWatchBridge.ts b/packages/kap-server/src/transport/ws/v1/fsWatchBridge.ts index 7a0d100903..e4978c5653 100644 --- a/packages/kap-server/src/transport/ws/v1/fsWatchBridge.ts +++ b/packages/kap-server/src/transport/ws/v1/fsWatchBridge.ts @@ -14,9 +14,10 @@ * {@link SessionEventBroadcaster}); it is **not** DI-registered and carries no * `_serviceBrand`. It owns the per-`(connection, session)` subscription sets, * fans the core feed out to each connection filtered by that connection's - * paths, and assigns a per-session monotonic `seq`. Frames are sent straight - * to the socket — they never enter the broadcaster / journal (fs changes are - * volatile: on overflow the client sees `truncated` and re-syncs). + * paths, and numbers the delivered frames per connection. Frames are sent + * straight to the socket — they never enter the broadcaster / journal. The + * `seq` / `truncated` / rebuild contract is pinned on `fsChangeEventSchema` in + * `@moonshot-ai/protocol` (`packages/protocol/src/fs.ts`) — see there. * * The core `ISessionFsWatchService` keeps a single subscription set per * session; the bridge drives it with the **union** of every connection's @@ -36,6 +37,7 @@ import { import type { FsChangeEntry, FsChangeEvent } from '@moonshot-ai/agent-core-v2/session/sessionFs/fsWatch'; import type { EventEnvelope, JournalLogger } from './sessionEventJournal'; +import { registerSessionReleaseHook } from './sessionReleaseHook'; const MAX_PATHS_PER_CONNECTION = 100; @@ -70,6 +72,8 @@ export interface FsWatchAck { interface ConnEntry { readonly conn: FsWatchConnection; readonly paths: Set; + /** Per-connection monotonic frame counter; starts at 0, pre-incremented per delivery. */ + seq: number; } interface SessionWatch { @@ -78,20 +82,30 @@ interface SessionWatch { readonly fsWatch: ISessionFsWatchService; readonly workspace: ISessionWorkspaceContext; readonly conns: Map; - union: Set; - seq: number; sub: IDisposable | undefined; } -export class FsWatchBridge { +export class FsWatchBridge implements IDisposable { private readonly core: Scope; private readonly logger: JournalLogger | undefined; + private readonly sessionReleaseHook: IDisposable; private readonly bySession = new Map(); private readonly connPathCount = new Map(); constructor(opts: { core: Scope; logger?: JournalLogger }) { this.core = opts.core; this.logger = opts.logger; + this.sessionReleaseHook = registerSessionReleaseHook( + this.core, + 'fsWatchBridge', + (sessionId) => this.releaseSession(sessionId), + ); + } + + dispose(): void { + this.sessionReleaseHook.dispose(); + for (const sw of Array.from(this.bySession.values())) this.dropSession(sw, 'teardown'); + this.connPathCount.clear(); } async addWatch( @@ -126,7 +140,7 @@ export class FsWatchBridge { } if (entry === undefined) { - entry = { conn, paths: new Set() }; + entry = { conn, paths: new Set(), seq: 0 }; sw.conns.set(conn.id, entry); } for (const rel of toAdd) entry.paths.add(rel); @@ -155,7 +169,7 @@ export class FsWatchBridge { this.connPathCount.set(conn.id, Math.max(0, this.countFor(conn.id) - removed)); if (entry.paths.size === 0) sw.conns.delete(conn.id); this.recomputeAndApply(sw); - if (sw.conns.size === 0) this.teardownSession(sw); + if (sw.conns.size === 0) this.dropSession(sw, 'teardown'); return this.ok(sw, conn); } @@ -168,7 +182,7 @@ export class FsWatchBridge { sw.conns.delete(conn.id); this.connPathCount.set(conn.id, Math.max(0, this.countFor(conn.id) - entry.paths.size)); this.recomputeAndApply(sw); - if (sw.conns.size === 0) this.teardownSession(sw); + if (sw.conns.size === 0) this.dropSession(sw, 'teardown'); } this.connPathCount.delete(conn.id); } @@ -184,8 +198,6 @@ export class FsWatchBridge { fsWatch: session.accessor.get(ISessionFsWatchService), workspace: session.accessor.get(ISessionWorkspaceContext), conns: new Map(), - union: new Set(), - seq: 0, sub: undefined, }; this.bySession.set(sessionId, sw); @@ -197,35 +209,51 @@ export class FsWatchBridge { for (const { paths } of sw.conns.values()) { for (const p of paths) union.add(p); } - sw.union = union; sw.fsWatch.setWatchedPaths([...union]); if (union.size > 0 && sw.sub === undefined) { sw.sub = sw.fsWatch.onDidChangeFiles((ev) => this.onSessionEvent(sw.id, ev)); } } - private teardownSession(sw: SessionWatch): void { + private releaseSession(sessionId: string): void { + const sw = this.bySession.get(sessionId); + if (sw === undefined) return; + this.dropSession(sw, 'scope-released'); + } + + private dropSession(sw: SessionWatch, reason: 'teardown' | 'scope-released'): void { sw.sub?.dispose(); sw.sub = undefined; - sw.fsWatch.setWatchedPaths([]); + for (const entry of sw.conns.values()) { + const remaining = Math.max(0, this.countFor(entry.conn.id) - entry.paths.size); + if (remaining === 0) this.connPathCount.delete(entry.conn.id); + else this.connPathCount.set(entry.conn.id, remaining); + } + sw.conns.clear(); + if (reason === 'teardown') sw.fsWatch.setWatchedPaths([]); this.bySession.delete(sw.id); } private onSessionEvent(sessionId: string, ev: FsChangeEvent): void { const sw = this.bySession.get(sessionId); if (sw === undefined) return; - for (const { conn, paths } of sw.conns.values()) { + for (const entry of sw.conns.values()) { + const { conn, paths } = entry; let changes: FsChangeEntry[]; if (ev.truncated === true) { + // A truncated window is broadcast to every connection of the session + // watch, even one whose path set matches nothing: after it, no + // incremental state can be trusted, so every client must resync. changes = []; } else { changes = ev.changes.filter((c) => isUnderAny(c.path, paths)); + // No frame for this connection → its seq does not advance. if (changes.length === 0) continue; } - sw.seq += 1; + entry.seq += 1; const frame: FsChangedFrame = { type: 'event.fs.changed', - seq: sw.seq, + seq: entry.seq, session_id: sessionId, timestamp: new Date().toISOString(), payload: { diff --git a/packages/kap-server/src/transport/ws/v1/registerWsV1.ts b/packages/kap-server/src/transport/ws/v1/registerWsV1.ts index a6b39c1081..58e66c77a8 100644 --- a/packages/kap-server/src/transport/ws/v1/registerWsV1.ts +++ b/packages/kap-server/src/transport/ws/v1/registerWsV1.ts @@ -15,6 +15,7 @@ import type { CredentialValidator } from '../../../services/auth/credentials'; import { type IConnectionRegistry } from '../connectionRegistry'; import type { SessionEventBroadcaster } from './sessionEventBroadcaster'; import type { FsWatchBridge } from './fsWatchBridge'; +import type { SkillCatalogBridge } from './skillCatalogBridge'; import type { JournalLogger } from './sessionEventJournal'; import { WsConnectionV1 } from './wsConnectionV1'; import { selectWsBearerProtocol } from '../bearerProtocol'; @@ -27,6 +28,7 @@ export interface RegisterWsV1Options { readonly registry: IConnectionRegistry; readonly broadcaster: SessionEventBroadcaster; readonly fsWatchBridge: FsWatchBridge; + readonly skillCatalogBridge: SkillCatalogBridge; readonly logger?: JournalLogger; readonly maxBufferSize?: number; readonly flushIntervalMs?: number; @@ -44,6 +46,7 @@ export function registerWsV1(core: Scope, opts: RegisterWsV1Options): WebSocketS socket, broadcaster, fsWatchBridge: opts.fsWatchBridge, + skillCatalogBridge: opts.skillCatalogBridge, connectionRegistry: registry, validateCredential: opts.validateCredential, remoteAddress: req.socket.remoteAddress ?? null, diff --git a/packages/kap-server/src/transport/ws/v1/sessionEventBroadcaster.ts b/packages/kap-server/src/transport/ws/v1/sessionEventBroadcaster.ts index 200f7311d4..0a273ee3f1 100644 --- a/packages/kap-server/src/transport/ws/v1/sessionEventBroadcaster.ts +++ b/packages/kap-server/src/transport/ws/v1/sessionEventBroadcaster.ts @@ -42,20 +42,29 @@ import type { Interaction, InteractionKind, ISessionScopeHandle, + SessionOwnershipDetails, Scope, } from '@moonshot-ai/agent-core-v2'; import { IAgentLifecycleService, IAgentActivityView, + ICrossProcessLockService, IEventBus, IEventService, ISessionInteractionService, ISessionIndex, ISessionLifecycleService, MAIN_AGENT_ID, + heldByPeerDetailsFromInspection, + sessionLeasePath, } from '@moonshot-ai/agent-core-v2'; import type { TurnEndReason } from '@moonshot-ai/agent-core-v2/agent/loop/turnEvents'; -import type { SessionCreatedEvent, SessionMetaUpdatedEvent, Event } from './events'; +import type { + ModelCatalogChangedEvent, + SessionCreatedEvent, + SessionMetaUpdatedEvent, + Event, +} from './events'; import { isVolatileEventType } from './events'; import type { SessionCursor } from '../../../protocol/ws-control'; import type { InFlightTurn, SnapshotSubagent } from '../../../protocol/rest-snapshot'; @@ -85,6 +94,7 @@ import { SessionEventJournal, sessionJournalPath, } from './sessionEventJournal'; +import { registerSessionReleaseHook } from './sessionReleaseHook'; export type ResyncReason = 'buffer_overflow' | 'session_recreated' | 'epoch_changed'; @@ -93,12 +103,17 @@ export interface BufferedSinceResult { /** When set, the client must rebuild from the snapshot and re-subscribe. */ resyncRequired: ResyncReason | false; currentSeq: number; - epoch: string; + /** + * Current journal epoch, if the journal has a baseline. `undefined` (absent + * on the wire) means "no baseline yet" — distinct from a changed baseline. + */ + epoch: string | undefined; } export interface SessionSnapshotState { seq: number; - epoch: string; + /** Current journal epoch, or `undefined` while the journal has no baseline. */ + epoch: string | undefined; inFlightTurn: InFlightTurn | null; subagents: SnapshotSubagent[]; } @@ -188,6 +203,9 @@ const TRANSCRIPT_RESET_TAIL_TURNS = 30; async function disposeSessionState(state: SessionState): Promise { for (const d of state.lifecycleDisposables) d.dispose(); for (const d of state.agentDisposables.values()) d.dispose(); + // Drain every queued dispatch (the journal append happens inside the queue) + // before the final flush — otherwise close() drops the dispatch tail. + await state.queue; await state.journal.close(); } @@ -205,13 +223,20 @@ export class SessionEventBroadcaster { * per-chunk `AABBCC` stream while every seq and offset still looks valid. */ private readonly pendingStates = new Map>(); + private readonly releasingSessions = new Map>(); private readonly maxBufferSize: number; private readonly coreEventSubscription: IDisposable; + private readonly sessionReleaseHook: IDisposable; + /** Fire-and-forget dispatches still in flight (possibly inside `ensureState`). */ + private readonly inFlightDispatches = new Set>(); private closed = false; + /** Process-local monotonic seq for volatile global events (never journaled). */ + private globalSeq = 0; constructor( private readonly opts: { readonly eventsDir: string; + readonly homeDir: string; readonly core: Scope; readonly logger?: JournalLogger; readonly maxBufferSize?: number; @@ -227,6 +252,11 @@ export class SessionEventBroadcaster { this.coreEventSubscription = opts.core.accessor .get(IEventService) .subscribe((event) => this.onCoreEvent(event)); + this.sessionReleaseHook = registerSessionReleaseHook( + opts.core, + 'sessionEventBroadcaster', + (sessionId) => this.releaseSessionState(sessionId), + ); } /** @@ -280,6 +310,16 @@ export class SessionEventBroadcaster { return true; } + async getSubscriptionFailure(sessionId: string): Promise { + const summary = await this.opts.core.accessor.get(ISessionIndex).get(sessionId); + if (summary === undefined) return undefined; + + const inspection = this.opts.core + .accessor.get(ICrossProcessLockService) + .inspect(sessionLeasePath(this.opts.homeDir, sessionId)); + return heldByPeerDetailsFromInspection(inspection); + } + /** * Whether `subscribeTranscript` will send at least one reset for this * (target, spec) pair right now — an upgrade over the previous grades or an @@ -523,11 +563,12 @@ export class SessionEventBroadcaster { ): Promise { const state = await this.ensureState(sessionId); if (state === undefined) { - return { events: [], resyncRequired: 'session_recreated', currentSeq: 0, epoch: '' }; + return { events: [], resyncRequired: 'session_recreated', currentSeq: 0, epoch: undefined }; } // Drain so the cursor reflects everything dispatched so far. await state.queue; const { journal, tail } = state; + if (journal.flushInFlight || cursor.seq === 0) await journal.flush(); const currentSeq = journal.seq; const { epoch } = journal; @@ -556,20 +597,29 @@ export class SessionEventBroadcaster { : entries.filter(({ envelope }) => matchesAgentFilter(envelope, filter)); // Serve from the memory tail when it fully covers the gap; else the journal. - const tailStart = tail[0]?.seq; - if (tailStart !== undefined && tailStart <= cursor.seq + 1) { - const events = applyFilter(tail.filter((e) => e.seq > cursor.seq)); - return { events, resyncRequired: false, currentSeq, epoch }; + // While the journal has unrecovered write failures, never serve from the + // tail: those events are not durable, and replaying them as if they were + // would resurrect the "fake durable" hole after a restart. `readSince` + // flushes first (which retries a transient write failure); once the + // journal is sticky the flush is a no-op and `readSince` throws the + // sticky JournalStorageError instead — the replay edge maps that to a + // client-visible resync. + if (!journal.writeFailure) { + const tailStart = tail[0]?.seq; + if (tailStart !== undefined && tailStart <= cursor.seq + 1) { + const events = applyFilter(tail.filter((e) => e.seq > cursor.seq)); + return { events, resyncRequired: false, currentSeq, epoch }; + } } const fromDisk = await journal.readSince(cursor.seq, this.maxBufferSize); return { events: applyFilter(fromDisk), resyncRequired: false, currentSeq, epoch }; } - async getCursor(sessionId: string): Promise<{ seq: number; epoch: string }> { + async getCursor(sessionId: string): Promise<{ seq: number; epoch: string | undefined }> { const state = await this.ensureState(sessionId); if (state === undefined) { const cold = await this.readColdWatermark(sessionId); - return cold ?? { seq: 0, epoch: '' }; + return cold ?? { seq: 0, epoch: undefined }; } await state.queue; return { seq: state.journal.seq, epoch: state.journal.epoch }; @@ -582,7 +632,7 @@ export class SessionEventBroadcaster { const cold = await this.readColdWatermark(sessionId); return cold !== undefined ? { ...cold, inFlightTurn: null, subagents: [] } - : { seq: 0, epoch: '', inFlightTurn: null, subagents: [] }; + : { seq: 0, epoch: undefined, inFlightTurn: null, subagents: [] }; } await state.queue; return { @@ -598,11 +648,12 @@ export class SessionEventBroadcaster { * (carried over from a prior process, or created by v1). Opens the journal * transiently — no agent/interaction listeners and not cached in * `this.sessions` — so a later live activation still attaches subscriptions. + * The open is read-only (zero bytes written, no fabricated epoch). * Returns `undefined` when the session is unknown to the index (truly absent). */ private async readColdWatermark( sessionId: string, - ): Promise<{ seq: number; epoch: string } | undefined> { + ): Promise<{ seq: number; epoch: string | undefined } | undefined> { const summary = await this.opts.core.accessor.get(ISessionIndex).get(sessionId); if (summary === undefined) return undefined; const journal = await SessionEventJournal.open( @@ -616,19 +667,34 @@ export class SessionEventBroadcaster { async close(): Promise { if (this.closed) return; + // Settle dispatches already in flight before flagging closed — a dispatch + // still inside `ensureState` would otherwise see `closed` and abandon its + // event (and its journal state). + await this.drainDispatches(); this.closed = true; this.coreEventSubscription.dispose(); - for (const [sessionId, state] of this.sessions) { - await disposeSessionState(state); - // Transcript bindings die with the session stream (its store - // subscriptions were disposed above; the producer binding goes here). - this.opts.transcriptService?.dropSession(sessionId); + this.sessionReleaseHook.dispose(); + const sessionIds = new Set([...this.sessions.keys(), ...this.pendingStates.keys()]); + for (const sessionId of sessionIds) { + await this.releaseSessionState(sessionId); + } + } + + /** + * Settle every fire-and-forget dispatch currently in flight. start.ts calls + * this BEFORE disposing session scopes (an `ensureState` resumed after its + * session's scope died would abandon the dispatch, losing e.g. the + * `event.session.created` journal entry for a short-lived session); close() + * calls it before flipping `closed` for the same reason. + */ + async drainDispatches(): Promise { + while (this.inFlightDispatches.size > 0) { + await Promise.all(this.inFlightDispatches); } - this.sessions.clear(); } private ensureState(sessionId: string): Promise { - if (this.closed) return Promise.resolve(undefined); + if (this.closed || this.releasingSessions.has(sessionId)) return Promise.resolve(undefined); const existing = this.sessions.get(sessionId); if (existing !== undefined) return Promise.resolve(existing); let pending = this.pendingStates.get(sessionId); @@ -653,7 +719,7 @@ export class SessionEventBroadcaster { sessionJournalPath(this.opts.eventsDir, sessionId), this.opts.logger, ); - if (this.closed) { + if (this.closed || this.releasingSessions.has(sessionId)) { await journal.close(); return undefined; } @@ -697,50 +763,57 @@ export class SessionEventBroadcaster { return state; } - private ensureGlobalState(): Promise { - const existing = this.sessions.get(GLOBAL_SESSION_ID); - if (existing !== undefined) return Promise.resolve(existing); - let pending = this.pendingStates.get(GLOBAL_SESSION_ID); - if (pending === undefined) { - pending = this.createGlobalState().finally(() => { - if (this.pendingStates.get(GLOBAL_SESSION_ID) === pending) { - this.pendingStates.delete(GLOBAL_SESSION_ID); - } - }); - this.pendingStates.set(GLOBAL_SESSION_ID, pending); - } - return pending as Promise; + private releaseSessionState(sessionId: string): Promise { + const existing = this.releasingSessions.get(sessionId); + if (existing !== undefined) return existing; + const release = this.disposeSession(sessionId).finally(() => { + if (this.releasingSessions.get(sessionId) === release) { + this.releasingSessions.delete(sessionId); + } + }); + this.releasingSessions.set(sessionId, release); + return release; } - private async createGlobalState(): Promise { - const journal = await SessionEventJournal.open( - sessionJournalPath(this.opts.eventsDir, GLOBAL_SESSION_ID), - this.opts.logger, - ); - const state: SessionState = { - sessionId: GLOBAL_SESSION_ID, - journal, - tracker: new InFlightTurnTracker(), - roster: new SubagentRosterTracker(), - activityByAgent: new Map(), - emittedBusy: false, - emittedMainTurnActive: false, - emittedPendingInteraction: 'none', - pendingInteraction: 'none', - tail: [], - targets: new Map(), - queue: Promise.resolve(), - agentDisposables: new Map(), - lifecycleDisposables: [], - knownInteractions: new Map(), - transcriptSeeded: new Set(), - deferredTranscriptSeeds: new Map(), - }; - this.sessions.set(GLOBAL_SESSION_ID, state); - return state; + private async disposeSession(sessionId: string): Promise { + await this.pendingStates.get(sessionId); + const state = this.sessions.get(sessionId); + if (state === undefined) return; + this.sessions.delete(sessionId); + try { + await disposeSessionState(state); + } finally { + this.opts.transcriptService?.dropSession(sessionId); + } } private onCoreEvent(event: GlobalEvent): void { + if (event.type === 'session.list_changed') { + // Published by `SessionListWatchService` when a workspace/session + // directory appears or disappears under the shared sessions tree + // (possibly by a peer instance). Payload-less go-refetch advice: fanned + // out volatile under the global watermark, never journaled. + this.dispatchGlobal({ + type: 'session.list_changed', + agentId: 'main', + sessionId: GLOBAL_SESSION_ID, + }); + return; + } + if (event.type === 'event.model_catalog.changed') { + const payload = modelCatalogChangedPayload(event.payload); + if (payload === undefined) return; + const modelEvent: ModelCatalogChangedEvent = { + type: 'event.model_catalog.changed', + ...payload, + }; + this.dispatchGlobal({ + ...modelEvent, + agentId: 'main', + sessionId: GLOBAL_SESSION_ID, + }); + return; + } if (event.type === 'event.session.created') { const payload = sessionCreatedPayload(event.payload); if (payload === undefined) return; @@ -752,13 +825,15 @@ export class SessionEventBroadcaster { // `sessionStatusChanged` reducer is a no-op for the unknown session and // kimi-web's Stop button (gated on session.status === 'running') never // renders. Mirrors v1's `isGlobalSessionEvent` broadcast of creation. - void this.dispatchSessionEvent(payload.sessionId, { - type: 'event.session.created', - session: payload.session, - agentId: 'main', - sessionId: payload.sessionId, - } as Event).catch((error: unknown) => - this.logDispatchError(payload.sessionId, 'event.session.created', error), + this.trackDispatch( + this.dispatchSessionEvent(payload.sessionId, { + type: 'event.session.created', + session: payload.session, + agentId: 'main', + sessionId: payload.sessionId, + } as Event), + payload.sessionId, + 'event.session.created', ); return; } @@ -775,22 +850,54 @@ export class SessionEventBroadcaster { // exactly like v1. const sessionId = sessionMetaUpdatedSessionId(event.payload); if (sessionId === undefined) return; - void this.dispatchSessionEvent(sessionId, { - type: 'session.meta.updated', - ...payload, - agentId: 'main', + this.trackDispatch( + this.dispatchSessionEvent(sessionId, { + type: 'session.meta.updated', + ...payload, + agentId: 'main', + sessionId, + } as Event), sessionId, - } as Event).catch((error: unknown) => - this.logDispatchError(sessionId, 'session.meta.updated', error), + 'session.meta.updated', ); } } - private async dispatchGlobal(event: Event): Promise { - const state = await this.ensureGlobalState(); - state.queue = state.queue - .then(() => this.dispatch(state, event, isVolatileEventType(event.type))) - .catch((error: unknown) => this.logDispatchDropped(state.sessionId, event.type, error)); + /** + * Fire a dispatch without awaiting it, but keep it drainable: errors are + * logged exactly once here, and `drainDispatches()` / `close()` can settle + * the still-running dispatch before session scopes are disposed. + */ + private trackDispatch(promise: Promise, sessionId: string, eventType: string): void { + const tracked = promise.catch((error: unknown) => + this.logDispatchError(sessionId, eventType, error), + ); + this.inFlightDispatches.add(tracked); + void tracked.finally(() => this.inFlightDispatches.delete(tracked)); + } + + /** + * Fan a global advisory event out to every live connection. Deliberately + * volatile: never journaled and never replayed — the old `__global__.jsonl` + * was a multi-writer artifact with no read path (pure disk garbage), and its + * only payload (`event.model_catalog.changed`) is cache-refresh advice. + * `seq` is a process-local monotonic counter, NOT a durable watermark, and + * the envelope carries no epoch. + */ + private dispatchGlobal(event: Event): void { + this.globalSeq += 1; + const envelope = this.buildEnvelope(this.globalSeq, GLOBAL_SESSION_ID, event, { + volatile: true, + }); + // Global events bypass the agent allowlist entirely (isGlobalEvent), same + // fan-out rule as `dispatch` — only transport is direct, not per-session. + for (const target of this.allTargets()) { + try { + target.send(envelope); + } catch { + // best-effort fan-out; a broken target is dropped, not fatal + } + } } /** @@ -1095,13 +1202,17 @@ export class SessionEventBroadcaster { } /** - * A queued dispatch rejected: the event is permanently lost (and, for durable - * events, the seq is skipped). Warn instead of swallowing it silently. + * A queued dispatch rejected: the event is permanently lost. For a durable + * event the seq is skipped, leaving an on-disk seq gap — that gap is itself + * the resync boundary: clients detect it and rebuild via resync, so the + * loss stays observable instead of silent. Journal storage failures also + * land here (`nextSeq()`/`append()` throw once the journal is sticky). + * Warn, never swallow. */ private logDispatchDropped(sessionId: string, eventType: string, error: unknown): void { this.opts.logger?.warn( { sessionId, eventType, err: error }, - 'session event dispatch failed; event dropped', + 'session event dispatch failed; event dropped (durable seq gap → client resyncs)', ); } @@ -1166,9 +1277,11 @@ export class SessionEventBroadcaster { } private *allTargets(): Iterable { + const targets = new Set(); for (const state of this.sessions.values()) { - for (const target of state.targets.keys()) yield target; + for (const target of state.targets.keys()) targets.add(target); } + yield* targets; } } @@ -1430,3 +1543,27 @@ function sessionCreatedPayload( if (sessionId === undefined || session === undefined) return undefined; return { sessionId, session }; } + +/** + * Validate the `event.model_catalog.changed` payload published on the core + * `IEventService` (`IModelCatalogService` refresh / auth change): the three + * per-provider diff arrays are all the v1 frame carries. + */ +function modelCatalogChangedPayload( + payload: unknown, +): Pick | undefined { + if (typeof payload !== 'object' || payload === null) return undefined; + const candidate = payload as Partial; + if ( + !Array.isArray(candidate.changed) || + !Array.isArray(candidate.unchanged) || + !Array.isArray(candidate.failed) + ) { + return undefined; + } + return { + changed: candidate.changed, + unchanged: candidate.unchanged, + failed: candidate.failed, + }; +} diff --git a/packages/kap-server/src/transport/ws/v1/sessionEventJournal.ts b/packages/kap-server/src/transport/ws/v1/sessionEventJournal.ts index a2a29ef5bf..8498c4fb1e 100644 --- a/packages/kap-server/src/transport/ws/v1/sessionEventJournal.ts +++ b/packages/kap-server/src/transport/ws/v1/sessionEventJournal.ts @@ -11,25 +11,67 @@ * Invariants: * - `seq` is assigned at append time, starts at 1, and is monotonic across * server restarts (recovered by scanning the file on open). - * - `epoch` identifies this journal incarnation. It changes only when the - * file is unreadable/corrupt at open (we start a fresh journal) — clients - * holding cursors from the old epoch get `resync_required(epoch_changed)`. + * - `epoch` identifies this journal incarnation. It is born lazily on the + * FIRST durable append (never on a cold read) and changes only when the + * file is unreadable/corrupt at open (the next append starts a fresh + * journal) — clients holding cursors from the old epoch get + * `resync_required(epoch_changed)`. A journal with no baseline yet reports + * `epoch: undefined`; absent on the wire means "no baseline", which is + * distinct from "baseline changed". + * - On open with several headers (crash after a rotation) the LAST header + * wins — only the newest incarnation is authoritative. * - Only durable events are written (volatile frames never touch the journal; * see `VOLATILE_EVENT_TYPES` in `./events`). * * Durability model: `append()` is synchronous (callers need the seq immediately - * for fan-out); bytes are flushed on a microtask-scheduled async batch. - * `readSince()` flushes first so replay never misses queued lines. A torn - * trailing line from a crash is tolerated and ignored on open. + * for fan-out); bytes are flushed on a microtask-scheduled async batch. Each + * batch uses a single `open(path, 'a')` → write → fsync → close cycle. Pending + * lines are dequeued only AFTER the batch is durable; a failed round keeps the + * whole batch (and the pending header) for a retry driven by the next + * append-scheduled or read-triggered flush. After + * {@link STICKY_FAILURE_THRESHOLD} consecutive failures the journal goes + * sticky: `flush()` turns into a no-op (the kept pending lines are never + * retried), `nextSeq()`/`append()` fail fast (pending can never grow + * unbounded), and `readSince()` throws a {@link JournalStorageError} instead + * of silently serving fewer events — "not served" must stay distinguishable + * from "nothing to serve". `readSince()` flushes first so replay never misses + * queued lines. + * A torn trailing line from a crash is tolerated and ignored on open, and a + * pure cold-read open → close writes zero bytes. */ import { createReadStream } from 'node:fs'; -import { appendFile, mkdir } from 'node:fs/promises'; +import { mkdir, open as openFile, readFile, rename, truncate } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { ulid } from 'ulid'; const JOURNAL_VERSION = 1; +/** + * Consecutive write failures that move the journal into the sticky storage + * failure state. Bounded so a single transient error is retried by the next + * flush round, while a persistently failing disk fails fast instead of growing + * the pending queue forever. + */ +const STICKY_FAILURE_THRESHOLD = 2; + +/** + * Explicit journal storage failure. Thrown by `nextSeq()`/`append()` (sticky + * fail-fast) and by `readSince()` (never answer a replay from a journal whose + * writes failed). Distinguishable via `instanceof` so the replay edge can map + * it to a client-visible resync instead of an empty event page. + */ +export class JournalStorageError extends Error { + readonly filePath: string; + + constructor(filePath: string, cause: unknown) { + const causeMessage = cause instanceof Error ? cause.message : String(cause); + super(`event journal storage failed for ${filePath}: ${causeMessage}`, { cause }); + this.name = 'JournalStorageError'; + this.filePath = filePath; + } +} + /** * Wire event envelope — matches `wsEventEnvelopeSchema` / * `sessionEventMessageSchema` in the local `protocol/ws-control` catalog. Defined @@ -59,6 +101,10 @@ interface JournalEventLine { envelope: EventEnvelope; } +type JournalTailRepair = + | { readonly kind: 'terminate' } + | { readonly kind: 'truncate'; readonly byteLength: number }; + export interface JournalEntry { seq: number; envelope: EventEnvelope; @@ -76,17 +122,22 @@ export class SessionEventJournal { private _seq: number; private pendingLines: string[] = []; private flushPromise: Promise | undefined; - private headerPending: boolean; + private headerPending = false; + private currentEpoch: string | undefined; + private consecutiveFailures = 0; + private stickyError: JournalStorageError | undefined; + private tailRepair: JournalTailRepair | undefined; private constructor( private readonly filePath: string, private readonly logger: JournalLogger, - public readonly epoch: string, + epoch: string | undefined, lastSeq: number, - isFresh: boolean, + tailRepair?: JournalTailRepair, ) { this._seq = lastSeq; - this.headerPending = isFresh; + this.currentEpoch = epoch; + this.tailRepair = tailRepair; } /** Highest durable seq appended (0 if none). */ @@ -95,55 +146,128 @@ export class SessionEventJournal { } /** - * Open (or create) the journal for `filePath`. Scans an existing file to - * recover `{epoch, lastSeq}`. A missing file or an unreadable header starts - * a fresh journal with a new epoch. + * Current journal epoch. `undefined` until the first durable append stamps a + * header — a journal with no baseline must present "absent", not a random + * placeholder (repeated cold reads of the same journal used to yield + * different fabricated epochs and trigger fake `epoch_changed` resyncs). + */ + get epoch(): string | undefined { + return this.currentEpoch; + } + + /** + * Whether writes have failed and not yet recovered, i.e. pending lines are + * not durably on disk. While this holds, replay must consult the journal + * itself (`readSince`, which retries the flush and throws once sticky) + * rather than any in-memory copy of the events. */ - static async open(filePath: string, logger: JournalLogger = noopLogger): Promise { + get writeFailure(): boolean { + return this.stickyError !== undefined || this.consecutiveFailures > 0; + } + + get flushInFlight(): boolean { + return this.flushPromise !== undefined; + } + + /** + * Open (or create-on-first-append) the journal for `filePath`. Scans an + * existing file to recover `{epoch, lastSeq}`. This open is READ-ONLY: a + * missing file or an unreadable/missing header yields a journal with + * `epoch: undefined` and seq 0, writes nothing, and defers the fresh epoch + * to the first real `append()`. With several headers the last one wins. + */ + static async open( + filePath: string, + logger: JournalLogger = noopLogger, + ): Promise { let epoch: string | undefined; let lastSeq = 0; let sawAnyLine = false; + let corrupt = false; + let segmentSeq = 0; + let tailRepair: JournalTailRepair | undefined; try { - for await (const raw of readLines(filePath)) { + for await (const line of readLines(filePath)) { + const raw = line.raw; + if (raw.trim().length === 0) { + if (!line.terminated) tailRepair = { kind: 'truncate', byteLength: line.startOffset }; + continue; + } sawAnyLine = true; const parsed = parseJournalLine(raw); - if (parsed === undefined) continue; // torn/corrupt line — skip + if (parsed === undefined) { + if (!line.terminated) { + tailRepair = { kind: 'truncate', byteLength: line.startOffset }; + continue; + } + corrupt = true; + break; + } + if (!line.terminated) tailRepair = { kind: 'terminate' }; if (parsed.kind === 'journal_header') { - if (epoch === undefined) epoch = parsed.epoch; + epoch = parsed.epoch; // last header wins + segmentSeq = 0; + lastSeq = 0; continue; } - if (parsed.seq > lastSeq) lastSeq = parsed.seq; + if (epoch === undefined || parsed.seq !== segmentSeq + 1) { + corrupt = true; + break; + } + segmentSeq = parsed.seq; + lastSeq = segmentSeq; } } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code !== 'ENOENT') { + corrupt = true; logger.warn( { filePath, err: String(error) }, - 'event journal unreadable; starting a fresh epoch', + 'event journal unreadable; starting a fresh epoch on next append', ); } } + if (corrupt || (sawAnyLine && epoch === undefined)) { + await quarantineCorruptJournal(filePath, logger); + epoch = undefined; + lastSeq = 0; + tailRepair = undefined; + } + if (epoch === undefined) { if (sawAnyLine) { - // File exists but has no parseable header — treat as corrupt and - // start a fresh incarnation. Old cursors will epoch-mismatch. - logger.warn({ filePath }, 'event journal missing header; rotating to a fresh epoch'); + // File exists but has no parseable header — treat as corrupt and let + // the next append start a fresh incarnation. Old cursors will + // epoch-mismatch once the new header lands. + logger.warn({ filePath }, 'event journal missing header; rotating to a fresh epoch on next append'); } - return new SessionEventJournal(filePath, logger, `ep_${ulid()}`, 0, true); + return new SessionEventJournal(filePath, logger, undefined, 0); } - return new SessionEventJournal(filePath, logger, epoch, lastSeq, false); + return new SessionEventJournal(filePath, logger, epoch, lastSeq, tailRepair); } /** Reserve the next durable seq. The caller must follow with `append()`. */ nextSeq(): number { + this.throwIfSticky(); + if (this.currentEpoch === undefined) { + // First durable write of this incarnation: seq and epoch are born + // together, so every envelope can carry the epoch — including the very + // first one (the broadcaster stamps envelopes before `append()` runs). + // This is the ONLY place an epoch materializes: cold reads never call + // `nextSeq`, so they stay byte-free. The header latch alone writes + // nothing — `flushOnce` only runs when lines are pending (see `flush`). + this.currentEpoch = `ep_${ulid()}`; + this.headerPending = true; + } this._seq += 1; return this._seq; } /** Queue a durable event line for write-behind flush. */ append(seq: number, envelope: EventEnvelope): void { + this.throwIfSticky(); const line: JournalEventLine = { kind: 'event', seq, envelope }; this.pendingLines.push(JSON.stringify(line)); this.scheduleFlush(); @@ -152,11 +276,32 @@ export class SessionEventJournal { /** Read journal entries with `seq > fromSeqExclusive`, capped at `limit`. */ async readSince(fromSeqExclusive: number, limit: number): Promise { await this.flush(); + // Never answer a replay from a journal whose writes failed: a partial + // read would be a lie. Surface the sticky error so the edge can force a + // client-visible resync instead. + if (this.stickyError !== undefined) throw this.stickyError; const out: JournalEntry[] = []; try { - for await (const raw of readLines(this.filePath)) { + let activeEpoch: string | undefined; + let expectedSeq = 0; + for await (const line of readLines(this.filePath)) { + const raw = line.raw; + if (raw.trim().length === 0) continue; const parsed = parseJournalLine(raw); - if (parsed === undefined || parsed.kind !== 'event') continue; + if (parsed === undefined) { + if (!line.terminated) continue; + throw new JournalStorageError(this.filePath, new Error('corrupt journal line')); + } + if (parsed.kind === 'journal_header') { + activeEpoch = parsed.epoch; + expectedSeq = 0; + continue; + } + if (activeEpoch === undefined || parsed.seq !== expectedSeq + 1) { + throw new JournalStorageError(this.filePath, new Error('journal sequence gap')); + } + expectedSeq = parsed.seq; + if (activeEpoch !== this.currentEpoch) continue; if (parsed.seq <= fromSeqExclusive) continue; out.push({ seq: parsed.seq, envelope: parsed.envelope }); if (out.length >= limit) break; @@ -169,58 +314,152 @@ export class SessionEventJournal { } async flush(): Promise { + if (this.stickyError !== undefined) return; while (this.flushPromise !== undefined || this.pendingLines.length > 0) { if (this.flushPromise === undefined) { - this.flushPromise = this.flushOnce().finally(() => { + this.flushPromise = this.flushOnce().then(() => { this.flushPromise = undefined; }); } await this.flushPromise; + // Give up once sticky instead of hot-spinning on a persistently failing + // disk: appends now fail fast and `readSince` throws, so the kept + // pending lines are never retried. + if (this.stickyError !== undefined) return; } } + /** Flush whatever is pending; never throws (the read edge throws instead). */ async close(): Promise { await this.flush(); } private scheduleFlush(): void { if (this.flushPromise !== undefined) return; - this.flushPromise = this.flushOnce().finally(() => { + this.flushPromise = this.flushOnce().then((succeeded) => { this.flushPromise = undefined; // Appends that arrived while this flush was in flight are still pending: // chain the next round instead of parking them until a later append (or - // `close()`) happens to trigger one. - if (this.pendingLines.length > 0) this.scheduleFlush(); + // `close()`) happens to trigger one. A FAILED round must NOT chain — its + // kept lines are retried only by the next append-scheduled or + // read-triggered round (see `flush`), otherwise a persistently failing + // disk hot-spins open attempts forever, even past `close()`. + if (succeeded && this.pendingLines.length > 0) this.scheduleFlush(); }); } - private async flushOnce(): Promise { - // Snapshot the queue first so appends during the await are picked up by - // the next flush round, never lost. - const lines: string[] = []; - if (this.headerPending) { - const header: JournalHeaderLine = { - kind: 'journal_header', - version: JOURNAL_VERSION, - epoch: this.epoch, - created_at: Date.now(), - }; - lines.push(JSON.stringify(header)); - this.headerPending = false; - } - lines.push(...this.pendingLines); - this.pendingLines = []; - if (lines.length === 0) return; + private async flushOnce(): Promise { + // Snapshot the queue WITHOUT clearing it: lines are dequeued only after + // the batch is durably on disk (write + fsync succeeded). A failed round + // keeps every line — and a pending header — for the next retry. + const headerLine = this.headerPending ? this.buildHeaderLine() : undefined; + const pendingSnapshot = this.pendingLines.slice(); + if (headerLine === undefined && pendingSnapshot.length === 0) return true; + let lines = headerLine !== undefined ? [headerLine, ...pendingSnapshot] : pendingSnapshot; try { await mkdir(dirname(this.filePath), { recursive: true }); - await appendFile(this.filePath, lines.join('\n') + '\n', 'utf8'); + await this.repairTail(); + if (this.consecutiveFailures > 0) { + const committed = await countCommittedPrefix(this.filePath, lines); + if (committed > 0) { + this.dropCommittedLines(committed, headerLine); + lines = lines.slice(committed); + if (lines.length === 0) { + this.consecutiveFailures = 0; + return true; + } + } + } + // One open per batch: write header+lines, fsync, close. + const handle = await openFile(this.filePath, 'a'); + try { + await handle.writeFile(lines.join('\n') + '\n', 'utf8'); + await handle.sync(); + } finally { + await handle.close(); + } } catch (error) { - this.logger.warn( - { filePath: this.filePath, err: String(error) }, - 'event journal write failed; events remain live-only this round', - ); + const committed = await countCommittedPrefix(this.filePath, lines); + if (committed > 0) { + this.dropCommittedLines(committed, headerLine); + this.stickyError ??= new JournalStorageError(this.filePath, error); + return true; + } + this.consecutiveFailures += 1; + if (this.consecutiveFailures >= STICKY_FAILURE_THRESHOLD && this.stickyError === undefined) { + // Sticky storage failure: durable events must never silently degrade + // to live-only. kap-server has no telemetry wiring today, so the + // sticky transition is an error-level log breadcrumb (the design's + // `session.journal_write_failed` event lands with the wiring). + const logError = this.logger.error?.bind(this.logger) ?? this.logger.warn.bind(this.logger); + logError( + { + filePath: this.filePath, + err: String(error), + consecutiveFailures: this.consecutiveFailures, + }, + 'event journal storage failed persistently; entering sticky failure state — appends fail fast and readSince throws', + ); + this.stickyError = new JournalStorageError(this.filePath, error); + } else { + this.logger.warn( + { filePath: this.filePath, err: String(error) }, + 'event journal write failed; batch kept pending for retry', + ); + } + return false; + } + // Success: dequeue exactly the lines written above (appends during the + // await stay queued for the next round) and release the header latch. + this.pendingLines.splice(0, pendingSnapshot.length); + if (headerLine !== undefined) this.headerPending = false; + this.consecutiveFailures = 0; + return true; + } + + /** + * Dequeue the prefix already durably on disk (counted by + * `countCommittedPrefix`) and release the header latch — the header leads + * the batch, so any committed prefix includes it. Shared by the retry path + * (a previous round may have written a prefix before failing) and the catch + * path (the write may have landed even though the round threw). + */ + private dropCommittedLines(committed: number, headerLine: string | undefined): void { + if (headerLine !== undefined) this.headerPending = false; + this.pendingLines.splice(0, Math.max(0, committed - (headerLine === undefined ? 0 : 1))); + } + + private async repairTail(): Promise { + const repair = this.tailRepair; + if (repair === undefined) return; + const handle = await openFile(this.filePath, repair.kind === 'truncate' ? 'r+' : 'a'); + try { + if (repair.kind === 'truncate') { + await handle.truncate(repair.byteLength); + } else { + await handle.writeFile('\n', 'utf8'); + } + await handle.sync(); + this.tailRepair = undefined; + } finally { + await handle.close(); } } + + private buildHeaderLine(): string | undefined { + if (this.currentEpoch === undefined) return undefined; + const header: JournalHeaderLine = { + kind: 'journal_header', + version: JOURNAL_VERSION, + epoch: this.currentEpoch, + created_at: Date.now(), + }; + return JSON.stringify(header); + } + + private throwIfSticky(): void { + if (this.stickyError !== undefined) throw this.stickyError; + } } /** Default per-session journal path under `/.jsonl`. */ @@ -241,30 +480,90 @@ function parseJournalLine(raw: string): JournalHeaderLine | JournalEventLine | u const kind = (value as { kind?: unknown }).kind; if (kind === 'journal_header') { const epoch = (value as { epoch?: unknown }).epoch; - if (typeof epoch !== 'string' || epoch.length === 0) return undefined; + const version = (value as { version?: unknown }).version; + if (typeof epoch !== 'string' || epoch.length === 0 || version !== JOURNAL_VERSION) return undefined; return value as JournalHeaderLine; } if (kind === 'event') { const seq = (value as { seq?: unknown }).seq; const envelope = (value as { envelope?: unknown }).envelope; if (typeof seq !== 'number' || !Number.isInteger(seq) || seq <= 0) return undefined; - if (typeof envelope !== 'object' || envelope === null) return undefined; + if (!isEventEnvelope(envelope) || envelope.seq !== seq) return undefined; return value as JournalEventLine; } return undefined; } -async function* readLines(filePath: string): AsyncIterable { +function isEventEnvelope(value: unknown): value is EventEnvelope { + if (typeof value !== 'object' || value === null) return false; + const envelope = value as Partial; + return ( + typeof envelope.type === 'string' && + typeof envelope.seq === 'number' && + Number.isInteger(envelope.seq) && + envelope.seq > 0 && + typeof envelope.timestamp === 'string' && + Object.prototype.hasOwnProperty.call(envelope, 'payload') + ); +} + +interface RawJournalLine { + raw: string; + terminated: boolean; + startOffset: number; +} + +async function* readLines(filePath: string): AsyncIterable { let buffered = ''; + let offset = 0; const stream = createReadStream(filePath, { encoding: 'utf8' }); for await (const chunk of stream) { buffered += chunk; let newlineIndex = buffered.indexOf('\n'); while (newlineIndex !== -1) { - yield buffered.slice(0, newlineIndex); + const raw = buffered.slice(0, newlineIndex); + yield { raw, terminated: true, startOffset: offset }; + offset += Buffer.byteLength(raw, 'utf8') + 1; buffered = buffered.slice(newlineIndex + 1); newlineIndex = buffered.indexOf('\n'); } } - if (buffered.length > 0) yield buffered; + if (buffered.length > 0) yield { raw: buffered, terminated: false, startOffset: offset }; +} + +async function quarantineCorruptJournal(filePath: string, logger: JournalLogger): Promise { + try { + await rename(filePath, `${filePath}.corrupt.${ulid()}`); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + logger.warn({ filePath, err: String(error) }, 'event journal quarantine failed; truncating file'); + await truncate(filePath, 0).catch(() => undefined); + } + } +} + +async function countCommittedPrefix(filePath: string, lines: readonly string[]): Promise { + if (lines.length === 0) return 0; + let raw: string; + try { + raw = await readFile(filePath, 'utf8'); + } catch { + return 0; + } + const existing = raw.split('\n'); + if (existing.at(-1) === '') existing.pop(); + let committed = 0; + for (let count = 1; count <= lines.length; count++) { + const start = existing.length - count; + if (start < 0) break; + let matches = true; + for (let index = 0; index < count; index++) { + if (existing[start + index] !== lines[index]) { + matches = false; + break; + } + } + if (matches) committed = count; + } + return committed; } diff --git a/packages/kap-server/src/transport/ws/v1/sessionReleaseHook.ts b/packages/kap-server/src/transport/ws/v1/sessionReleaseHook.ts new file mode 100644 index 0000000000..69f99b7e7d --- /dev/null +++ b/packages/kap-server/src/transport/ws/v1/sessionReleaseHook.ts @@ -0,0 +1,26 @@ +/** + * Shared registration for the `onWillReleaseSession` lifecycle hook used by the + * v1 WS transport pieces ({@link SessionEventBroadcaster}, {@link FsWatchBridge}, + * {@link SkillCatalogBridge}): each drops its own per-session state for the + * released session, then lets the release chain proceed. + */ + +import { + type IDisposable, + ISessionLifecycleService, + type Scope, +} from '@moonshot-ai/agent-core-v2'; + +/** Register `release(sessionId)` to run (before `next()`) when a session scope is released. */ +export function registerSessionReleaseHook( + core: Scope, + name: string, + release: (sessionId: string) => void | Promise, +): IDisposable { + return core.accessor + .get(ISessionLifecycleService) + .hooks.onWillReleaseSession.register(name, async ({ sessionId }, next) => { + await release(sessionId); + await next(); + }); +} diff --git a/packages/kap-server/src/transport/ws/v1/skillCatalogBridge.ts b/packages/kap-server/src/transport/ws/v1/skillCatalogBridge.ts new file mode 100644 index 0000000000..ccc695e814 --- /dev/null +++ b/packages/kap-server/src/transport/ws/v1/skillCatalogBridge.ts @@ -0,0 +1,154 @@ +/** + * `SkillCatalogBridge` — volatile `/api/v1/ws` delivery for skill-catalog changes. + * + * Turns the core `ISessionSkillCatalog.onDidChange` feed (payload: the changed + * source id) into `skill_catalog.changed` frames on the v1 WebSocket: + * + * server → `{type:'skill_catalog.changed', seq, session_id, timestamp, volatile:true, payload}` + * + * Sibling of {@link FsWatchBridge}: the frame is a pure go-refetch hint + * (clients re-pull `GET /sessions/{sid}/skills`), so it is sent straight to the + * socket and never enters the broadcaster / journal — no durable `seq`, no + * replay after reconnect. Each connection numbers only the frames actually + * delivered to it (per-connection monotonic `seq`, starting at 1); unlike the + * fs channel, gaps carry no meaning because every hint triggers a full + * re-pull, and `volatile: true` keeps older clients from mistaking the + * connection-local `seq` for the durable watermark. + * + * Delivery set = connections subscribed to the session (the broadcaster's + * subscribe/unsubscribe lifecycle, driven by {@link WsConnectionV1}), NOT the + * fs-watch path sets: the bridge keeps one core subscription per session with + * at least one attached connection and releases it when the last detaches. + */ + +import { + type IDisposable, + ISessionSkillCatalog, + ISessionLifecycleService, + type Scope, +} from '@moonshot-ai/agent-core-v2'; +import type { SkillCatalogChangedEvent } from './events'; +import { registerSessionReleaseHook } from './sessionReleaseHook'; + +import type { EventEnvelope, JournalLogger } from './sessionEventJournal'; + +export interface SkillCatalogChangedFrame { + readonly type: 'skill_catalog.changed'; + readonly seq: number; + readonly session_id: string; + readonly timestamp: string; + readonly volatile: true; + readonly payload: SkillCatalogChangedEvent; +} + +/** Minimal connection surface the bridge needs (satisfied by `WsConnectionV1`). */ +export interface SkillCatalogConnection { + readonly id: string; + send(envelope: EventEnvelope): void; +} + +interface ConnEntry { + readonly conn: SkillCatalogConnection; + /** Per-connection monotonic frame counter; starts at 0, pre-incremented per delivery. */ + seq: number; +} + +interface SessionWatch { + readonly id: string; + readonly skillCatalog: ISessionSkillCatalog; + readonly conns: Map; + sub: IDisposable | undefined; +} + +export class SkillCatalogBridge implements IDisposable { + private readonly core: Scope; + private readonly logger: JournalLogger | undefined; + private readonly sessionReleaseHook: IDisposable; + private readonly bySession = new Map(); + + constructor(opts: { core: Scope; logger?: JournalLogger }) { + this.core = opts.core; + this.logger = opts.logger; + this.sessionReleaseHook = registerSessionReleaseHook( + this.core, + 'skillCatalogBridge', + (sessionId) => this.releaseSession(sessionId), + ); + } + + dispose(): void { + this.sessionReleaseHook.dispose(); + for (const sw of Array.from(this.bySession.values())) this.teardownSession(sw); + } + + /** Start delivering the session's catalog hints to `conn` (session subscribe). */ + attachSession(conn: SkillCatalogConnection, sessionId: string): void { + let sw = this.bySession.get(sessionId); + if (sw === undefined) { + const session = this.core.accessor.get(ISessionLifecycleService).get(sessionId); + if (session === undefined) return; + sw = { + id: sessionId, + skillCatalog: session.accessor.get(ISessionSkillCatalog), + conns: new Map(), + sub: undefined, + }; + this.bySession.set(sessionId, sw); + } + if (!sw.conns.has(conn.id)) { + sw.conns.set(conn.id, { conn, seq: 0 }); + } + sw.sub ??= sw.skillCatalog.onDidChange((sourceId) => { + this.onSessionEvent(sessionId, sourceId); + }); + } + + /** Stop delivering to `conn` (session unsubscribe); releases the core + * subscription when the session has no connections left. */ + detachSession(conn: SkillCatalogConnection, sessionId: string): void { + const sw = this.bySession.get(sessionId); + if (sw === undefined) return; + sw.conns.delete(conn.id); + if (sw.conns.size === 0) this.teardownSession(sw); + } + + /** Drop every subscription held by `conn` (called on socket close). */ + detachConnection(conn: SkillCatalogConnection): void { + for (const sw of Array.from(this.bySession.values())) { + if (!sw.conns.delete(conn.id)) continue; + if (sw.conns.size === 0) this.teardownSession(sw); + } + } + + private teardownSession(sw: SessionWatch): void { + sw.sub?.dispose(); + sw.sub = undefined; + this.bySession.delete(sw.id); + } + + private releaseSession(sessionId: string): void { + const sw = this.bySession.get(sessionId); + if (sw !== undefined) this.teardownSession(sw); + } + + private onSessionEvent(sessionId: string, sourceId: string): void { + const sw = this.bySession.get(sessionId); + if (sw === undefined) return; + for (const entry of sw.conns.values()) { + entry.seq += 1; + const frame: SkillCatalogChangedFrame = { + type: 'skill_catalog.changed', + seq: entry.seq, + session_id: sessionId, + timestamp: new Date().toISOString(), + volatile: true, + payload: { type: 'skill_catalog.changed', sourceId }, + }; + try { + entry.conn.send(frame as EventEnvelope); + } catch (error) { + this.logger?.warn({ sessionId, err: String(error) }, 'skill-catalog send failed'); + } + } + } +} diff --git a/packages/kap-server/src/transport/ws/v1/wsConnectionV1.ts b/packages/kap-server/src/transport/ws/v1/wsConnectionV1.ts index 1536873993..e0cd5c538e 100644 --- a/packages/kap-server/src/transport/ws/v1/wsConnectionV1.ts +++ b/packages/kap-server/src/transport/ws/v1/wsConnectionV1.ts @@ -15,6 +15,7 @@ * down. */ +import { ErrorCode } from '../../../protocol/error-codes'; import { WS_PROTOCOL_VERSION, type SessionCursor } from '../../../protocol/ws-control'; import { transcriptSubscriptionSchema, type TranscriptGradeSpec } from '@moonshot-ai/transcript'; import { ulid } from 'ulid'; @@ -24,6 +25,7 @@ import type { CredentialValidator } from '../../../services/auth/credentials'; import type { IConnectionRegistry } from '../connectionRegistry'; import { type EventEnvelope, + JournalStorageError, type JournalLogger, } from './sessionEventJournal'; import { @@ -38,7 +40,9 @@ import { type SessionEventBroadcaster, type TargetSubscription, } from './sessionEventBroadcaster'; +import type { SessionOwnershipDetails } from '@moonshot-ai/agent-core-v2'; import { FsWatchBridge } from './fsWatchBridge'; +import { SkillCatalogBridge } from './skillCatalogBridge'; const DEFAULT_MAX_BUFFER_SIZE = 1000; @@ -64,6 +68,7 @@ export interface WsConnectionV1Options { readonly socket: WebSocket; readonly broadcaster: SessionEventBroadcaster; readonly fsWatchBridge?: FsWatchBridge; + readonly skillCatalogBridge?: SkillCatalogBridge; readonly connectionRegistry: IConnectionRegistry; /** * Present-only credential check for the post-connect `client_hello` @@ -94,6 +99,7 @@ export class WsConnectionV1 implements BroadcastTarget { private readonly socket: WebSocket; private readonly broadcaster: SessionEventBroadcaster; private readonly fsWatchBridge?: FsWatchBridge; + private readonly skillCatalogBridge?: SkillCatalogBridge; private readonly validateCredential?: CredentialValidator; private readonly maxBufferSize: number; private readonly flushIntervalMs: number; @@ -121,6 +127,7 @@ export class WsConnectionV1 implements BroadcastTarget { this.socket = opts.socket; this.broadcaster = opts.broadcaster; this.fsWatchBridge = opts.fsWatchBridge; + this.skillCatalogBridge = opts.skillCatalogBridge; this.validateCredential = opts.validateCredential; this.logger = opts.logger; this.maxBufferSize = opts.maxBufferSize ?? DEFAULT_MAX_BUFFER_SIZE; @@ -200,7 +207,9 @@ export class WsConnectionV1 implements BroadcastTarget { const transcript = parseTranscriptSubscription(payload['transcript']); const accepted: string[] = []; + const notFound: string[] = []; const resyncRequired: string[] = []; + const ownershipDetails: Record = {}; const serverCursors: Record = {}; for (const sid of subscriptions) { @@ -210,17 +219,21 @@ export class WsConnectionV1 implements BroadcastTarget { agentFilter?.[sid], transcript?.[sid], accepted, + notFound, resyncRequired, + ownershipDetails, serverCursors, ); } - this.sendFrame( - buildAck(frame.id ?? '', 0, 'success', { - accepted_subscriptions: accepted, - resync_required: resyncRequired, - cursors: serverCursors, - }), + this.sendSubscribeAck( + frame.id, + 'accepted_subscriptions', + accepted, + notFound, + resyncRequired, + ownershipDetails, + serverCursors, ); } @@ -234,6 +247,7 @@ export class WsConnectionV1 implements BroadcastTarget { const accepted: string[] = []; const notFound: string[] = []; const resyncRequired: string[] = []; + const ownershipDetails: Record = {}; const serverCursors: Record = {}; for (const sid of sessionIds) { @@ -246,10 +260,11 @@ export class WsConnectionV1 implements BroadcastTarget { deferTranscriptReset: cursor !== undefined, }); if (!ok) { - notFound.push(sid); + await this.classifySubscriptionFailure(sid, ownershipDetails, notFound); continue; } this.subscriptions.set(sid, { agentFilter: filter, transcriptGrades: grades }); + this.skillCatalogBridge?.attachSession(this, sid); accepted.push(sid); if (cursor !== undefined) { await this.replay(sid, cursor, filter, resyncRequired, serverCursors); @@ -260,13 +275,14 @@ export class WsConnectionV1 implements BroadcastTarget { } } - this.sendFrame( - buildAck(frame.id ?? '', 0, 'success', { - accepted, - not_found: notFound, - resync_required: resyncRequired, - cursors: serverCursors, - }), + this.sendSubscribeAck( + frame.id, + 'accepted', + accepted, + notFound, + resyncRequired, + ownershipDetails, + serverCursors, ); } @@ -276,6 +292,7 @@ export class WsConnectionV1 implements BroadcastTarget { for (const sid of sessionIds) { this.broadcaster.unsubscribe(sid, this); this.subscriptions.delete(sid); + this.skillCatalogBridge?.detachSession(this, sid); } this.sendFrame( buildAck(frame.id ?? '', 0, 'success', { @@ -322,7 +339,9 @@ export class WsConnectionV1 implements BroadcastTarget { filter: AgentFilter | undefined, transcriptGrades: TranscriptGradeSpec | undefined, accepted: string[], + notFound: string[], resyncRequired: string[], + ownershipDetails: Record, serverCursors: Record, ): Promise { // Same ordering rule as onSubscribe: baseline after the cursor replay. @@ -330,10 +349,11 @@ export class WsConnectionV1 implements BroadcastTarget { deferTranscriptReset: cursor !== undefined, }); if (!ok) { - resyncRequired.push(sid); + await this.classifySubscriptionFailure(sid, ownershipDetails, notFound); return; } this.subscriptions.set(sid, { agentFilter: filter, transcriptGrades }); + this.skillCatalogBridge?.attachSession(this, sid); accepted.push(sid); if (cursor !== undefined) { await this.replay(sid, cursor, filter, resyncRequired, serverCursors); @@ -344,6 +364,53 @@ export class WsConnectionV1 implements BroadcastTarget { } } + /** + * Classify a failed subscribe: an ownership conflict carries the structured + * details so the client can redirect to the holder; anything else is a plain + * not-found. + */ + private async classifySubscriptionFailure( + sid: string, + ownershipDetails: Record, + notFound: string[], + ): Promise { + const ownership = await this.broadcaster.getSubscriptionFailure(sid); + if (ownership !== undefined) ownershipDetails[sid] = ownership; + else notFound.push(sid); + } + + /** + * Ack for `client_hello` / `subscribe`: an ownership failure on any session + * flips the whole ack to SESSION_HELD_BY_PEER (the per-session details ride + * `ownership_details`). The accepted-list key differs by entry point — + * `accepted_subscriptions` for `client_hello`, `accepted` for `subscribe`. + */ + private sendSubscribeAck( + frameId: string | undefined, + acceptedKey: 'accepted' | 'accepted_subscriptions', + accepted: string[], + notFound: string[], + resyncRequired: string[], + ownershipDetails: Record, + serverCursors: Record, + ): void { + const hasOwnershipFailure = Object.keys(ownershipDetails).length > 0; + this.sendFrame( + buildAck( + frameId ?? '', + hasOwnershipFailure ? ErrorCode.SESSION_HELD_BY_PEER : 0, + hasOwnershipFailure ? 'session held by peer' : 'success', + { + [acceptedKey]: accepted, + not_found: notFound, + resync_required: resyncRequired, + ownership_details: ownershipDetails, + cursors: serverCursors, + }, + ), + ); + } + private async replay( sid: string, cursor: SessionCursor, @@ -351,7 +418,24 @@ export class WsConnectionV1 implements BroadcastTarget { resyncRequired: string[], serverCursors: Record, ): Promise { - const result = await this.broadcaster.getBufferedSince(sid, cursor, filter); + let result; + try { + result = await this.broadcaster.getBufferedSince(sid, cursor, filter); + } catch (error) { + if (error instanceof JournalStorageError) { + // The journal cannot serve the gap (sticky storage failure): answering + // with an empty page would be a lie — "not served" must stay + // distinguishable from "nothing to serve". Force a resync so the + // client rebuilds from the snapshot and re-subscribes. The protocol + // reason enum has no storage-failure entry, so this rides the + // existing `epoch_changed` branch; the client behavior (rebuild from + // snapshot) is identical for every reason. + this.sendFrame(buildResyncRequired(sid, 'epoch_changed', cursor.seq, cursor.epoch)); + resyncRequired.push(sid); + return; + } + throw error; + } if (result.resyncRequired !== false) { this.sendFrame( buildResyncRequired(sid, result.resyncRequired as ResyncReason, result.currentSeq, result.epoch), @@ -480,6 +564,7 @@ export class WsConnectionV1 implements BroadcastTarget { this.outbound = []; for (const sid of this.subscriptions.keys()) this.broadcaster.unsubscribe(sid, this); this.fsWatchBridge?.detachConnection(this); + this.skillCatalogBridge?.detachConnection(this); // registry removal is handled by registerWsV1 on the socket 'close' event. } } diff --git a/packages/kap-server/test/boot.test.ts b/packages/kap-server/test/boot.test.ts index dba60db6fc..f64e688e95 100644 --- a/packages/kap-server/test/boot.test.ts +++ b/packages/kap-server/test/boot.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { createServer, type Server } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -6,9 +6,18 @@ import { join } from 'node:path'; import { pino } from 'pino'; import { afterEach, describe, expect, it } from 'vitest'; -import { hostRequestHeadersSeed, IHostRequestHeaders, ISkillCatalogRuntimeOptions } from '@moonshot-ai/agent-core-v2'; +import { + type DomainEvent, + hostRequestHeadersSeed, + IAgentLifecycleService, + IEventBus, + IHostRequestHeaders, + ISessionLifecycleService, + ISkillCatalogRuntimeOptions, +} from '@moonshot-ai/agent-core-v2'; import { listLiveServerInstances } from '../src/instanceRegistry'; +import { formatServerOrigin } from '../src/serverOrigin'; import { listenWithPortRetry, type RunningServer, startServer } from '../src/start'; import { getServerVersion } from '../src/version'; import { authedFetch } from './helpers/auth'; @@ -158,6 +167,104 @@ describe('server-v2 boot', () => { }); expect(server.core.accessor.get(ISkillCatalogRuntimeOptions).explicitDirs).toBeUndefined(); }); + + it('close() is idempotent under concurrent triggers and safe to re-invoke', async () => { + home = await mkdtemp(join(tmpdir(), 'kimi-server-v2-close-')); + server = await startServer({ + host: '127.0.0.1', + port: 0, + homeDir: home, + logLevel: 'silent', + }); + + // The remote /api/v1/shutdown path and the CLI signal handler converge on + // this same close(): double triggers must share ONE in-flight close + // rather than racing fastify.close / core.dispose re-entrantly. + const first = server.close(); + const second = server.close(); + expect(second).toBe(first); + await expect(first).resolves.toBeUndefined(); + + // A late third invocation resolves replays the completed close instead of + // re-running it (fastify would reject a second app.close()). + await expect(server.close()).resolves.toBeUndefined(); + server = undefined; + }); + + it('does not dispose the server when lifecycle closeAll fails', async () => { + home = await mkdtemp(join(tmpdir(), 'kimi-server-v2-close-failure-')); + server = await startServer({ + host: '127.0.0.1', + port: 0, + homeDir: home, + logLevel: 'silent', + }); + const lifecycle = server.core.accessor.get(ISessionLifecycleService); + const originalCloseAll = lifecycle.closeAll.bind(lifecycle); + const failure = new Error('session shutdown failed'); + lifecycle.closeAll = async () => { + throw failure; + }; + + await expect(server.close()).rejects.toBe(failure); + + lifecycle.closeAll = originalCloseAll; + await server.close(); + server = undefined; + }); + + it('flushes the dispatch tail into the session journal before close() resolves', async () => { + home = await mkdtemp(join(tmpdir(), 'kimi-server-v2-close-tail-')); + server = await startServer({ + host: '127.0.0.1', + port: 0, + homeDir: home, + logLevel: 'silent', + }); + const base = `http://127.0.0.1:${server.port}`; + + // Create a session and arm its journal (the snapshot route activates the + // broadcaster state for the session without a WS connection). + const created = await authedFetch(server, base, '/api/v1/sessions', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ metadata: { cwd: home } }), + }); + const createdBody = (await created.json()) as { code: number; data: { id: string } }; + expect(createdBody.code).toBe(0); + const sid = createdBody.data.id; + + const session = server.core.accessor.get(ISessionLifecycleService).get(sid); + expect(session).toBeDefined(); + const agents = session!.accessor.get(IAgentLifecycleService); + if (agents.get('main') === undefined) { + await agents.create({ agentId: 'main' }); + } + const snapshot = await authedFetch(server, base, `/api/v1/sessions/${sid}/snapshot`); + expect(snapshot.status).toBe(200); + + // Emit a durable event and shut down IMMEDIATELY — the dispatch is still + // sitting in the broadcaster queue when close() starts. It must be + // journaled (with fsync) before close() resolves: sessions close first, + // the journal flush comes last. + agents + .get('main')! + .accessor.get(IEventBus) + .publish({ type: 'turn.started', turnId: 1 } as unknown as DomainEvent); + await server.close(); + server = undefined; + + const journal = await readFile(join(home, 'server', 'events', `${sid}.jsonl`), 'utf8'); + expect(journal).toContain('journal_header'); + expect(journal).toContain('turn.started'); + }); +}); + +describe('server origin formatting', () => { + it('bracket-wraps IPv6 hosts and keeps IPv4 hosts unchanged', () => { + expect(formatServerOrigin('::1', 58627)).toBe('http://[::1]:58627'); + expect(formatServerOrigin('127.0.0.1', 58627)).toBe('http://127.0.0.1:58627'); + }); }); function silentLogger() { diff --git a/packages/kap-server/test/fs-watch.e2e.test.ts b/packages/kap-server/test/fs-watch.e2e.test.ts index e674e625e3..0a8350e45b 100644 --- a/packages/kap-server/test/fs-watch.e2e.test.ts +++ b/packages/kap-server/test/fs-watch.e2e.test.ts @@ -4,11 +4,16 @@ * Mirrors `packages/server/test/fs-watch.e2e.test.ts` (v1) so the wire contract * stays byte-compatible: * 1. subscribe `src` → create file → receive `event.fs.changed` - * 2. burst > 500 changes / 200ms → `truncated` event - * 3. two clients, disjoint paths → no cross-delivery + * 2. burst > 500 changes / 200ms → `truncated` event reaches every + * connection with its own next seq; the client then rebuilds its + * baseline via REST `fs:list` + * 3. two clients, disjoint paths → no cross-delivery; each connection + * numbers only the frames it actually receives (per-connection seq) * 4. > 100 paths per connection → `42902 fs.watch_limit_exceeded` * 5. idempotent add of the same path * 6. `watch_fs_remove` updates `watched_paths`; `..` → `41304` + * 7. two clients on the same path see the same event with the same + * per-connection seq, numbered from 1 without gaps * * Boots `startServer` in-process (loopback, auth disabled) against a tmp * workspace, drives `/api/v1/ws` clients with the raw `ws` library, and mutates @@ -83,6 +88,18 @@ async function createSession(r: RunningServer): Promise { return env.data.id; } +async function runSessionAction( + r: RunningServer, + sessionId: string, + action: 'archive' | 'restore', +): Promise { + const res = await fetch(`${addressOf(r)}/api/v1/sessions/${sessionId}:${action}`, { + method: 'POST', + }); + const env = (await res.json()) as { code: number }; + if (env.code !== 0) throw new Error(`${action} session failed: ${JSON.stringify(env)}`); +} + interface WsFrame { type: string; payload?: Record; @@ -224,6 +241,8 @@ describe('WS fs watch (kap-server)', () => { const sid = await createSession(r); const conn = await openConn(wsUrl(r)); await helloAndSubscribe(conn, 'A', sid); + const bystander = await openConn(wsUrl(r)); + await helloAndSubscribe(bystander, 'B', sid); conn.ws.send( JSON.stringify({ @@ -233,31 +252,66 @@ describe('WS fs watch (kap-server)', () => { }), ); await receiveType(conn, 'ack', 1000); + bystander.ws.send( + JSON.stringify({ + type: 'watch_fs_add', + id: 'wB', + payload: { session_id: sid, paths: ['docs'] }, + }), + ); + await receiveType(bystander, 'ack', 1000); await sleep(WATCH_SETTLE_MS); const burstDir = join(workspace, 'burst'); mkdirSync(burstDir, { recursive: true }); for (let i = 0; i < 600; i++) writeFileSync(join(burstDir, `f${i}.txt`), `x${i}`); - const deadline = Date.now() + 12000; - let sawTruncated = false; - while (Date.now() < deadline) { - let frame: WsFrame; - try { - frame = await receive(conn, deadline - Date.now()); - } catch { - break; + const waitForTruncated = async (c: Conn): Promise => { + const deadline = Date.now() + 12000; + while (Date.now() < deadline) { + let frame: WsFrame; + try { + frame = await receive(c, deadline - Date.now()); + } catch { + break; + } + if (frame.type !== 'event.fs.changed') continue; + const payload = frame.payload as { truncated?: boolean; count?: number }; + if (payload.truncated === true) { + expect(payload.count).toBeGreaterThan(100); + return frame; + } } - if (frame.type !== 'event.fs.changed') continue; - const payload = frame.payload as { truncated?: boolean; count?: number }; - if (payload.truncated === true) { - expect(payload.count).toBeGreaterThan(100); - sawTruncated = true; - break; - } - } - expect(sawTruncated).toBe(true); + throw new Error('no truncated frame received'); + }; + + const [truncatedA, truncatedB] = await Promise.all([ + waitForTruncated(conn), + waitForTruncated(bystander), + ]); + // A truncated window is broadcast to every connection of the session + // watch — even one whose paths match nothing — and each frame carries + // that connection's own next seq. The bystander received no matched + // frame before, so its truncated frame is its first (seq 1). + expect(typeof truncatedA.seq).toBe('number'); + expect(truncatedB.seq).toBe(1); + + // Client-side recovery path after `truncated`: rebuild the baseline + // with a full REST listing (snapshot + incremental + resync). + const listRes = await fetch(`${addressOf(r)}/api/v1/sessions/${sid}/fs:list`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ path: '.', depth: 1, limit: 100 }), + }); + const listEnv = (await listRes.json()) as { + code: number; + data: { items: Array<{ path: string }> } | null; + }; + expect(listEnv.code).toBe(0); + expect(listEnv.data?.items.map((i) => i.path)).toContain('burst'); + conn.ws.close(); + bystander.ws.close(); }, ); @@ -292,6 +346,21 @@ describe('WS fs watch (kap-server)', () => { expect(pathsB.some((p) => p.startsWith('docs/'))).toBe(true); expect(pathsB.some((p) => p.startsWith('src/'))).toBe(false); + // Per-connection monotonic seq: each connection numbers only the frames + // actually delivered to it, so the same logical window arrives as each + // connection's own seq 1 — not a shared per-session counter. + expect(evA.seq).toBe(1); + expect(evB.seq).toBe(1); + + writeFileSync(join(workspace, 'src', 'a2.ts'), 'a2'); + const evA2 = await receiveType(a, 'event.fs.changed', 2000); + expect(evA2.seq).toBe(2); + + writeFileSync(join(workspace, 'docs', 'b2.md'), 'b2'); + const evB2 = await receiveType(b, 'event.fs.changed', 2000); + // The src-only window never reached B, so it consumed none of B's seq. + expect(evB2.seq).toBe(2); + a.ws.close(); b.ws.close(); }); @@ -382,6 +451,38 @@ describe('WS fs watch (kap-server)', () => { conn.ws.close(); }); + it('rebinds file watches after an archived session is restored on the same connection', async () => { + const r = await boot(); + const sid = await createSession(r); + const conn = await openConn(wsUrl(r)); + await helloAndSubscribe(conn, 'A', sid); + + conn.ws.send( + JSON.stringify({ type: 'watch_fs_add', id: 'before', payload: { session_id: sid, paths: ['src'] } }), + ); + await receiveType(conn, 'ack', 1000); + await runSessionAction(r, sid, 'archive'); + await runSessionAction(r, sid, 'restore'); + + conn.ws.send( + JSON.stringify({ type: 'watch_fs_add', id: 'after', payload: { session_id: sid, paths: ['src'] } }), + ); + const ack = await receiveType(conn, 'ack', 1000); + expect(ack.code).toBe(0); + expect(ack.payload).toMatchObject({ watched_paths: ['src'], current_count: 1 }); + + await sleep(WATCH_SETTLE_MS); + writeFileSync(join(workspace, 'src', 'restored.ts'), 'export const restored = true;\n'); + + const ev = await receiveType(conn, 'event.fs.changed', 2000); + expect(ev.session_id).toBe(sid); + expect((ev.payload as { changes: Array<{ path: string }> }).changes).toEqual( + expect.arrayContaining([expect.objectContaining({ path: 'src/restored.ts' })]), + ); + + conn.ws.close(); + }); + it('watch_fs_add for `..` path → 41304 fs.path_escapes_session', async () => { const r = await boot(); const sid = await createSession(r); diff --git a/packages/kap-server/test/helpers/sessionEvents.ts b/packages/kap-server/test/helpers/sessionEvents.ts new file mode 100644 index 0000000000..7e88507a9a --- /dev/null +++ b/packages/kap-server/test/helpers/sessionEvents.ts @@ -0,0 +1,26 @@ +/** + * Shared fakes for the WS v1 session-event tests (`sessionEventBroadcaster`, + * `sessionEventJournal`, `skillCatalogBridge`). + */ +import { type SessionLifecycleHooks } from '@moonshot-ai/agent-core-v2'; +import { createHooks } from '@moonshot-ai/agent-core-v2/hooks'; + +/** The lifecycle hook set the fake cores expose at `ISessionLifecycleService.hooks`. */ +export function createSessionLifecycleHooks() { + return createHooks([ + 'onDidCreateSession', + 'onWillCloseSession', + 'onWillReleaseSession', + ]); +} + +/** + * Build the write failure the `node:fs/promises` mocks inject into journal + * flushes: a plain `EACCES` errno error — the ordinary storage failure the + * journal's retry-then-sticky durability tests drive. + */ +export function injectWriteFailure(): Error { + const error = new Error('injected journal write failure') as NodeJS.ErrnoException; + error.code = 'EACCES'; + return error; +} diff --git a/packages/kap-server/test/sessionEventBroadcaster.test.ts b/packages/kap-server/test/sessionEventBroadcaster.test.ts index fa9551e60b..a6a6520f4b 100644 --- a/packages/kap-server/test/sessionEventBroadcaster.test.ts +++ b/packages/kap-server/test/sessionEventBroadcaster.test.ts @@ -2,11 +2,10 @@ * `SessionEventBroadcaster` — seq stamping, volatile vs durable, fan-out, replay. */ -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdtemp, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { IScopeHandle, Scope } from '@moonshot-ai/agent-core-v2'; import { ContextSizeModel, IAgentActivityView, @@ -21,6 +20,9 @@ import { IWireService, ISessionMetadata, SessionInteractionService, + type IScopeHandle, + type SessionLifecycleHooks, + type Scope, } from '@moonshot-ai/agent-core-v2'; import type { AgentEvent } from '../src/transport/ws/v1/events'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; @@ -29,8 +31,50 @@ import { type BroadcastTarget, SessionEventBroadcaster, } from '../src/transport/ws/v1/sessionEventBroadcaster'; -import type { EventEnvelope } from '../src/transport/ws/v1/sessionEventJournal'; +import { + type EventEnvelope, + JournalStorageError, +} from '../src/transport/ws/v1/sessionEventJournal'; import { TranscriptService } from '../src/services/transcript/transcriptService'; +import { createSessionLifecycleHooks } from './helpers/sessionEvents'; + +/** + * Controllable deferred-failure injection for the journal under test. The + * journal opens its file via `fs/promises.open(path, 'a')` per flush batch; + * while `active` is set, opens of journal paths in this file's tmp dirs are + * parked until the test rejects them (precise per-round failure control). + * Everything else passes through to the real module. + */ +const journalFs = vi.hoisted(() => ({ + active: false, + rejecters: [] as Array<(error: Error) => void>, +})); + +vi.mock('node:fs/promises', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + open: (path: Parameters[0], flags?: string, mode?: number) => { + if ( + journalFs.active && + typeof path === 'string' && + path.includes('kimi-broadcaster-test-') && + path.endsWith('.jsonl') + ) { + return new Promise((_resolve, reject) => { + journalFs.rejecters.push(reject); + }); + } + return actual.open(path, flags, mode); + }, + }; +}); + +function injectWriteFailure(): Error { + const error = new Error('injected journal write failure') as NodeJS.ErrnoException; + error.code = 'EACCES'; + return error; +} // --------------------------------------------------------------------------- // Fakes @@ -173,6 +217,7 @@ function makeCore( sessions: Map, eventBus = new FakeEventBus(), metaAgents: Record = {}, + lifecycleHooks = createSessionLifecycleHooks(), ): Scope { const accessor = { get(token: unknown): unknown { @@ -182,6 +227,7 @@ function makeCore( // Inert lifecycle events (TranscriptService subscribes on construction). onDidCloseSession: () => ({ dispose: () => {} }), onDidArchiveSession: () => ({ dispose: () => {} }), + hooks: lifecycleHooks, get: (sid: string) => { const lifecycle = sessions.get(sid); if (lifecycle === undefined) return undefined; @@ -228,20 +274,27 @@ describe('SessionEventBroadcaster', () => { let dir: string; let sessions: Map; let eventBus: FakeEventBus; + let lifecycleHooks: ReturnType; let bc: SessionEventBroadcaster; beforeEach(async () => { + journalFs.active = false; + journalFs.rejecters = []; dir = await mkdtemp(join(tmpdir(), 'kimi-broadcaster-test-')); sessions = new Map(); eventBus = new FakeEventBus(); + lifecycleHooks = createSessionLifecycleHooks(); bc = new SessionEventBroadcaster({ eventsDir: dir, - core: makeCore(sessions, eventBus), + homeDir: dir, + core: makeCore(sessions, eventBus, {}, lifecycleHooks), maxBufferSize: 3, }); }); afterEach(async () => { + journalFs.active = false; + journalFs.rejecters = []; await bc.close(); await rm(dir, { recursive: true, force: true }); }); @@ -275,10 +328,46 @@ describe('SessionEventBroadcaster', () => { type: 'event.session.work_changed', payload: { busy: false, last_turn_reason: 'completed' }, }); - expect(envelopes.every((e) => e.epoch === envelopes[0]!.epoch)).toBe(true); + // The epoch is born lazily with the first durable append (`nextSeq`), so + // the pre-baseline volatile phase frame carries none. Every envelope that + // HAS an epoch shares the journal's single incarnation — no mid-stream + // epoch flip-flop. + expect(durable[0]!.epoch).toMatch(/^ep_/); + expect(envelopes.every((e) => e.epoch === undefined || e.epoch === durable[0]!.epoch)).toBe( + true, + ); expect(durable[0]!.volatile).toBeUndefined(); }); + it('rebuilds subscriptions and the journal writer after a session is released', async () => { + const firstLifecycle = new FakeLifecycle(); + const firstMain = firstLifecycle.addAgent('main'); + sessions.set('s1', firstLifecycle); + const { target, envelopes } = collectingTarget(); + await bc.subscribe('s1', target); + firstMain.bus.emit(agentEvent('turn.started', { turnId: 1 })); + const firstSeq = (await bc.getCursor('s1')).seq; + expect(firstSeq).toBe(2); + + await lifecycleHooks.onWillReleaseSession.run({ sessionId: 's1', reason: 'archive' }); + firstMain.bus.emit(agentEvent('turn.ended', { turnId: 1, reason: 'completed' })); + + const restoredLifecycle = new FakeLifecycle(); + const restoredMain = restoredLifecycle.addAgent('main'); + sessions.set('s1', restoredLifecycle); + expect(await bc.subscribe('s1', target)).toBe(true); + restoredMain.bus.emit(agentEvent('turn.started', { turnId: 2 })); + await bc.getCursor('s1'); + + const durable = envelopes.filter((event) => event.volatile !== true); + expect(durable.some((event) => event.type === 'turn.ended')).toBe(false); + expect(durable.at(-1)).toMatchObject({ + type: 'turn.started', + seq: firstSeq + 2, + payload: { turnId: 2, agentId: 'main', sessionId: 's1' }, + }); + }); + it('fans out volatile events with the current watermark + offset, not journaled', async () => { const lc = new FakeLifecycle(); const main = lc.addAgent('main'); @@ -576,6 +665,88 @@ describe('SessionEventBroadcaster', () => { expect(next.subagents).toEqual([]); }); + it.each([ + { + event: { + type: 'event.model_catalog.changed', + payload: { + changed: [ + { provider_id: 'managed:kimi-code', provider_name: 'Kimi Code', added: 1, removed: 0 }, + ], + unchanged: [], + failed: [], + }, + }, + }, + // The multi-instance sessions-tree watcher publishes session.list_changed + // when a workspace/session directory appears or disappears (possibly + // created by a peer instance) — same delivery class as model-catalog. + { event: { type: 'session.list_changed', payload: {} } }, + ])('fans $event.type out live as a volatile __global__ hint without journaling', async ({ event }) => { + const lc = new FakeLifecycle(); + lc.addAgent('main'); + sessions.set('s1', lc); + const { target, envelopes } = collectingTarget(); + await bc.subscribe('s1', target); + + eventBus.emit(event); + + await vi.waitFor(() => expect(envelopes).toHaveLength(1)); + expect(envelopes[0]).toMatchObject({ + type: event.type, + session_id: '__global__', + volatile: true, + payload: { type: event.type, agentId: 'main', sessionId: '__global__' }, + }); + // Advisory-only: fanned out live with a process-local seq and never + // journaled — no __global__.jsonl garbage file under the events dir. + await expect(stat(join(dir, '__global__.jsonl'))).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('sends one global hint when a target subscribes to multiple sessions', async () => { + sessions.set('s1', new FakeLifecycle()); + sessions.set('s2', new FakeLifecycle()); + const { target, envelopes } = collectingTarget(); + await bc.subscribe('s1', target); + await bc.subscribe('s2', target); + + eventBus.emit({ type: 'session.list_changed', payload: {} }); + + await vi.waitFor(() => expect(envelopes).toHaveLength(1)); + expect(envelopes[0]).toMatchObject({ type: 'session.list_changed', session_id: '__global__' }); + }); + + it('never serves replay from the in-memory tail while the journal has write failures', async () => { + const lc = new FakeLifecycle(); + const main = lc.addAgent('main'); + sessions.set('s1', lc); + const { target } = collectingTarget(); + await bc.subscribe('s1', target); + + main.bus.emit(agentEvent('turn.started', { turnId: 1 })); + // Force batch 1 durable on disk (replay flushes first): the durable + // status_changed + turn.started. + const warm = await bc.getBufferedSince('s1', { seq: 0 }); + expect(warm.events).toHaveLength(2); + + // Storage starts failing: the next flush round parks on a deferred open. + journalFs.active = true; + main.bus.emit(agentEvent('turn.ended', { turnId: 1 })); + await bc.getCursor('s1'); // drain the queue — the append is queued to the journal + await vi.waitFor(() => expect(journalFs.rejecters).toHaveLength(1)); + journalFs.rejecters.shift()!(injectWriteFailure()); + await new Promise((resolve) => setImmediate(resolve)); // settle the failure state + + // The memory tail still holds the undurable turn.ended, but replay must + // not be served from it: the journal-side retry fails again → sticky → + // the explicit storage error surfaces to the replay edge. (Cursor seq 1 + // keeps the gap inside the maxBufferSize=3 cap.) + const replay = bc.getBufferedSince('s1', { seq: 1 }); + await vi.waitFor(() => expect(journalFs.rejecters).toHaveLength(1)); + journalFs.rejecters.shift()!(injectWriteFailure()); + await expect(replay).rejects.toBeInstanceOf(JournalStorageError); + }); + it('subscribe returns false for an unknown session', async () => { const { target } = collectingTarget(); expect(await bc.subscribe('nope', target)).toBe(false); @@ -1277,6 +1448,7 @@ describe('SessionEventBroadcaster', () => { const dir2 = await mkdtemp(join(tmpdir(), 'kimi-broadcaster-test-')); const bc2 = new SessionEventBroadcaster({ eventsDir: dir2, + homeDir: dir2, core: makeCore(sessions, eventBus), maxBufferSize: 20, }); @@ -1355,6 +1527,7 @@ describe('SessionEventBroadcaster', () => { const core = makeCore(sessions, eventBus, metaAgents); return new SessionEventBroadcaster({ eventsDir: dir, + homeDir: dir, core, maxBufferSize: 3, transcriptService: new TranscriptService({ homeDir: dir, core }), @@ -1612,6 +1785,7 @@ describe('SessionEventBroadcaster', () => { }); bc = new SessionEventBroadcaster({ eventsDir: dir, + homeDir: dir, core, maxBufferSize: 3, transcriptService: service, @@ -1638,6 +1812,7 @@ describe('SessionEventBroadcaster', () => { const service = new TranscriptService({ homeDir: dir, core }); bc = new SessionEventBroadcaster({ eventsDir: dir, + homeDir: dir, core, maxBufferSize: 3, transcriptService: service, diff --git a/packages/kap-server/test/sessionEventJournal.test.ts b/packages/kap-server/test/sessionEventJournal.test.ts index 33a62e099d..94378c2de5 100644 --- a/packages/kap-server/test/sessionEventJournal.test.ts +++ b/packages/kap-server/test/sessionEventJournal.test.ts @@ -1,18 +1,47 @@ /** - * `SessionEventJournal` — seq assignment, durability, recovery, epoch rotation. + * `SessionEventJournal` — seq assignment, durability, recovery, epoch rotation, + * write-failure semantics. */ -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type EventEnvelope, + JournalStorageError, SessionEventJournal, } from '../src/transport/ws/v1/sessionEventJournal'; +/** + * Controllable write-failure injection. The journal opens its journal file via + * `fs/promises.open(path, 'a')` per flush batch (single open → write → fsync → + * close), so failing `open` fails the whole batch. All other calls (including + * this test's own mkdtemp/readFile/...) pass through to the real module. + */ +const journalFs = vi.hoisted(() => ({ + failOpens: false, + openAttempts: 0, +})); + +vi.mock('node:fs/promises', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + open: (path: Parameters[0], flags?: string, mode?: number) => { + journalFs.openAttempts += 1; + if (journalFs.failOpens) { + const error = new Error('injected journal write failure') as NodeJS.ErrnoException; + error.code = 'EACCES'; + return Promise.reject(error); + } + return actual.open(path, flags, mode); + }, + }; +}); + function envelope(seq: number): EventEnvelope { return { type: 'turn.started', @@ -27,17 +56,19 @@ describe('SessionEventJournal', () => { let filePath: string; beforeEach(async () => { + journalFs.failOpens = false; + journalFs.openAttempts = 0; dir = await mkdtemp(join(tmpdir(), 'kimi-journal-test-')); filePath = join(dir, 'sess_1.jsonl'); }); afterEach(async () => { + journalFs.failOpens = false; await rm(dir, { recursive: true, force: true }); }); it('assigns monotonic seq and reads back in order', async () => { const j = await SessionEventJournal.open(filePath); - expect(j.epoch).toMatch(/^ep_/); expect(j.seq).toBe(0); j.append(j.nextSeq(), envelope(1)); @@ -52,9 +83,10 @@ describe('SessionEventJournal', () => { it('recovers seq and epoch across reopen', async () => { const j1 = await SessionEventJournal.open(filePath); - const epoch = j1.epoch; j1.append(j1.nextSeq(), envelope(1)); j1.append(j1.nextSeq(), envelope(2)); + const epoch = j1.epoch; + expect(epoch).toMatch(/^ep_/); await j1.close(); const j2 = await SessionEventJournal.open(filePath); @@ -66,20 +98,39 @@ describe('SessionEventJournal', () => { it('rotates to a fresh epoch when the header is corrupt', async () => { const j1 = await SessionEventJournal.open(filePath); - const epoch = j1.epoch; j1.append(j1.nextSeq(), envelope(1)); + const epoch = j1.epoch; await j1.close(); // Corrupt the file: overwrite with a garbage first line (no header). await writeFile(filePath, 'this is not json\n', 'utf8'); const j2 = await SessionEventJournal.open(filePath); + // The baseline is lost, but the next incarnation (and its epoch) is born + // only on the next real append. + expect(j2.epoch).toBeUndefined(); + expect(j2.seq).toBe(0); + j2.append(j2.nextSeq(), envelope(1)); expect(j2.epoch).toMatch(/^ep_/); expect(j2.epoch).not.toBe(epoch); - expect(j2.seq).toBe(0); await j2.close(); }); + it('repairs a torn trailing line before appending the next event', async () => { + const j1 = await SessionEventJournal.open(filePath); + j1.append(j1.nextSeq(), envelope(1)); + await j1.close(); + + const durable = await readFile(filePath, 'utf8'); + await writeFile(filePath, `${durable}{"kind":"event"}`, 'utf8'); + const j2 = await SessionEventJournal.open(filePath); + expect(j2.seq).toBe(1); + j2.append(j2.nextSeq(), envelope(2)); + await j2.close(); + + expect((await j2.readSince(0, 100)).map((entry) => entry.seq)).toEqual([1, 2]); + }); + it('readSince honors the exclusive lower bound and the limit', async () => { const j = await SessionEventJournal.open(filePath); for (let i = 1; i <= 5; i++) j.append(j.nextSeq(), envelope(i)); @@ -91,8 +142,60 @@ describe('SessionEventJournal', () => { it('readSince on a missing file returns empty', async () => { const j = await SessionEventJournal.open(filePath); - const out = await j.readSince(0, 100); - expect(out).toEqual([]); + expect(await j.readSince(0, 100)).toEqual([]); + await j.close(); + }); + + it('keeps pending lines across a failed flush and replays them after a retry', async () => { + const j = await SessionEventJournal.open(filePath); + j.append(j.nextSeq(), envelope(1)); + await j.readSince(0, 100); // batch 1 durable (header + seq 1) + expect(journalFs.openAttempts).toBe(1); + + // Batch 2 fails: the lines must NOT be dropped — the file still holds seq 1. + journalFs.failOpens = true; + j.append(j.nextSeq(), envelope(2)); + j.append(j.nextSeq(), envelope(3)); + await vi.waitFor(() => expect(journalFs.openAttempts).toBe(2)); + journalFs.failOpens = false; + + const mid = await SessionEventJournal.open(filePath); + expect(mid.seq).toBe(1); + await mid.close(); + + // readSince retries the pending flush, then serves the full tail. + const all = await j.readSince(1, 100); + expect(all.map((e) => e.seq)).toEqual([2, 3]); + await j.close(); + + const reopened = await SessionEventJournal.open(filePath); + expect(reopened.seq).toBe(3); + expect(reopened.epoch).toBe(j.epoch); + await reopened.close(); + }); + + it('enters a sticky failure state after consecutive write failures', async () => { + const j = await SessionEventJournal.open(filePath); + journalFs.failOpens = true; + j.append(j.nextSeq(), envelope(1)); + + // The scheduled round fails (1); the flush inside readSince retries and + // fails again (2) → sticky — and the read fails loudly instead of + // silently serving fewer events. + await expect(j.readSince(0, 100)).rejects.toBeInstanceOf(JournalStorageError); + + // Sticky: subsequent writes fail fast (no unbounded pending growth), and + // reads keep failing explicitly. + expect(() => j.nextSeq()).toThrow(JournalStorageError); + expect(() => j.append(2, envelope(2))).toThrow(JournalStorageError); + await expect(j.readSince(0, 100)).rejects.toBeInstanceOf(JournalStorageError); + + // The sticky state is terminal for this instance — storage "recovering" + // does not silently un-stick it mid-flight. + journalFs.failOpens = false; + expect(() => j.append(2, envelope(2))).toThrow(JournalStorageError); + + // close() must neither throw nor hot-spin on the persistent failure. await j.close(); }); diff --git a/packages/kap-server/test/sessionListWatch.test.ts b/packages/kap-server/test/sessionListWatch.test.ts new file mode 100644 index 0000000000..10f1913f89 --- /dev/null +++ b/packages/kap-server/test/sessionListWatch.test.ts @@ -0,0 +1,179 @@ +/** + * `SessionListWatchService` — two-layer watcher over the shared sessions tree + * (design §3.8 event plane; run: + * `pnpm --filter @moonshot-ai/kap-server exec vitest run test/sessionListWatch.test.ts`). + * + * Real chokidar (`HostFsWatchService`) + real tmp dirs; only the core event + * bus is faked (a collect-only `publish`). Cases pin the §3.8 contract: + * existing workspace + new session dir → hint; new workspace appears → hint; + * session/workspace dirs deleted → hint; boot with pre-existing dirs is + * silent; a same-window burst collapses into ONE hint; nothing fires after + * dispose. + */ +import { mkdirSync, mkdtemp, rmSync } from 'node:fs'; +import { rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { + HostFsWatchService, + type DomainEvent, + type IEventService, +} from '@moonshot-ai/agent-core-v2'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { SessionListWatchService } from '../src/services/sessionListWatch/sessionListWatchService'; + +const DEBOUNCE_MS = 40; +/** Time given to chokidar to register newly-watched paths before mutating + (same settle constant as the fs-watch e2e suite). */ +const WATCH_SETTLE_MS = 150; +/** How long to wait for a hint before declaring it lost. */ +const HINT_TIMEOUT_MS = 8_000; + +const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); + +function collectingEvents(): { events: IEventService; published: DomainEvent[] } { + const published: DomainEvent[] = []; + return { + published, + events: { + publish: (event: DomainEvent) => published.push(event), + } as unknown as IEventService, + }; +} + +describe('SessionListWatchService', () => { + let root: string; + let sessionsDir: string; + let published: DomainEvent[]; + let service: SessionListWatchService | undefined; + + beforeEach(async () => { + root = await new Promise((resolve, reject) => + mkdtemp(join(tmpdir(), 'kimi-sessionlist-watch-'), (err, dir) => + err === null ? resolve(dir) : reject(err), + ), + ); + sessionsDir = join(root, 'sessions'); + ({ published } = collectingEvents()); + }); + + afterEach(async () => { + service?.dispose(); + service = undefined; + if (root !== '') { + await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }); + } + }); + + async function boot(opts?: { debounceMs?: number }): Promise { + const events = collectingEvents(); + published = events.published; + service = new SessionListWatchService({ + sessionsDir, + fsWatch: new HostFsWatchService(), + events: events.events, + debounceMs: opts?.debounceMs ?? DEBOUNCE_MS, + }); + await service.start(); + await sleep(WATCH_SETTLE_MS); + } + + /** Wait until at least `count` hints were published; fails the test on timeout. */ + async function waitForHints(count: number): Promise { + await vi.waitFor(() => expect(published.length).toBeGreaterThanOrEqual(count), { + timeout: HINT_TIMEOUT_MS, + interval: 25, + }); + } + + function expectHintShape(event: DomainEvent | undefined): void { + expect(event).toEqual({ type: 'session.list_changed', payload: {} }); + } + + it( + 'emits a hint when a new workspace appears, and again for a session created inside it', + { timeout: 20_000 }, + async () => { + await boot(); + expect(published).toHaveLength(0); + + // Root watcher: workspace dir appears. + mkdirSync(join(sessionsDir, 'wsA')); + await waitForHints(1); + expectHintShape(published[0]); + + // The dynamically added per-workspace watcher picks up sessions created + // inside the new workspace afterwards. + await sleep(WATCH_SETTLE_MS); + mkdirSync(join(sessionsDir, 'wsA', 's1')); + await waitForHints(2); + expectHintShape(published[1]); + }, + ); + + it( + 'emits a hint when a session dir is deleted', + { timeout: 20_000 }, + async () => { + mkdirSync(join(sessionsDir, 'ws1', 's1'), { recursive: true }); + mkdirSync(join(sessionsDir, 'ws1', 's2'), { recursive: true }); + await boot(); + + // ignoreInitial: pre-existing workspace/session trees stay silent. + expect(published).toHaveLength(0); + + rmSync(join(sessionsDir, 'ws1', 's2'), { recursive: true, force: true }); + await waitForHints(1); + expectHintShape(published[0]); + }, + ); + + it( + 'emits a hint when a workspace dir is deleted', + { timeout: 20_000 }, + async () => { + mkdirSync(join(sessionsDir, 'ws1'), { recursive: true }); + await boot(); + + rmSync(join(sessionsDir, 'ws1'), { recursive: true, force: true }); + await waitForHints(1); + expectHintShape(published[0]); + }, + ); + + it( + 'debounces a same-window burst into a single hint', + { timeout: 20_000 }, + async () => { + mkdirSync(join(sessionsDir, 'ws1'), { recursive: true }); + await boot(); + + mkdirSync(join(sessionsDir, 'ws1', 's1')); + mkdirSync(join(sessionsDir, 'ws1', 's2')); + mkdirSync(join(sessionsDir, 'ws1', 's3')); + await waitForHints(1); + // Silence several debounce windows: nothing more may arrive — the burst + // was coalesced, and repeated hints would inflate the count. + await sleep(DEBOUNCE_MS * 6); + expect(published).toHaveLength(1); + }, + ); + + it( + 'stops emitting after dispose', + { timeout: 20_000 }, + async () => { + mkdirSync(join(sessionsDir, 'ws1'), { recursive: true }); + await boot(); + + service?.dispose(); + + mkdirSync(join(sessionsDir, 'ws1', 's1')); + mkdirSync(join(sessionsDir, 'ws2')); + await sleep(WATCH_SETTLE_MS + DEBOUNCE_MS * 5); + expect(published).toHaveLength(0); + }, + ); +}); diff --git a/packages/kap-server/test/sessions.test.ts b/packages/kap-server/test/sessions.test.ts index 69cda99b17..df90dd46ad 100644 --- a/packages/kap-server/test/sessions.test.ts +++ b/packages/kap-server/test/sessions.test.ts @@ -17,10 +17,12 @@ import { type DomainEvent, IAgentGoalService, IAgentLifecycleService, + ICrossProcessLockService, IEventBus, IEventService, ISessionLifecycleService, MAIN_AGENT_ID, + sessionLeasePath, type ServiceIdentifier, } from '@moonshot-ai/agent-core-v2'; import { sessionWarningsResponseSchema } from '@moonshot-ai/agent-core-v2/app/sessionLegacy/sessionProtocol'; @@ -49,6 +51,7 @@ interface SessionWire { pending_interaction: 'none' | 'approval' | 'question'; last_turn_reason?: 'completed' | 'cancelled' | 'failed'; archived?: boolean; + ownership?: { held_by: 'self' | 'peer' | 'none'; address?: string }; metadata: { cwd: string } & Record; agent_config: { model: string }; usage: { input_tokens: number }; @@ -354,6 +357,47 @@ describe('server-v2 /api/v1/sessions', () => { expect(typeof body.data.has_more).toBe('boolean'); }); + it('joins the lease into ownership: self while materialized, none after close', async () => { + const cwd = home as string; + const created = await postJson('/api/v1/sessions', { metadata: { cwd } }); + const id = created.body.data.id; + + const listed = await getJson('/api/v1/sessions'); + expect(listed.body.data.items.find((s) => s.id === id)?.ownership).toEqual({ + held_by: 'self', + }); + + await (server as RunningServer).core.accessor.get(ISessionLifecycleService).close(id); + + const after = await getJson('/api/v1/sessions'); + expect(after.body.data.items.find((s) => s.id === id)?.ownership).toEqual({ + held_by: 'none', + }); + }); + + it('joins the lease into ownership: peer with address when another instance holds it', async () => { + const cwd = home as string; + const created = await postJson('/api/v1/sessions', { metadata: { cwd } }); + const id = created.body.data.id; + await (server as RunningServer).core.accessor.get(ISessionLifecycleService).close(id); + + // Simulate a peer instance: with no local live scope, any held lease + // classifies as 'peer' and the owner metadata's address is joined in. + const locks = (server as RunningServer).core.accessor.get(ICrossProcessLockService); + const peer = await locks.acquire(sessionLeasePath(home as string, id), { + address: 'http://127.0.0.1:59999', + }); + try { + const listed = await getJson('/api/v1/sessions'); + expect(listed.body.data.items.find((s) => s.id === id)?.ownership).toEqual({ + held_by: 'peer', + address: 'http://127.0.0.1:59999', + }); + } finally { + peer.release(); + } + }); + it('supports exclude_empty when listing sessions', async () => { const cwd = home as string; const created = await postJson('/api/v1/sessions', { metadata: { cwd } }); diff --git a/packages/kap-server/test/skillCatalogBridge.test.ts b/packages/kap-server/test/skillCatalogBridge.test.ts new file mode 100644 index 0000000000..737952171e --- /dev/null +++ b/packages/kap-server/test/skillCatalogBridge.test.ts @@ -0,0 +1,149 @@ +/** + * `SkillCatalogBridge` — volatile `skill_catalog.changed` delivery: one frame + * per core change fanned out under each connection's own `seq` (no journal + * epoch), and refcounted release of the core subscription on last detach. + * Fake core accessor (per `wsConnectionV1.test.ts`); `onDidChange` hand-fired. + */ +import { + type IDisposable, + ISessionSkillCatalog, + ISessionLifecycleService, + type ISessionScopeHandle, + type Scope, +} from '@moonshot-ai/agent-core-v2'; +import { describe, expect, it, vi } from 'vitest'; + +import { + SkillCatalogBridge, + type SkillCatalogChangedFrame, + type SkillCatalogConnection, +} from '../src/transport/ws/v1/skillCatalogBridge'; +import type { EventEnvelope } from '../src/transport/ws/v1/sessionEventJournal'; +import { createSessionLifecycleHooks } from './helpers/sessionEvents'; + +function makeConn(id: string): { conn: SkillCatalogConnection; frames: SkillCatalogChangedFrame[] } { + const frames: SkillCatalogChangedFrame[] = []; + return { + frames, + conn: { + id, + send: (envelope: EventEnvelope) => frames.push(envelope as unknown as SkillCatalogChangedFrame), + }, + }; +} + +interface FakeCatalog { + readonly service: ISessionSkillCatalog; + readonly onDidChange: ReturnType; + readonly dispose: ReturnType; + fire(sourceId: string): void; + listenerCount(): number; +} + +function makeCatalog(): FakeCatalog { + let listener: ((sourceId: string) => void) | undefined; + const dispose = vi.fn(() => { + listener = undefined; + }); + const onDidChange = vi.fn((cb: (sourceId: string) => void): IDisposable => { + listener = cb; + return { dispose }; + }); + return { + service: { onDidChange } as unknown as ISessionSkillCatalog, + onDidChange, + dispose, + fire: (sourceId: string) => listener?.(sourceId), + listenerCount: () => (listener === undefined ? 0 : 1), + }; +} + +function makeCore(sessions: Map): Scope { + const hooks = createSessionLifecycleHooks(); + const lifecycle = { get: (sid: string) => sessions.get(sid), hooks }; + return { + accessor: { + get: (decorator: unknown) => { + if (decorator === ISessionLifecycleService) return lifecycle; + throw new Error('unexpected service lookup'); + }, + }, + } as unknown as Scope; +} + +function makeSession(catalog: ISessionSkillCatalog): ISessionScopeHandle { + return { + accessor: { + get: (decorator: unknown) => { + if (decorator === ISessionSkillCatalog) return catalog; + throw new Error('unexpected service lookup'); + }, + }, + } as unknown as ISessionScopeHandle; +} + +describe('SkillCatalogBridge', () => { + it('fans one volatile frame per core change out to every subscribed connection', () => { + const catalog = makeCatalog(); + const sessions = new Map([['sess_1', makeSession(catalog.service)]]); + const bridge = new SkillCatalogBridge({ core: makeCore(sessions) }); + const a = makeConn('conn_a'); + const b = makeConn('conn_b'); + + bridge.attachSession(a.conn, 'sess_1'); + bridge.attachSession(b.conn, 'sess_1'); + // One core subscription shared by both connections of the session. + expect(catalog.onDidChange).toHaveBeenCalledTimes(1); + expect(a.frames).toHaveLength(0); + expect(b.frames).toHaveLength(0); + + catalog.fire('workspace-file'); + + expect(a.frames).toHaveLength(1); + expect(b.frames).toHaveLength(1); + for (const frames of [a.frames, b.frames]) { + const frame = frames[0]!; + expect(frame).toMatchObject({ + type: 'skill_catalog.changed', + // Per-connection monotonic seq — not the durable session watermark. + seq: 1, + session_id: 'sess_1', + volatile: true, + payload: { type: 'skill_catalog.changed', sourceId: 'workspace-file' }, + }); + // No journalling surface: no journal epoch is attached to the frame. + expect('epoch' in frame).toBe(false); + } + + // A second change advances each connection's own seq only. + catalog.fire('plugin'); + expect(a.frames[1]).toMatchObject({ seq: 2, payload: { sourceId: 'plugin' } }); + expect(b.frames[1]).toMatchObject({ seq: 2, payload: { sourceId: 'plugin' } }); + }); + + it('stops delivering to an unsubscribed connection and releases the core subscription on last detach', () => { + const catalog = makeCatalog(); + const sessions = new Map([['sess_1', makeSession(catalog.service)]]); + const bridge = new SkillCatalogBridge({ core: makeCore(sessions) }); + const a = makeConn('conn_a'); + const b = makeConn('conn_b'); + + bridge.attachSession(a.conn, 'sess_1'); + bridge.attachSession(b.conn, 'sess_1'); + bridge.detachSession(a.conn, 'sess_1'); + catalog.fire('workspace-file'); + + expect(a.frames).toHaveLength(0); + expect(b.frames).toHaveLength(1); + // conn_b still holds the session: core subscription is NOT released. + expect(catalog.dispose).not.toHaveBeenCalled(); + + bridge.detachSession(b.conn, 'sess_1'); + expect(catalog.dispose).toHaveBeenCalledTimes(1); + expect(catalog.listenerCount()).toBe(0); + + catalog.fire('plugin'); + expect(a.frames).toHaveLength(0); + expect(b.frames).toHaveLength(1); + }); +}); diff --git a/packages/kap-server/test/transport-errors.test.ts b/packages/kap-server/test/transport-errors.test.ts index 238a6963b0..2e450514dd 100644 --- a/packages/kap-server/test/transport-errors.test.ts +++ b/packages/kap-server/test/transport-errors.test.ts @@ -20,6 +20,7 @@ describe('/api/v1/debug transport mapError', () => { [ErrorCodes.STORAGE_IO_FAILED, ErrorCode.PERSISTENCE_FAILURE], [ErrorCodes.STORAGE_LOCKED, ErrorCode.PERSISTENCE_FAILURE], [ErrorCodes.GOAL_UNSUPPORTED_AGENT, ErrorCode.GOAL_UNSUPPORTED_AGENT], + [ErrorCodes.SESSION_HELD_BY_PEER, ErrorCode.SESSION_HELD_BY_PEER], ])('maps domain code %s to its wire equivalent', (code, wire) => { const env = mapError(new Error2(code, 'boom'), 'req-1'); expect(env.code).toBe(wire); @@ -29,4 +30,16 @@ describe('/api/v1/debug transport mapError', () => { const env = mapError(new Error2(ErrorCodes.OS_FS_UNKNOWN, 'boom'), 'req-1'); expect(env.code).toBe(ErrorCode.INTERNAL_ERROR); }); + + it('forwards Error2 details onto the mapped envelope verbatim', () => { + const details = { kind: 'held-by-peer', phase: 'routable', address: 'http://127.0.0.1:58628' }; + const env = mapError( + new Error2(ErrorCodes.SESSION_HELD_BY_PEER, 'session is owned by another instance', { + details, + }), + 'req-1', + ); + expect(env.code).toBe(ErrorCode.SESSION_HELD_BY_PEER); + expect(env.details).toEqual(details); + }); }); diff --git a/packages/kap-server/test/workspaceFs.test.ts b/packages/kap-server/test/workspaceFs.test.ts index 544e32f258..7f0a62719e 100644 --- a/packages/kap-server/test/workspaceFs.test.ts +++ b/packages/kap-server/test/workspaceFs.test.ts @@ -123,7 +123,9 @@ describe('server-v2 /api/v1 fs folder picker', () => { expect(body.code).toBe(0); expect(body.data.path).toBe(await realpath(root)); const names = body.data.entries.map((e) => e.name).sort(); - expect(names).toEqual(['alpha', 'beta']); + // `sessions/` is created by the server itself at boot (the always-on + // session-list watch mkdirs `/sessions` before watching it). + expect(names).toEqual(['alpha', 'beta', 'sessions']); for (const entry of body.data.entries) { expect(entry.is_dir).toBe(true); expect(entry.path).toBe(join(await realpath(root), entry.name)); @@ -139,7 +141,9 @@ describe('server-v2 /api/v1 fs folder picker', () => { `/api/v1/fs:browse?path=${encodeURIComponent(root)}`, ); expect(body.code).toBe(0); - expect(body.data.entries.map((e) => e.name)).toEqual(['alpha', '.zeta']); + // `sessions/` sits between regular and dot-directories: the always-on + // session-list watch creates `/sessions` at boot. + expect(body.data.entries.map((e) => e.name)).toEqual(['alpha', 'sessions', '.zeta']); }); it('returns parent=null for the filesystem root', async () => { diff --git a/packages/kap-server/test/wsConnectionV1.test.ts b/packages/kap-server/test/wsConnectionV1.test.ts index 3c91224b8f..f3f06ba8b9 100644 --- a/packages/kap-server/test/wsConnectionV1.test.ts +++ b/packages/kap-server/test/wsConnectionV1.test.ts @@ -62,6 +62,7 @@ function makeBroadcaster(): SessionEventBroadcaster { return { subscribe: async () => true, unsubscribe: () => {}, + getSubscriptionFailure: async () => undefined, getCursor: async () => ({ seq: 0, epoch: '' }), getBufferedSince: async () => ({ events: [], diff --git a/packages/kap-server/test/wsV1Resync.test.ts b/packages/kap-server/test/wsV1Resync.test.ts index ff01d33851..b007ba63d6 100644 --- a/packages/kap-server/test/wsV1Resync.test.ts +++ b/packages/kap-server/test/wsV1Resync.test.ts @@ -4,7 +4,7 @@ * cursor-based replay. */ -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdir, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -192,7 +192,7 @@ describe('server-v2 /api/v1/ws resync', () => { c.send({ type: 'client_hello', id: 'h1', payload: withToken({ client_id: 'cli', subscriptions: [sid] }) }); await c.next((f) => f.type === 'ack' && f.id === 'h1'); - emitAgentEvent(sid, { type: 'turn.started', turnId: 1 } as unknown as DomainEvent); + emitAgentEvent(sid, { type: 'turn.started', turnId: 1, origin: { kind: 'user' } } as unknown as DomainEvent); const ev = await c.next((f) => f.type === 'turn.started'); expect(ev.seq).toBeGreaterThanOrEqual(1); @@ -212,7 +212,7 @@ describe('server-v2 /api/v1/ws resync', () => { await c1.next((f) => f.type === 'server_hello'); c1.send({ type: 'client_hello', id: 'h1', payload: withToken({ client_id: 'cli', subscriptions: [sid] }) }); await c1.next((f) => f.type === 'ack' && f.id === 'h1'); - emitAgentEvent(sid, { type: 'turn.started', turnId: 1 } as unknown as DomainEvent); + emitAgentEvent(sid, { type: 'turn.started', turnId: 1, origin: { kind: 'user' } } as unknown as DomainEvent); emitAgentEvent(sid, { type: 'turn.ended', turnId: 1 } as unknown as DomainEvent); await c1.next((f) => f.type === 'turn.ended'); c1.ws.close(); @@ -255,6 +255,58 @@ describe('server-v2 /api/v1/ws resync', () => { await c.closed; }); + it('sends resync_required when the journal cannot serve replay (storage failure)', async () => { + const sid = await createSession(); + await ensureMainAgent(sid); + const journalPath = join(home as string, 'server', 'events', `${sid}.jsonl`); + + // Subscribe (activates the journal) and land one durable event on disk. + const c1 = await openConn(wsUrl, server!.authTokenService.getToken()); + await c1.next((f) => f.type === 'server_hello'); + c1.send({ type: 'client_hello', id: 'h1', payload: withToken({ client_id: 'cli', subscriptions: [sid] }) }); + await c1.next((f) => f.type === 'ack' && f.id === 'h1'); + emitAgentEvent(sid, { type: 'turn.started', turnId: 1, origin: { kind: 'user' } } as unknown as DomainEvent); + await c1.next((f) => f.type === 'turn.started'); + + // Sabotage the journal: replace the file with a same-named directory so + // every subsequent open('a') fails. The next durable event's flush round + // drives the journal into its failure state. + await rm(journalPath, { force: true }); + await mkdir(journalPath); + emitAgentEvent(sid, { type: 'turn.ended', turnId: 1 } as unknown as DomainEvent); + await new Promise((resolve) => setTimeout(resolve, 200)); + + // Replaying from a stale cursor: the journal cannot serve the gap, and a + // "not served" must surface as resync_required — never as an empty page + // (or as the not-yet-persisted memory tail). The failure flush rounds + // settle asynchronously, so retry the subscribe a few times; each attempt + // retries the journal flush and converges to the sticky state. + const c2 = await openConn(wsUrl, server!.authTokenService.getToken()); + await c2.next((f) => f.type === 'server_hello'); + let resync: Frame | undefined; + for (let attempt = 0; attempt < 5 && resync === undefined; attempt++) { + c2.send({ + type: 'subscribe', + id: `s${attempt}`, + payload: withToken({ session_ids: [sid], cursors: { [sid]: { seq: 0 } } }), + }); + resync = await c2 + .next( + (f) => + f.type === 'resync_required' && + (f.payload as { session_id?: string } | undefined)?.session_id === sid, + 400, + ) + .catch(() => undefined); + } + expect(resync).toBeDefined(); + + c1.ws.close(); + c2.ws.close(); + await c1.closed; + await c2.closed; + }); + it('delivers only the allowlisted agent events via agent_filter', async () => { const sid = await createSession(); await ensureMainAgent(sid); diff --git a/packages/kernel-file-lock/package.json b/packages/kernel-file-lock/package.json new file mode 100644 index 0000000000..454ad6ba24 --- /dev/null +++ b/packages/kernel-file-lock/package.json @@ -0,0 +1,26 @@ +{ + "name": "@moonshot-ai/kernel-file-lock", + "version": "0.1.0", + "private": true, + "description": "Cross-platform kernel-backed advisory file locks for Kimi Code.", + "license": "MIT", + "type": "module", + "files": [ + "dist" + ], + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "scripts": { + "build": "tsdown", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run", + "clean": "rm -rf dist" + }, + "dependencies": { + "fs-native-extensions": "1.5.0" + } +} diff --git a/packages/kernel-file-lock/src/index.ts b/packages/kernel-file-lock/src/index.ts new file mode 100644 index 0000000000..4b117febfb --- /dev/null +++ b/packages/kernel-file-lock/src/index.ts @@ -0,0 +1,94 @@ +import { closeSync, fstatSync, mkdirSync, openSync, statSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { dirname } from 'node:path'; + +export interface KernelFileLockBinding { + tryLock(fd: number): boolean; + unlock(fd: number): void; +} + +export type KernelFileLockBindingLoader = () => KernelFileLockBinding | undefined; + +export interface KernelFileLockHandle { + checkHeld(): boolean; + release(): void; +} + +const bindingLoaderKey = Symbol.for('@moonshot-ai/kernel-file-lock/binding-loader'); +const nodeRequire = createRequire(import.meta.url); + +type GlobalWithBindingLoader = typeof globalThis & { + [bindingLoaderKey]?: KernelFileLockBindingLoader; +}; + +let binding: KernelFileLockBinding | undefined; + +function defaultBindingLoader(): KernelFileLockBinding { + return nodeRequire('fs-native-extensions') as KernelFileLockBinding; +} + +function getBinding(): KernelFileLockBinding { + if (binding !== undefined) return binding; + const override = (globalThis as GlobalWithBindingLoader)[bindingLoaderKey]; + binding = override?.() ?? defaultBindingLoader(); + return binding; +} + +class KernelFileLockHandleImpl implements KernelFileLockHandle { + private released = false; + + constructor( + private readonly path: string, + private readonly fd: number, + private readonly binding: KernelFileLockBinding, + ) {} + + checkHeld(): boolean { + if (this.released) return false; + try { + const opened = fstatSync(this.fd); + const current = statSync(this.path); + return opened.dev === current.dev && opened.ino === current.ino; + } catch { + return false; + } + } + + release(): void { + if (this.released) return; + this.released = true; + try { + this.binding.unlock(this.fd); + } finally { + closeSync(this.fd); + } + } +} + +export function setKernelFileLockBindingLoader( + loader: KernelFileLockBindingLoader | undefined, +): void { + const target = globalThis as GlobalWithBindingLoader; + if (loader === undefined) { + delete target[bindingLoaderKey]; + } else { + target[bindingLoaderKey] = loader; + } + binding = undefined; +} + +export function tryAcquireKernelFileLock(path: string): KernelFileLockHandle | undefined { + const nativeBinding = getBinding(); + mkdirSync(dirname(path), { recursive: true }); + const fd = openSync(path, 'a+', 0o600); + try { + if (!nativeBinding.tryLock(fd)) { + closeSync(fd); + return undefined; + } + return new KernelFileLockHandleImpl(path, fd, nativeBinding); + } catch (error) { + closeSync(fd); + throw error; + } +} diff --git a/packages/kernel-file-lock/test/holder.ts b/packages/kernel-file-lock/test/holder.ts new file mode 100644 index 0000000000..3ce659208f --- /dev/null +++ b/packages/kernel-file-lock/test/holder.ts @@ -0,0 +1,13 @@ +import { tryAcquireKernelFileLock } from '../src/index.js'; + +const path = process.argv[2]; +if (path === undefined) throw new Error('lock path is required'); + +const handle = tryAcquireKernelFileLock(path); +if (handle === undefined) process.exit(2); + +process.stdout.write('locked\n'); +process.stdin.once('data', () => { + handle.release(); + process.exit(0); +}); diff --git a/packages/kernel-file-lock/test/kernel-file-lock.test.ts b/packages/kernel-file-lock/test/kernel-file-lock.test.ts new file mode 100644 index 0000000000..568927ee83 --- /dev/null +++ b/packages/kernel-file-lock/test/kernel-file-lock.test.ts @@ -0,0 +1,111 @@ +import { existsSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { fileURLToPath } from 'node:url'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { tryAcquireKernelFileLock, type KernelFileLockHandle } from '../src/index.js'; + +let tmpDir: string; +let lockPath: string; +const handles: KernelFileLockHandle[] = []; + +async function withTimeout(promise: Promise, timeoutMs: number, message: string): Promise { + let timer: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(message)), timeoutMs); + }); + try { + return await Promise.race([promise, timeout]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'kernel-file-lock-')); + lockPath = join(tmpDir, 'resource.lock'); +}); + +afterEach(() => { + for (const handle of handles.splice(0)) handle.release(); + rmSync(tmpDir, { recursive: true, force: true }); +}); + +describe('kernel-file-lock', () => { + it('holds an exclusive lock and keeps the sentinel after release', () => { + const first = tryAcquireKernelFileLock(lockPath); + expect(first).toBeDefined(); + handles.push(first!); + expect(tryAcquireKernelFileLock(lockPath)).toBeUndefined(); + + first!.release(); + expect(existsSync(lockPath)).toBe(true); + const second = tryAcquireKernelFileLock(lockPath); + expect(second).toBeDefined(); + handles.push(second!); + }); + + it('coordinates with a separate process', async () => { + const holderPath = fileURLToPath(new URL('./holder.ts', import.meta.url)); + const child = spawn(process.execPath, ['--import', 'tsx', holderPath, lockPath], { + stdio: ['pipe', 'pipe', 'inherit'], + }); + const exit = once(child, 'exit'); + void exit.catch(() => {}); + const ready = Promise.race([ + once(child.stdout!, 'data'), + exit.then(() => { + throw new Error('holder exited before becoming ready'); + }), + ]); + try { + await withTimeout(ready, 5_000, 'holder did not become ready'); + expect(tryAcquireKernelFileLock(lockPath)).toBeUndefined(); + + child.stdin!.end('release\n'); + await withTimeout(exit, 5_000, 'holder did not exit after release'); + const handle = tryAcquireKernelFileLock(lockPath); + expect(handle).toBeDefined(); + handles.push(handle!); + } finally { + child.stdin?.end(); + if (child.pid !== undefined && child.exitCode === null && child.signalCode === null) { + child.kill(); + try { + await withTimeout(exit, 1_000, 'holder did not terminate'); + } catch { + child.kill('SIGKILL'); + await withTimeout(exit, 1_000, 'holder did not terminate after SIGKILL'); + } + } + } + }); + + it('releases the lock when the holder process is killed without releasing', async () => { + const holderPath = fileURLToPath(new URL('./holder.ts', import.meta.url)); + const child = spawn(process.execPath, ['--import', 'tsx', holderPath, lockPath], { + stdio: ['pipe', 'pipe', 'inherit'], + }); + const exit = once(child, 'exit'); + void exit.catch(() => {}); + const ready = Promise.race([ + once(child.stdout!, 'data'), + exit.then(() => { + throw new Error('holder exited before becoming ready'); + }), + ]); + await withTimeout(ready, 5_000, 'holder did not become ready'); + expect(tryAcquireKernelFileLock(lockPath)).toBeUndefined(); + + // No clean release: the kernel must drop the lock with the dead process. + child.kill('SIGKILL'); + await withTimeout(exit, 5_000, 'holder did not exit after SIGKILL'); + const handle = tryAcquireKernelFileLock(lockPath); + expect(handle).toBeDefined(); + handles.push(handle!); + }); +}); diff --git a/packages/kernel-file-lock/tsconfig.json b/packages/kernel-file-lock/tsconfig.json new file mode 100644 index 0000000000..ef502e89c8 --- /dev/null +++ b/packages/kernel-file-lock/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src", "test"] +} diff --git a/packages/kernel-file-lock/tsdown.config.ts b/packages/kernel-file-lock/tsdown.config.ts new file mode 100644 index 0000000000..2a7eb793a6 --- /dev/null +++ b/packages/kernel-file-lock/tsdown.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from 'tsdown'; + +export default defineConfig({ + entry: ['./src/index.ts'], + format: ['esm'], + dts: true, + outDir: 'dist', + clean: true, + deps: { + neverBundle: ['fs-native-extensions'], + }, +}); diff --git a/packages/kernel-file-lock/vitest.config.ts b/packages/kernel-file-lock/vitest.config.ts new file mode 100644 index 0000000000..cbd8944d3b --- /dev/null +++ b/packages/kernel-file-lock/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + name: 'kernel-file-lock', + include: ['test/**/*.test.ts'], + }, +}); diff --git a/packages/klient/AGENTS.md b/packages/klient/AGENTS.md index d4724f483b..790bc400ca 100644 --- a/packages/klient/AGENTS.md +++ b/packages/klient/AGENTS.md @@ -40,10 +40,39 @@ terminal surface are v1-only and live in the legacy suites. suites (moved from server-e2e). They skip unless `KIMI_SERVER_URL` points at a running server and **must keep running unchanged**; the v1 surface has no in-memory equivalent, so these stay live-server-only — do not try - to run them against the in-process transports. + to run them against the in-process transports. Exception: the + dual-instance / session-ownership suite (`legacy/session-ownership.test.ts`) + boots its own kap-server instances via the helpers below and runs without + `KIMI_SERVER_URL`. - The retired `scenarios/` scripts were rewritten as suites: image-upload and terminal (v1-only surfaces) live in `test/e2e/legacy/`. +## Dual-instance helpers + +Multi-server e2e cases boot two `kap-server` instances on ONE shared home via +`test/e2e/harness/testing/` (re-exported from `test/e2e/harness/index.js`). + +- **`startServerPair()`** — two in-process instances on one shared + `mkdtemp` home, each with `port: 0`, `logLevel: 'silent'`, and + `disableAuth: true`. Returns + `{ a, b, home, cwd, urlA, urlB, connectClient(server), dispose() }`; + `connectClient` returns an `HttpClient`. `cwd` is + the shared workspace cwd — pass it as `metadata.cwd` in `createSession` on + both instances ("same cwd" is session-level, never a server flag). + `dispose()` closes both instances (best-effort) and removes the home. + +Hard rules: + +- Always `port: 0`. A fixed busy port silently walks to `port + 1`, which + breaks registry/port assertions and cross-test isolation. +- One pair per test file/worker; never share a `RunningServer` + across files — vitest runs files in parallel workers. +- Readiness when NOT using these helpers: poll `GET /api/v1/healthz` + (auth-exempt) until 200 — not `/api/v1/meta` (token-gated). +- The helper imports `@moonshot-ai/kap-server` lazily at call time so the + harness barrel stays loadable under plain `tsx` without the raw-text + loader; keep that pattern when extending it. + ## Observability (inherited from server-e2e) - Keep observability inside each e2e case; every live case prints structured, @@ -61,6 +90,9 @@ terminal surface are v1-only and live in the legacy suites. conformance + e2e; live cases skip without their env). - `KIMI_SERVER_URL=http://127.0.0.1:58627 pnpm --filter @moonshot-ai/klient test` — include the live legacy cases against a running server. +- `pnpm --filter @moonshot-ai/klient exec vitest run test/e2e/legacy/session-ownership.test.ts` + — multi-server ownership e2e (boots its own in-process servers; no + external server needed). - `pnpm --filter @moonshot-ai/klient docker:e2e` — docker e2e; the run derives its runner name/namespace from the current workspace to avoid cross-workspace conflicts. diff --git a/packages/klient/package.json b/packages/klient/package.json index 57891771be..69906a9496 100644 --- a/packages/klient/package.json +++ b/packages/klient/package.json @@ -46,6 +46,7 @@ "zod": "catalog:" }, "devDependencies": { + "@moonshot-ai/kap-server": "workspace:^", "@moonshot-ai/protocol": "workspace:^", "@types/ws": "^8.18.0", "ulid": "^3.0.1", diff --git a/packages/klient/src/core/facade/session.ts b/packages/klient/src/core/facade/session.ts index cce957aa60..a5147a3a3e 100644 --- a/packages/klient/src/core/facade/session.ts +++ b/packages/klient/src/core/facade/session.ts @@ -66,6 +66,7 @@ export interface SessionFacade { update(patch: SessionMetaPatch): Promise; setArchived(archived: boolean): Promise; status(): Promise; + /** Release the session; failures abandon internally before rejecting. */ close(): Promise; archive(): Promise; /** Re-materialize a closed session; `false` when it no longer exists. */ diff --git a/packages/klient/test/e2e/harness/index.ts b/packages/klient/test/e2e/harness/index.ts index 3a27695868..019f345487 100644 --- a/packages/klient/test/e2e/harness/index.ts +++ b/packages/klient/test/e2e/harness/index.ts @@ -68,3 +68,8 @@ export type { ReverseRpcOptions } from './reverse-rpc.js'; export { DEFAULT_FRAME_TIMEOUT_MS, waitForFrame, waitForSessionBusy } from './wait.js'; +// ── Dual-instance test helpers (additive) ───────────────────────────────── +// Boot helper for multi-server e2e cases: `startServerPair` (in-process). It +// imports `@moonshot-ai/kap-server` lazily at call time, so this barrel stays +// loadable under plain `tsx` without the raw-text loader. +export * from './testing/index.js'; diff --git a/packages/klient/test/e2e/harness/testing/index.ts b/packages/klient/test/e2e/harness/testing/index.ts new file mode 100644 index 0000000000..17461b9503 --- /dev/null +++ b/packages/klient/test/e2e/harness/testing/index.ts @@ -0,0 +1,5 @@ +/** + * Dual-instance test helpers — see the "Dual-instance helpers" section of + * `packages/klient/AGENTS.md`. + */ +export * from './serverPair.js'; diff --git a/packages/klient/test/e2e/harness/testing/serverPair.ts b/packages/klient/test/e2e/harness/testing/serverPair.ts new file mode 100644 index 0000000000..adf57e609e --- /dev/null +++ b/packages/klient/test/e2e/harness/testing/serverPair.ts @@ -0,0 +1,106 @@ +/** + * In-process dual-instance boot: two `kap-server` instances sharing ONE home + * directory inside the current (test) process. + * + * One hard requirement this helper encapsulates: both instances must bind + * `port: 0` (OS-assigned) — a fixed busy port silently walks to `port + 1`, + * which breaks assertions on the registry. + * + * `@moonshot-ai/kap-server` is imported lazily *inside* the function: its + * module graph contains `*.md?raw` imports that plain `tsx` (running without + * the raw-text loader) cannot resolve. Static imports would make the whole + * harness barrel unloadable there. Type-only imports are erased at compile + * time and stay safe. + */ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import type { RunningServer } from '@moonshot-ai/kap-server'; + +import { HttpClient } from '../http.js'; + +// `recursive` rm can hit ENOTEMPTY on macOS while the closing server is still +// flushing/unlinking its own files — retry briefly. +const RM_HOME_OPTIONS = { recursive: true, force: true, maxRetries: 5, retryDelay: 100 } as const; + +export interface ServerPair { + readonly a: RunningServer; + readonly b: RunningServer; + readonly home: string; + /** + * The shared workspace cwd for sessions created against the pair (always + * the pair home). "Same cwd" is a session-level concept — pass this as + * `metadata.cwd` in `createSession` on both instances. + */ + readonly cwd: string; + /** Base URL of instance `a` (`http://host:port`). */ + readonly urlA: string; + /** Base URL of instance `b` (`http://host:port`). */ + readonly urlB: string; + /** REST client for one instance (the pair always boots with `disableAuth`). */ + connectClient(server: RunningServer): HttpClient; + /** Close both instances (idempotent, best-effort) and remove the home. */ + dispose(): Promise; +} + +export async function startServerPair(): Promise { + const home = await mkdtemp(join(tmpdir(), 'kimi-e2e-pair-')); + try { + const { startServer } = await import('@moonshot-ai/kap-server'); + const boot = (): Promise => + startServer({ + host: '127.0.0.1', + port: 0, + homeDir: home, + logLevel: 'silent', + disableAuth: true, + }); + const a = await boot(); + let b: RunningServer; + try { + b = await boot(); + } catch (error) { + await a.close(); + throw error; + } + + const baseUrl = (server: RunningServer): string => `http://${server.host}:${server.port}`; + let disposed = false; + return { + a, + b, + home, + cwd: home, + urlA: baseUrl(a), + urlB: baseUrl(b), + connectClient: (server) => + new HttpClient({ + baseUrl: baseUrl(server), + apiPrefix: '/api/v1', + fetchImpl: fetch, + }), + dispose: async () => { + if (disposed) return; + disposed = true; + // Best-effort: a failed close must not mask the other instance's + // teardown, but it also must not disappear silently. + const results = await Promise.allSettled([a.close(), b.close()]); + for (const [label, result] of [ + ['a', results[0]], + ['b', results[1]], + ] as const) { + if (result?.status === 'rejected') { + process.stderr.write( + `[server-e2e] startServerPair dispose: instance ${label} close failed: ${String(result.reason)}\n`, + ); + } + } + await rm(home, RM_HOME_OPTIONS); + }, + }; + } catch (error) { + await rm(home, RM_HOME_OPTIONS); + throw error; + } +} diff --git a/packages/klient/test/e2e/harness/wait.ts b/packages/klient/test/e2e/harness/wait.ts index e70cadb7a8..7983d81cfd 100644 --- a/packages/klient/test/e2e/harness/wait.ts +++ b/packages/klient/test/e2e/harness/wait.ts @@ -51,6 +51,6 @@ export async function waitForSessionBusy( ); } -function sleep(ms: number): Promise { +export function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/packages/klient/test/e2e/legacy/session-ownership.test.ts b/packages/klient/test/e2e/legacy/session-ownership.test.ts new file mode 100644 index 0000000000..7af3370979 --- /dev/null +++ b/packages/klient/test/e2e/legacy/session-ownership.test.ts @@ -0,0 +1,161 @@ +/** + * Phase-2 session-ownership verification — multi-instance e2e over real REST + * boundaries (design §3.10). + * + * One session write lease per session under `/session-leases/.lock`. + * The permanent sentinel is protected by a kernel lock; the sibling + * `.lock.owner.json` document only advertises holder metadata. + * Materializing routes on a peer-held session answer HTTP 200 with envelope + * `code 40921 session.held_by_peer` + ownership details. kap-server's own e2e + * already pins the dual-open envelope schema and graceful-close takeover; the + * unique value here is cross-instance behavior and byte-level file integrity: + * A creates the session, then `GET .../warnings` fires on A and B + * concurrently — A always serves (code 0), B always loses with 40921 phase + * `routable` + A's address — followed by a `*.jsonl` byte-integrity sweep of + * the shared home (no torn records) and a single-lease assertion. + */ +import { readdir, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { sessionLeasePath } from '@moonshot-ai/agent-core-v2'; +import { ErrorCode, sessionOwnershipDetailsSchema, type Envelope } from '@moonshot-ai/protocol'; +import { describe, expect, it } from 'vitest'; + +import { startServerPair } from '../harness/testing/index.js'; +import { createCaseLogger } from './log.js'; + +describe('session ownership: concurrent dual materialization race (in-process pair)', () => { + it( + 'holder serves, peer gets 40921 routable every round, no torn JSONL on disk', + { timeout: 90_000 }, + async () => { + const log = createCaseLogger('session-ownership/materialization-race'); + const pair = await startServerPair(); + try { + const { id: sessionId } = await pair + .connectClient(pair.a) + .createSession({ metadata: { cwd: pair.cwd } }); + log('session created on A', { sessionId, urlA: pair.urlA, urlB: pair.urlB }); + + const lease = await readLease(pair.home, sessionId); + log('lease after create', lease); + expect(lease?.['address']).toBe(pair.urlA); + const lockId = lease?.['lock_id']; + expect(typeof lockId).toBe('string'); + + // Two concurrent rounds pin the same holder/peer split as any larger + // storm — each round is a sequential await, so more rounds only cost + // wall time. + const ROUNDS = 2; + for (let round = 1; round <= ROUNDS; round += 1) { + const [a, b] = await Promise.all([ + getEnvelope(pair.urlA, warningsPath(sessionId)), + getEnvelope(pair.urlB, warningsPath(sessionId)), + ]); + expect(a.status).toBe(200); + expect(a.body.code).toBe(0); + expect(b.status).toBe(200); + expect(b.body.code).toBe(ErrorCode.SESSION_HELD_BY_PEER); + const details = sessionOwnershipDetailsSchema.parse(b.body.details); + expect(details).toEqual({ kind: 'held-by-peer', phase: 'routable', address: pair.urlA }); + if (round === 1 || round === ROUNDS) { + log(`concurrent round ${round}/${ROUNDS}`, { + holder: { status: a.status, code: a.body.code }, + peer: { status: b.status, code: b.body.code, details }, + }); + } + } + + // The permanent sentinel and owner metadata remain stable under A. + const leaseFilenames = await listLeaseFilenames(pair.home); + const related = leaseFilenames.filter((name) => name.startsWith(`${sessionId}.lock`)); + expect(related).toEqual([`${sessionId}.lock`, `${sessionId}.lock.owner.json`]); + const after = await readLease(pair.home, sessionId); + expect(after?.['lock_id']).toBe(lockId); + expect(after?.['address']).toBe(pair.urlA); + + const sweep = await assertJsonlIntegrity(pair.home); + log('byte-integrity sweep', sweep); + expect(sweep.files).toBeGreaterThan(0); + } finally { + await pair.dispose(); + } + }, + ); +}); + +// ── Local helpers ────────────────────────────────────────────────────────── + +function warningsPath(sessionId: string): string { + return `/sessions/${encodeURIComponent(sessionId)}/warnings`; +} + +async function getEnvelope( + baseUrl: string, + path: string, +): Promise<{ status: number; body: Envelope }> { + const res = await fetch(`${baseUrl}/api/v1${path}`); + return { status: res.status, body: (await res.json()) as Envelope }; +} + +type LeasePayload = Record; + +/** Read diagnostic owner metadata; undefined when no holder has published it. */ +async function readLease(home: string, sessionId: string): Promise { + try { + return JSON.parse( + await readFile(`${sessionLeasePath(home, sessionId)}.owner.json`, 'utf8'), + ) as LeasePayload; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } +} + +async function listLeaseFilenames(home: string): Promise { + try { + return (await readdir(join(home, 'session-leases'))).sort(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } +} + +/** + * Byte-level integrity sweep: every non-empty line of every `*.jsonl` under + * `root` must parse as one complete JSON record — a torn / interleaved write + * from a double-materialized session shows up here as a parse failure. + */ +async function assertJsonlIntegrity(root: string): Promise<{ files: number; records: number }> { + const files = await listJsonlFiles(root); + const violations: string[] = []; + let records = 0; + for (const file of files) { + const content = await readFile(file, 'utf8'); + content.split('\n').forEach((line, index) => { + if (line.trim().length === 0) return; + try { + JSON.parse(line); + records += 1; + } catch { + violations.push(`${file}:${index + 1} :: ${line.slice(0, 120)}`); + } + }); + } + expect(violations).toEqual([]); + return { files: files.length, records }; +} + +async function listJsonlFiles(root: string): Promise { + let entries; + try { + entries = await readdir(root, { recursive: true, withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + return entries + .filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl')) + .map((entry) => join(entry.parentPath, entry.name)) + .sort(); +} diff --git a/packages/klient/tsconfig.dev.json b/packages/klient/tsconfig.dev.json new file mode 100644 index 0000000000..3127c1672e --- /dev/null +++ b/packages/klient/tsconfig.dev.json @@ -0,0 +1,16 @@ +{ + "extends": "./tsconfig.json", + // tsx (esbuild transform) applies a custom `--tsconfig` per file via its + // `include` globs. The `include` must therefore cover every workspace + // package the spawned server imports (mirrors + // `apps/kimi-code/tsconfig.dev.json`): otherwise files outside the globs + // get no tsconfig at all and DI parameter decorators in the agent-core + // graph fail with "Parameter decorators only work when experimental + // decorators are enabled". + "include": [ + "src", + "test", + "../../packages/*/src/**/*.ts", + "../../packages/*/src/**/*.tsx" + ] +} diff --git a/packages/minidb/README.md b/packages/minidb/README.md index 977003848a..39864530a7 100644 --- a/packages/minidb/README.md +++ b/packages/minidb/README.md @@ -397,11 +397,11 @@ long-run behavior: | `soak.test.js` | sustained ops + heap stability (opt-in: `SOAK=30 npm run test:e2e`) | The **cluster suite** (`test/cluster/*.test.ts`) covers the `ClusterDb` -sharding layer: topology/routing, merged scans, lock contention and lease -renewal in-process, cross-shard indexes/compaction, true multi-process +sharding layer: topology/routing, merged scans, lock contention and bounded +lock handoff in-process, cross-shard indexes/compaction, true multi-process scenarios (concurrent writers on disjoint and shared shards, live cross-process -read visibility, read/write storms) and crash takeovers (`kill -9` → contiguous -recovery + stale-lock handoff). +read visibility, read/write storms) and crash recovery (`kill -9` → contiguous +recovery + kernel-lock reacquisition). ## Design in one paragraph @@ -419,9 +419,11 @@ source-code study behind each choice. ## Concurrency & multi-process minidb is **single-writer**. Opening a directory for writing acquires an exclusive -lock file (`db.lock`); a second writer is rejected with a `LockError`. A lock is -taken over only when its owner PID is dead (stale-lock recovery), never merely -because it is old. +kernel lock on `db.lock`; a second writer is rejected with a `LockError`. The +writer keeps the file descriptor or handle open for its lifetime, and the OS +releases the lock automatically when the process exits. The `db.lock` sentinel +is permanent: it is never deleted, renamed, or replaced during acquisition or +release. ```js // second process: throws LockError @@ -475,8 +477,8 @@ await db.close(); `search` merge per-shard results (text scores are per-shard). Index management acquires every shard writer — run it from one process, off the hot path. -- Crash recovery is per shard: a lock left by a dead PID is taken over by the - next opener, exactly like single MiniDb. +- Crash recovery is per shard: process exit releases the kernel lock, so the + next opener can acquire the existing sentinel exactly like single MiniDb. `crossShard: '2pc'` is reserved for a future two-phase commit and is rejected today. Performance numbers across process/shard counts: run diff --git a/packages/minidb/package.json b/packages/minidb/package.json index ad8d807db9..1b91b13509 100644 --- a/packages/minidb/package.json +++ b/packages/minidb/package.json @@ -46,5 +46,8 @@ "bench": "node --import tsx bench/bench.ts", "bench:cluster": "node --import tsx bench/cluster.ts", "clean": "rm -rf dist" + }, + "dependencies": { + "@moonshot-ai/kernel-file-lock": "workspace:^" } } diff --git a/packages/minidb/src/cluster/index.ts b/packages/minidb/src/cluster/index.ts index 9d0ff3f1fe..b95b2a8d4b 100644 --- a/packages/minidb/src/cluster/index.ts +++ b/packages/minidb/src/cluster/index.ts @@ -8,8 +8,8 @@ // - Each shard keeps minidb's single-writer model (its own db.lock), so // concurrency scales with the number of distinct shards being written. // - Writes route through a per-process writer pool (lock-pool.ts): cached -// shard writers hold their lock and renew its timestamp; acquisition -// retries live holders up to lockAcquireTimeoutMs. +// shard writers hold their kernel lock; acquisition retries contenders up +// to lockAcquireTimeoutMs. // - Reads never take write locks: they use the cached writer when this // process holds the shard, else a read-only MiniDb revalidated against // the shard files' fingerprint on every use. @@ -106,7 +106,6 @@ export class ClusterDb { valueMode: opts.valueMode, recovery: opts.recovery, }, - lockRenewMs: opts.lockRenewMs ?? 10_000, lockAcquireTimeoutMs: opts.lockAcquireTimeoutMs ?? 30_000, lockHoldMs: opts.lockHoldMs ?? 250, maxWriters: opts.lockPoolMaxShards ?? 16, diff --git a/packages/minidb/src/cluster/lock-pool.ts b/packages/minidb/src/cluster/lock-pool.ts index 236046362e..53adff346c 100644 --- a/packages/minidb/src/cluster/lock-pool.ts +++ b/packages/minidb/src/cluster/lock-pool.ts @@ -32,7 +32,6 @@ import { sleep } from './utils.js'; export interface LockPoolOptions { writerOpts: ShardOpenOptions; readerOpts: ShardOpenOptions; - lockRenewMs: number; lockAcquireTimeoutMs: number; lockHoldMs: number; maxWriters: number; @@ -212,7 +211,7 @@ export class ShardLockPool { let delay = 10; for (;;) { try { - const handle = await ShardHandle.openWriter(shardId, dir, this.opts.writerOpts, this.opts.lockRenewMs); + const handle = await ShardHandle.openWriter(shardId, dir, this.opts.writerOpts); this.stats.writerOpens++; try { await this.opts.applyDefs(handle.db); diff --git a/packages/minidb/src/cluster/shard.ts b/packages/minidb/src/cluster/shard.ts index bbd5ea7f14..c8f15909d4 100644 --- a/packages/minidb/src/cluster/shard.ts +++ b/packages/minidb/src/cluster/shard.ts @@ -1,7 +1,7 @@ // src/cluster/shard.ts // // A single shard: one MiniDb instance in either writer mode (holding the -// shard's db.lock, with a lease timer refreshing the lock timestamp) or +// shard's kernel-backed db.lock) or // reader mode (read-only, no lock, coexisting with another process's writer). import { MiniDb } from '../index.js'; @@ -11,8 +11,6 @@ import type { OpenOptions } from '../index.js'; export type ShardOpenOptions = Omit; export class ShardHandle { - private leaseTimer: NodeJS.Timeout | null = null; - private constructor( readonly shardId: number, readonly dir: string, @@ -20,25 +18,14 @@ export class ShardHandle { readonly writer: boolean, ) {} - /** Open the shard for writing: acquires db.lock via MiniDb.open and starts - * refreshing the lock timestamp every renewMs (0 disables renewal). Throws - * LockError if the lock is held by a live process. */ + /** Open the shard for writing and hold db.lock until close. */ static async openWriter( shardId: number, dir: string, opts: ShardOpenOptions, - renewMs: number, ): Promise { const db = await MiniDb.open({ ...opts, dir }); - const handle = new ShardHandle(shardId, dir, db as MiniDb, true); - if (renewMs > 0) { - handle.leaseTimer = setInterval(() => { - void db.renewLock().catch(() => {}); - }, renewMs); - // Never keep a worker process alive just for lease renewal. - handle.leaseTimer.unref(); - } - return handle; + return new ShardHandle(shardId, dir, db as MiniDb, true); } /** Open the shard read-only. Does not touch db.lock, never fsyncs, and @@ -56,10 +43,6 @@ export class ShardHandle { } async close(): Promise { - if (this.leaseTimer) { - clearInterval(this.leaseTimer); - this.leaseTimer = null; - } await this.db.close(); } } diff --git a/packages/minidb/src/cluster/types.ts b/packages/minidb/src/cluster/types.ts index 9c37716a00..5a252504aa 100644 --- a/packages/minidb/src/cluster/types.ts +++ b/packages/minidb/src/cluster/types.ts @@ -36,15 +36,6 @@ export interface ClusterOpenOptions { * throw; reads always go through revalidated read-only shard instances. */ readOnly?: boolean; - /** How long a lock timestamp may go unrefreshed before the lock would be - * considered abandoned (default 30000). Reserved: the underlying LockFile - * currently takes a lock over only when the recorded owner PID is dead, - * never merely because it is old; lockRenewMs keeps the timestamp fresh - * for observability and for a future lease-based check. */ - lockLeaseMs?: number; - /** How often a cached shard writer refreshes its lock timestamp - * (default 10000). Set to 0 to disable renewal. */ - lockRenewMs?: number; /** Maximum number of shard write locks cached in this process; least * recently used, non-busy shards are evicted beyond the cap (default 16). */ lockPoolMaxShards?: number; diff --git a/packages/minidb/src/index.ts b/packages/minidb/src/index.ts index a6ea101928..fd44fe92d6 100644 --- a/packages/minidb/src/index.ts +++ b/packages/minidb/src/index.ts @@ -375,7 +375,11 @@ export class MiniDb { db.valueReader?.close(); db.store?.close(); if (db.lock) { - await db.lock.release().catch(() => {}); + try { + db.lock.releaseSync(); + } catch { + // best-effort cleanup + } db.lock = null; } throw err; @@ -1605,13 +1609,6 @@ export class MiniDb { // ---- maintenance -------------------------------------------------------- - /** Refresh the write lock's timestamp (see {@link LockFile.renew}). No-op - * for a read-only instance. Exposed for lease-style holders such as the - * cluster shard pool, which renew on a timer to prove liveness. */ - async renewLock(): Promise { - await this.lock?.renew(); - } - /** Advanced/internal (read-replica owners such as the cluster shard pool): * incrementally apply WAL frames appended to db.wal after `offset` — the * same frames open-time recovery would replay, interpreted identically @@ -1654,7 +1651,7 @@ export class MiniDb { this.valueReader?.close(); await this.wal.close(); if (this.lock) { - await this.lock.release(); + this.lock.releaseSync(); this.lock = null; } } @@ -1664,5 +1661,6 @@ export class MiniDb { } private ensureWritable(): void { if (this.readOnly) throw new Error('MiniDb is open in read-only mode'); + this.lock?.assertHeld(); } } diff --git a/packages/minidb/src/lockfile.ts b/packages/minidb/src/lockfile.ts index 43727f1a41..e0afbb1700 100644 --- a/packages/minidb/src/lockfile.ts +++ b/packages/minidb/src/lockfile.ts @@ -1,293 +1,56 @@ -// src/lockfile.ts -// -// A small exclusive file lock using O_EXCL creation. Used to prevent two -// processes from opening the same database directory for writing (which would -// corrupt it). A lock is considered stale and is taken over only when the -// recorded owner PID is no longer alive — never merely because it is old. - -import fs from 'node:fs/promises'; -import fsSync from 'node:fs'; -import path from 'node:path'; -import { renameReplace } from './rename-replace.js'; +import { + type KernelFileLockHandle, + tryAcquireKernelFileLock, +} from '@moonshot-ai/kernel-file-lock'; export class LockError extends Error { readonly code = 'ELOCKED'; + constructor(message: string) { super(message); this.name = 'LockError'; } } -function pidAlive(pid: unknown): boolean { - if (!pid || typeof pid !== 'number') return false; - try { - process.kill(pid, 0); - return true; - } catch (e) { - return (e as NodeJS.ErrnoException).code === 'EPERM'; - } -} - -// Track held locks so we can release them on process exit as a safety net. -const HELD = new Set(); -// Distinct sidecar names per acquire attempt: two lock users in the same -// process (e.g. independent shard pools) must never share a tmp/bid/watch -// path, or one user's cleanup would delete the other's in-flight file. -let sidecarSeq = 0; -const nextSidecarSeq = (): number => ++sidecarSeq; -let exitHooked = false; -// Co-bidders all replace the stale corpse within the same wave (they woke on -// the same event); the settle pause before the winner claims the lock must -// outlast that wave so the last bidder to land is unambiguous. A fixed value -// (even a generous one) loses on shared CI runners that deschedule a bidder -// for hundreds of milliseconds inside its own atomic-op sequence, so the -// settle is ADAPTIVE: it scales with how long our own takeover attempt took -// (4x the wall clock of inspect+writeBid+rename — a stalled machine stalls -// every bidder), floored at 60ms and capped at 2s so a healthy takeover stays -// fast. Residual (bounded-delay, inherent to file-based takeover): a bidder -// whose writeBid is delayed past the winner's final verify can still -// double-win — the bid-file sweep at verification shrinks this window to -// "competitor had not even started writing their bid yet", which requires a -// full-attempt+settle-sized skew and is effectively a process-level pause. -const TAKEOVER_SETTLE_BASE_MS = 60; -const TAKEOVER_SETTLE_MAX_MS = 2_000; -function hookExit(): void { - if (exitHooked) return; - exitHooked = true; - process.on('beforeExit', () => { - for (const lock of HELD) lock.releaseSync(); - }); -} - export class LockFile { readonly path: string; held = false; + private handle: KernelFileLockHandle | undefined; + constructor(path: string) { this.path = path; } - /** Try to acquire the lock exactly once. Returns true when this call created - * the lock file, either directly or by winning a stale-lock takeover. Returns - * false whenever the lock was already held at attempt time — by a live owner - * or by a competing takeover. After observing a held lock this call never - * re-races: callers that want to wait retry acquire() at a higher level - * (see the cluster lock pool). */ async acquire(): Promise { - // Register a "watch" BEFORE touching the lock: every contender is visible - // to every other for its whole attempt, regardless of where the scheduler - // stalls it. (Settle-window heuristics alone could not survive a bidder - // descheduled before its bid write on a shard-parallel CI runner — see the - // takeover loop below; a stalled contender is only in the way, not - // invisible.) - const watch = `${this.path}.watch-${process.pid}-${nextSidecarSeq()}`; - await fs.writeFile(watch, JSON.stringify({ pid: process.pid, ts: Date.now() })); - try { - await this.reapDeadWatches(); - - if (await this.tryCreate()) return true; - - // The lock exists. Only a DEAD owner's lock may be taken over; everything - // else (a live owner, or a takeover bid made by another racer in the - // meantime) is respected. - const seen = await this.inspect(); - if (seen === null || seen.alive) return false; - - // Takeover via atomic bid-replace, NOT unlink-then-create. Unlinking a - // stale lock and then racing to re-create it left a window in which a - // loser could delete the winner's just-linked file, after which several - // processes all believed they held the lock. Rename atomically replaces - // the corpse with our bid. - // - // Windows cannot rename over a destination while ANY process holds it - // open (co-racers reading/stat'ing the corpse make the rename EPERM), so - // the rename is retried with jitter. Crucially, each retry re-inspects - // the corpse first: a blind retry loop could land our bid seconds late, - // OVERWRITING an already-verified winner's lock line and double-holding - // (exactly the failure this loop is careful not to reintroduce). - const bid = `${this.path}.bid-${process.pid}-${nextSidecarSeq()}`; - const attemptStart = Date.now(); - try { - await fs.writeFile(bid, JSON.stringify({ pid: process.pid, ts: Date.now() })); - for (let attempt = 0; ; attempt++) { - // The corpse must still be there and dead. A competitor who landed - // wins by being alive in the file now — back off instead of - // overwriting their lock. (Unconditional, not just win32: the same - // overwrite hazard exists on POSIX when a co-bidder is descheduled - // between its first inspect and its rename.) - const gate = await this.inspect(); - if (gate === null || gate.alive || gate.mine) { - await fs.unlink(bid).catch(() => {}); - return false; - } - try { - await fs.rename(bid, this.path); - break; - } catch (e) { - const code = (e as NodeJS.ErrnoException).code; - const epermRetryable = - code === 'EPERM' && process.platform === 'win32' && attempt < 50; - if (!epermRetryable) { - await fs.unlink(bid).catch(() => {}); - // EEXIST races another creator; a persistent EPERM (Windows - // retries exhausted) means some holder kept the path pinned — - // either way the corpse could not be displaced this round, so - // decline like a live lock and let callers retry higher up. - if (code === 'EEXIST' || code === 'EPERM') return false; - throw e; - } - await new Promise((r) => setTimeout(r, 20 + Math.floor(Math.random() * 30))); - } - } - } catch (e) { - await fs.unlink(bid).catch(() => {}); - throw e; - } - - // Adaptive settle: scale with how long our own attempt took (a stalled - // machine stalls every bidder), floored and capped (see the constants). - const elapsedMs = Date.now() - attemptStart; - let settleMs = Math.min(TAKEOVER_SETTLE_MAX_MS, Math.max(TAKEOVER_SETTLE_BASE_MS, elapsedMs * 4)); - for (;;) { - await new Promise((resolve) => setTimeout(resolve, settleMs)); - const cur = await this.inspect(); - if (cur === null || !cur.mine) return false; - // Any live foreign watch means a contender is still in flight (its - // registration precedes its whole attempt): wait for its loop to - // finish instead of claiming on stale evidence. This is the check - // that makes exactly-one a construction, not a timing bet. - if (!(await this.hasLiveForeignWatch())) break; - settleMs = Math.min(TAKEOVER_SETTLE_MAX_MS, settleMs * 2); - } - this.markHeld(); - return true; - } finally { - await fs.unlink(watch).catch(() => {}); - } - } - - /** Delete watch registrations whose owner pid is no longer alive. */ - private async reapDeadWatches(): Promise { - const dir = path.dirname(this.path); - const prefix = `${path.basename(this.path)}.watch-`; - for (const f of await fs.readdir(dir).catch(() => [] as string[])) { - if (!f.startsWith(prefix)) continue; - const pid = Number(f.slice(prefix.length).split('-')[0]); - if (Number.isInteger(pid) && pid !== process.pid && !pidAlive(pid)) { - await fs.unlink(path.join(dir, f)).catch(() => {}); - } - } + if (this.held) return true; + const handle = tryAcquireKernelFileLock(this.path); + if (handle === undefined) return false; + this.handle = handle; + this.held = true; + return true; } - /** True when any OTHER process's liveness watch exists (reaping dead ones on sight). */ - private async hasLiveForeignWatch(): Promise { - const dir = path.dirname(this.path); - const prefix = `${path.basename(this.path)}.watch-`; - for (const f of await fs.readdir(dir).catch(() => [] as string[])) { - if (!f.startsWith(prefix)) continue; - const pid = Number(f.slice(prefix.length).split('-')[0]); - if (!Number.isInteger(pid) || pid === process.pid) continue; - if (pidAlive(pid)) return true; - await fs.unlink(path.join(dir, f)).catch(() => {}); - } + checkHeld(): boolean { + if (!this.held || this.handle === undefined) return false; + if (this.handle.checkHeld()) return true; + this.markLost(); return false; } - /** Atomic create-if-absent publish: tmp write + hard link (EEXIST-safe). */ - private async tryCreate(): Promise { - const tmp = `${this.path}.tmp-${process.pid}-${nextSidecarSeq()}`; - try { - await fs.writeFile(tmp, JSON.stringify({ pid: process.pid, ts: Date.now() })); - await fs.link(tmp, this.path); - this.markHeld(); - return true; - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== 'EEXIST') throw e; - return false; - } finally { - await fs.unlink(tmp).catch(() => {}); - } - } - - /** Read the lock file and decide its state. null = the file vanished. */ - private async inspect(): Promise<{ ino: number | bigint; alive: boolean; mine: boolean } | null> { - let raw: string; - let st: { ino: number | bigint }; - try { - [raw, st] = await Promise.all([fs.readFile(this.path, 'utf8'), fs.stat(this.path)]); - } catch (e) { - if ((e as NodeJS.ErrnoException).code === 'ENOENT') return null; - throw e; - } - let pid: number | undefined; - try { - pid = (JSON.parse(raw) as { pid?: number }).pid; - } catch { - pid = undefined; // unparsable content looks abandoned, same as a dead PID - } - return { ino: st.ino, alive: pidAlive(pid), mine: pid === process.pid }; - } - - private inspectSync(): { ino: number | bigint; alive: boolean; mine: boolean } | null { - let raw: string; - let st: { ino: number | bigint }; - try { - raw = fsSync.readFileSync(this.path, 'utf8'); - st = fsSync.statSync(this.path); - } catch (e) { - if ((e as NodeJS.ErrnoException).code === 'ENOENT') return null; - throw e; - } - let pid: number | undefined; - try { - pid = (JSON.parse(raw) as { pid?: number }).pid; - } catch { - pid = undefined; - } - return { ino: st.ino, alive: pidAlive(pid), mine: pid === process.pid }; - } - - /** Refresh the lock timestamp (proves liveness to processes inspecting the - * lock file). No-op when the lock is not held. Uses write-tmp-then-rename - * so a crash mid-renew cannot leave a truncated, "stale-looking" lock file - * behind for a lock that is actually still owned. */ - async renew(): Promise { - if (!this.held) return; - const tmp = `${this.path}.tmp-${process.pid}-${nextSidecarSeq()}`; - await fs.writeFile(tmp, JSON.stringify({ pid: process.pid, ts: Date.now() })); - // Windows: replacing our own lock can still clash with a co-process's - // readFile/stat of it (EPERM) — the helper rides out such transients. - await renameReplace(tmp, this.path, { retries: 20 }); + assertHeld(): void { + if (!this.checkHeld()) throw new LockError(`database write lock was lost: ${this.path}`); } - private markHeld(): void { - this.held = true; - HELD.add(this); - hookExit(); - } - - async release(): Promise { + releaseSync(): void { if (!this.held) return; - // Unlink ONLY the file this instance actually owns. The content at this - // path may have been replaced since we acquired it (a supervisor re-plant a - // dead-man's marker, a concurrent takeover…), and deleting such a file - // would drop a lock that no longer belongs to us. - const cur = await this.inspect(); - if (cur?.mine) await fs.unlink(this.path).catch(() => {}); this.held = false; - HELD.delete(this); + const handle = this.handle; + this.handle = undefined; + handle?.release(); } - /** Best-effort sync release for the exit hook. */ - releaseSync(): void { - if (!this.held) return; - try { - const cur = this.inspectSync(); - if (cur?.mine) fsSync.unlinkSync(this.path); - } catch { - /* ignore */ - } - this.held = false; - HELD.delete(this); + private markLost(): void { + this.releaseSync(); } } diff --git a/packages/minidb/test/cluster/lock.test.ts b/packages/minidb/test/cluster/lock.test.ts index 9f24a797ec..02599961e6 100644 --- a/packages/minidb/test/cluster/lock.test.ts +++ b/packages/minidb/test/cluster/lock.test.ts @@ -1,17 +1,12 @@ // test/cluster/lock.test.js // -// Lock semantics inside one process: same-shard writer contention with -// acquire timeout, per-shard independence, read-only coexistence, lock lease -// renewal, and writer handoff after close. +// Lock semantics in one process: contention, per-shard independence, read-only coexistence, handoff. import { test } from 'vitest'; import assert from 'node:assert/strict'; -import fs from 'node:fs/promises'; -import path from 'node:path'; import { ClusterDb } from '../../src/cluster/index.js'; -import { shardDirName } from '../../src/cluster/utils.js'; import { tmpDir, rmrf } from '../e2e/helpers/tmp.js'; -import { keyOnShard, sleep } from './helpers.js'; +import { keyOnShard } from './helpers.js'; test('two writers contend on the same shard; loser times out with LockError', async () => { const dir = await tmpDir('minidb-cluster-'); @@ -91,26 +86,6 @@ test('read-only instance coexists with a live writer and sees its commits', asyn } }); -test('lock lease: db.lock timestamp advances while a writer is held', async () => { - const dir = await tmpDir('minidb-cluster-'); - try { - const db = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json', lockRenewMs: 80, lockHoldMs: 0 }); - const key = keyOnShard('lease', 1, 4); - await db.set(key, { v: 1 }); // grabs shard 1 and starts the lease timer - - const lockPath = path.join(dir, shardDirName(1, 4), 'db.lock'); - const read = async () => JSON.parse(await fs.readFile(lockPath, 'utf8')) as { pid: number; ts: number }; - const first = await read(); - assert.equal(first.pid, process.pid); - await sleep(300); - const second = await read(); - assert.ok(second.ts > first.ts, `timestamp renewed (${first.ts} -> ${second.ts})`); - await db.close(); - } finally { - await rmrf(dir); - } -}); - test('close releases every shard lock it holds', async () => { const dir = await tmpDir('minidb-cluster-'); try { diff --git a/packages/minidb/test/defense.test.ts b/packages/minidb/test/defense.test.ts index 69712fdccd..649f696dd3 100644 --- a/packages/minidb/test/defense.test.ts +++ b/packages/minidb/test/defense.test.ts @@ -12,7 +12,6 @@ import net from 'node:net'; import { WAL } from '../src/wal.js'; import { encodeFrame, decodeBatchOps, TYPE_SET } from '../src/codec.js'; import { MiniDb } from '../src/index.js'; -import { LockFile } from '../src/lockfile.js'; import { startServer } from '../src/server.js'; async function tmpDir() { @@ -113,46 +112,6 @@ test('WAL open and close are idempotent', async () => { await fs.rm(dir, { recursive: true, force: true }); }); -// --- LockFile stale / corrupt handling ------------------------------------- - -test('LockFile.acquire returns false when a live process holds the lock', async () => { - const dir = await tmpDir(); - const p = path.join(dir, 'db.lock'); - const a = new LockFile(p); - assert.equal(await a.acquire(), true); - const b = new LockFile(p); - assert.equal(await b.acquire(), false); // same PID is alive -> not stale - await a.release(); - await fs.rm(dir, { recursive: true, force: true }); -}); - -test('a corrupt lock file is treated as stale and taken over', async () => { - const dir = await tmpDir(); - await fs.writeFile(path.join(dir, 'db.lock'), 'not-json'); - const db = await MiniDb.open({ dir, valueCodec: 'string' }); - await db.set('a', '1'); - assert.equal(db.get('a'), '1'); - await db.close(); - await fs.rm(dir, { recursive: true, force: true }); -}); - -test('a lock file with a non-numeric pid is treated as stale', async () => { - const dir = await tmpDir(); - await fs.writeFile(path.join(dir, 'db.lock'), JSON.stringify({ pid: 'abc' })); - const db = await MiniDb.open({ dir, valueCodec: 'string' }); - assert.equal(db.readOnly, false); - await db.close(); - await fs.rm(dir, { recursive: true, force: true }); -}); - -test('LockFile release/releaseSync are no-ops when not held', async () => { - const dir = await tmpDir(); - const lock = new LockFile(path.join(dir, 'db.lock')); - await assert.doesNotReject(() => lock.release()); - assert.doesNotThrow(() => lock.releaseSync()); - await fs.rm(dir, { recursive: true, force: true }); -}); - // --- snapshot / compaction edge cases -------------------------------------- test('compacting an empty database produces an empty snapshot and keeps data intact', async () => { diff --git a/packages/minidb/test/e2e/helpers/lock-racer.ts b/packages/minidb/test/e2e/helpers/lock-racer.ts index a2e77989f5..648b8de11b 100644 --- a/packages/minidb/test/e2e/helpers/lock-racer.ts +++ b/packages/minidb/test/e2e/helpers/lock-racer.ts @@ -1,6 +1,6 @@ // test/e2e/helpers/lock-racer.ts // -// Child-side counterpart of the lock-takeover stress test. Sits in a loop; +// Child-side counterpart of the kernel-lock stress test. Sits in a loop; // for each round it waits for `go-` to appear in the gate directory, // then races to acquire the lock and prints "R <0|1>". @@ -35,12 +35,24 @@ for (let r = 0; r < rounds; r++) { const got = await lf.acquire(); process.stdout.write(`R${r} ${process.pid} ${got ? 1 : 0}\n`); if (got) { - // Hold the lock briefly so every loser resolves its acquire() while the - // winner still holds (an immediate release makes sequential re-acquisters - // indistinguishable from simultaneous holders). Sized generously for - // heavily loaded machines, where a racer can be descheduled for tens of - // milliseconds inside its own call. - await sleep(50); - await lf.release(); + // Hold until the parent has collected every racer's line for this round + // (it plants release- once all RACERS lines are in). No fixed hold + // duration survives per-round wake-up stagger: on a loaded runner a racer + // can be descheduled past any hold+release window and then acquire + // SEQUENTIALLY, truthfully reporting 1 — which the parent would misread + // as two simultaneous holders. The gate makes every late racer see a + // live, held lock and resolve as a loser deterministically. The timeout + // is only a safety net for a dead parent. + const releaseGate = `${gateDir}/release-${r}`; + for (let waited = 0; ; waited += 1) { + try { + fs.statSync(releaseGate); + break; + } catch { + if (waited >= 30_000) break; + await sleep(1); + } + } + lf.releaseSync(); } } diff --git a/packages/minidb/test/e2e/stress.test.ts b/packages/minidb/test/e2e/stress.test.ts index 7a77126f2a..14b42b957e 100644 --- a/packages/minidb/test/e2e/stress.test.ts +++ b/packages/minidb/test/e2e/stress.test.ts @@ -184,25 +184,18 @@ test( ); // =========================================================================== -// Bug evidence: stale-lock takeover admits multiple owners +// Kernel lock exclusivity under simultaneous acquisition // =========================================================================== -// LockFile.acquire unlinks a stale lock and retries blindly. When several -// processes race to take over a crashed owner's lock, the loser's unlink can -// delete the winner's fresh lock file (and ENOENT on readFile is also treated -// as "stale"), so many racers end up believing they hold the lock at once — -// i.e. several writers on one database directory. test( - 'stress: a stale lock must be taken over by exactly one process', + 'stress: a kernel lock admits exactly one process per acquisition wave', { timeout: 180_000 }, async () => { const dir = await tmpDir('minidb-stress-lock-'); const lockPath = path.join(dir, 'db.lock'); const RACER = path.join(__dirname, 'helpers', 'lock-racer.ts'); - const DEAD_PID = 2 ** 30 - 3; - // 6-way simultaneous takeover still exposes any cascade (the historical - // failure mode grants everyone the lock), while fitting the test's process - // budget on 2-core runners (every racer is a full node+tsx child). + // Six simultaneous contenders exercise the OS-backed exclusion path while + // fitting the test's process budget on two-core runners. const RACERS = 6; const ROUNDS = 25; @@ -233,7 +226,6 @@ test( await new Promise((res) => setTimeout(res, 5)); } for (let r = 0; r < ROUNDS; r++) { - await fs.writeFile(lockPath, JSON.stringify({ pid: DEAD_PID, ts: Date.now() })); await fs.writeFile(`${dir}/go-${r}`, '1'); const want = `R${r} `; for (;;) { @@ -241,6 +233,11 @@ test( if (lines.length >= RACERS) { const holders = lines.filter((l) => l.endsWith(' 1')); if (holders.length !== 1) violations.push(`round ${r}: ${holders.length} holders (${holders.join(', ')})`); + // Every racer has resolved this round: let the winner release. + // Without the gate a racer descheduled past the winner's hold + // could acquire sequentially afterwards and report a truthful + // second "1" (see lock-racer.ts). + await fs.writeFile(`${dir}/release-${r}`, '1'); break; } await new Promise((res) => setTimeout(res, 1)); @@ -842,4 +839,3 @@ test( } }, ); - diff --git a/packages/minidb/test/lock.test.ts b/packages/minidb/test/lock.test.ts index 3e5c22b83c..fb39da254a 100644 --- a/packages/minidb/test/lock.test.ts +++ b/packages/minidb/test/lock.test.ts @@ -1,16 +1,21 @@ -// test/lock.test.js -import { test } from 'vitest'; import assert from 'node:assert/strict'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; + +import { test } from 'vitest'; + import { MiniDb } from '../src/index.js'; -import { LockError } from '../src/lockfile.js'; +import { LockError, LockFile } from '../src/lockfile.js'; -async function tmpDir() { +async function tmpDir(): Promise { return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-lock-')); } +async function cleanup(dir: string): Promise { + await fs.rm(dir, { recursive: true, force: true }); +} + test('a second writer on the same dir is rejected with LockError', async () => { const dir = await tmpDir(); const db1 = await MiniDb.open({ dir, valueCodec: 'string' }); @@ -18,7 +23,7 @@ test('a second writer on the same dir is rejected with LockError', async () => { await assert.rejects(() => MiniDb.open({ dir, valueCodec: 'string' }), LockError); } finally { await db1.close(); - await fs.rm(dir, { recursive: true, force: true }); + await cleanup(dir); } }); @@ -31,7 +36,7 @@ test('lock is released on close, allowing another writer', async () => { const db2 = await MiniDb.open({ dir, valueCodec: 'string' }); assert.equal(db2.get('a'), '1'); await db2.close(); - await fs.rm(dir, { recursive: true, force: true }); + await cleanup(dir); }); test('readOnly open succeeds alongside a writer and rejects writes', async () => { @@ -46,7 +51,7 @@ test('readOnly open succeeds alongside a writer and rejects writes', async () => await ro.close(); } finally { await db1.close(); - await fs.rm(dir, { recursive: true, force: true }); + await cleanup(dir); } }); @@ -59,17 +64,31 @@ test("onLockFail: 'readonly' degrades instead of throwing", async () => { await db2.close(); } finally { await db1.close(); - await fs.rm(dir, { recursive: true, force: true }); + await cleanup(dir); } }); -test('a stale lock (dead PID) is taken over', async () => { +test('pre-existing sentinel contents do not imply ownership', async () => { const dir = await tmpDir(); - await fs.writeFile(path.join(dir, 'db.lock'), JSON.stringify({ pid: 999999, ts: Date.now() })); + const lockPath = path.join(dir, 'db.lock'); + await fs.writeFile(lockPath, JSON.stringify({ pid: process.pid, lock_id: 'legacy' })); + const db = await MiniDb.open({ dir, valueCodec: 'string' }); - assert.equal(db.readOnly, false); await db.set('a', '1'); assert.equal(db.get('a'), '1'); await db.close(); - await fs.rm(dir, { recursive: true, force: true }); + + assert.equal(await fs.readFile(lockPath, 'utf8'), JSON.stringify({ pid: process.pid, lock_id: 'legacy' })); + assert.equal((await fs.readdir(dir)).some((entry) => entry.includes('.stale.')), false); + await cleanup(dir); +}); + +test('releaseSync is idempotent', async () => { + const dir = await tmpDir(); + const lock = new LockFile(path.join(dir, 'db.lock')); + assert.doesNotThrow(() => lock.releaseSync()); + assert.equal(await lock.acquire(), true); + lock.releaseSync(); + assert.doesNotThrow(() => lock.releaseSync()); + await cleanup(dir); }); diff --git a/packages/minidb/test/review-fixes.test.ts b/packages/minidb/test/review-fixes.test.ts index 0b761bab35..7a4e559d1f 100644 --- a/packages/minidb/test/review-fixes.test.ts +++ b/packages/minidb/test/review-fixes.test.ts @@ -84,6 +84,23 @@ test('expired keys are removed from secondary indexes', async () => { } }); +test('editing the sentinel payload cannot create a successor generation', async () => { + const dir = await tmpDir(); + try { + const oldWriter = await MiniDb.open({ dir, valueCodec: 'string', autoCompact: false }); + await oldWriter.set('generation', 'old'); + + await fs.writeFile(path.join(dir, 'db.lock'), 'successor-generation'); + await assert.rejects(() => MiniDb.open({ dir, valueCodec: 'string', autoCompact: false }), /locked/); + await oldWriter.set('generation', 'old-again'); + assert.equal(oldWriter.get('generation'), 'old-again'); + + await oldWriter.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + test('expired keys are removed from the full-text index', async () => { const dir = await tmpDir(); try { diff --git a/packages/node-sdk/test/session-event-types.test.ts b/packages/node-sdk/test/session-event-types.test.ts index 61797a9e65..6874a047b6 100644 --- a/packages/node-sdk/test/session-event-types.test.ts +++ b/packages/node-sdk/test/session-event-types.test.ts @@ -68,6 +68,7 @@ describe('Event public types', () => { switch (event.type) { case 'agent.status.updated': case 'session.meta.updated': + case 'session.list_changed': case 'event.session.created': case 'event.session.status_changed': case 'event.session.work_changed': @@ -78,6 +79,7 @@ describe('Event public types', () => { case 'event.model_catalog.changed': case 'goal.updated': case 'skill.activated': + case 'skill_catalog.changed': case 'plugin_command.activated': case 'error': case 'warning': diff --git a/packages/protocol/src/__tests__/envelope.test.ts b/packages/protocol/src/__tests__/envelope.test.ts index cc72a134bb..1a14e98f4b 100644 --- a/packages/protocol/src/__tests__/envelope.test.ts +++ b/packages/protocol/src/__tests__/envelope.test.ts @@ -77,6 +77,28 @@ describe('envelope', () => { '{"code":50001,"msg":"boom","data":null,"request_id":"req_s"}', ); }); + + it('errEnvelope carries structured details and omits them when absent', () => { + const details = { kind: 'held-by-peer', phase: 'routable', address: 'http://127.0.0.1:58628' }; + const withDetails = errEnvelope( + ErrorCode.SESSION_HELD_BY_PEER, + 'session is owned by another instance', + 'req_d', + undefined, + details, + ); + expect(withDetails.details).toEqual(details); + expect(JSON.stringify(withDetails)).toBe( + '{"code":40921,"msg":"session is owned by another instance","data":null,"request_id":"req_d","details":{"kind":"held-by-peer","phase":"routable","address":"http://127.0.0.1:58628"}}', + ); + expect(envelopeSchema(z.any()).parse(withDetails).details).toEqual(details); + + // No details → field is absent and the wire shape is byte-identical to before. + const without = errEnvelope(ErrorCode.SESSION_HELD_BY_PEER, 'owned by peer', 'req_f'); + expect(JSON.stringify(without)).toBe( + '{"code":40921,"msg":"owned by peer","data":null,"request_id":"req_f"}', + ); + }); }); describe('error-codes', () => { @@ -85,6 +107,7 @@ describe('error-codes', () => { expect(ErrorCode.VALIDATION_FAILED).toBe(40001); expect(ErrorCode.SESSION_NOT_FOUND).toBe(40401); expect(ErrorCode.GOAL_UNSUPPORTED_AGENT).toBe(40920); + expect(ErrorCode.SESSION_HELD_BY_PEER).toBe(40921); expect(ErrorCode.APPROVAL_EXPIRED).toBe(41001); expect(ErrorCode.FS_WATCH_LIMIT_EXCEEDED).toBe(42902); expect(ErrorCode.INTERNAL_ERROR).toBe(50001); @@ -98,6 +121,7 @@ describe('error-codes', () => { expect(ErrorCodeReason[ErrorCode.VALIDATION_FAILED]).toBe('validation.failed'); expect(ErrorCodeReason[ErrorCode.FS_WATCH_LIMIT_EXCEEDED]).toBe('fs.watch_limit_exceeded'); expect(ErrorCodeReason[ErrorCode.GOAL_UNSUPPORTED_AGENT]).toBe('goal.unsupported_agent'); + expect(ErrorCodeReason[ErrorCode.SESSION_HELD_BY_PEER]).toBe('session.held_by_peer'); }); it('reserved codes are not redefined (40101, 50002 absent)', () => { diff --git a/packages/protocol/src/__tests__/session-ownership.test.ts b/packages/protocol/src/__tests__/session-ownership.test.ts new file mode 100644 index 0000000000..4d1bb09a03 --- /dev/null +++ b/packages/protocol/src/__tests__/session-ownership.test.ts @@ -0,0 +1,67 @@ +/** + * Scenario: session-ownership error details payloads (SESSION_HELD_BY_PEER). + * Responsibilities: verify the discriminated-union wire shape, phase enum, and + * retryhint validation. + * Wiring: pure protocol schemas; no external boundaries. + * Run: `pnpm --filter @moonshot-ai/protocol exec vitest run src/__tests__/session-ownership.test.ts`. + */ +import { describe, expect, it } from 'vitest'; + +import { + sessionOwnershipDetailsSchema, + sessionOwnershipPhaseSchema, + type SessionOwnershipDetails, +} from '../session-ownership'; + +describe('sessionOwnershipPhaseSchema', () => { + it('accepts every documented phase', () => { + for (const phase of ['creating', 'routable', 'held-by-local-instance']) { + expect(sessionOwnershipPhaseSchema.parse(phase)).toBe(phase); + } + }); + + it('rejects unknown phases', () => { + expect(sessionOwnershipPhaseSchema.safeParse('redirect').success).toBe(false); + expect(sessionOwnershipPhaseSchema.safeParse('').success).toBe(false); + }); +}); + +describe('sessionOwnershipDetailsSchema', () => { + it('parses a routable held-by-peer payload carrying an address', () => { + const payload: SessionOwnershipDetails = { + kind: 'held-by-peer', + phase: 'routable', + address: 'http://127.0.0.1:58628', + }; + expect(sessionOwnershipDetailsSchema.parse(payload)).toEqual(payload); + }); + + it('parses a retryable payload carrying retry_after_ms', () => { + const payload = { kind: 'held-by-peer', phase: 'creating', retry_after_ms: 1500 }; + expect(sessionOwnershipDetailsSchema.parse(payload)).toEqual(payload); + }); + + it('parses a bare held-by-peer payload with no optional hints', () => { + const payload = { kind: 'held-by-peer', phase: 'held-by-local-instance' }; + expect(sessionOwnershipDetailsSchema.parse(payload)).toEqual(payload); + }); + + it('rejects an unknown kind', () => { + expect(sessionOwnershipDetailsSchema.safeParse({ kind: 'stolen', phase: 'routable' }).success).toBe(false); + }); + + it('rejects a held-by-peer payload without phase', () => { + expect(sessionOwnershipDetailsSchema.safeParse({ kind: 'held-by-peer' }).success).toBe(false); + }); + + it('rejects a negative or fractional retry_after_ms', () => { + expect( + sessionOwnershipDetailsSchema.safeParse({ kind: 'held-by-peer', phase: 'creating', retry_after_ms: -1 }) + .success, + ).toBe(false); + expect( + sessionOwnershipDetailsSchema.safeParse({ kind: 'held-by-peer', phase: 'creating', retry_after_ms: 1.5 }) + .success, + ).toBe(false); + }); +}); diff --git a/packages/protocol/src/__tests__/session.test.ts b/packages/protocol/src/__tests__/session.test.ts index ef78b8f4bd..100d44f274 100644 --- a/packages/protocol/src/__tests__/session.test.ts +++ b/packages/protocol/src/__tests__/session.test.ts @@ -114,6 +114,22 @@ describe('sessionSchema', () => { const parsed = sessionSchema.parse(fullSession); expect(parsed.last_prompt).toBeUndefined(); }); + + it('accepts the optional ownership field and rejects unknown held_by', () => { + expect( + sessionSchema.parse({ ...fullSession, ownership: { held_by: 'self' } }).ownership, + ).toEqual({ held_by: 'self' }); + expect( + sessionSchema.parse({ + ...fullSession, + ownership: { held_by: 'peer', address: 'http://127.0.0.1:58627' }, + }).ownership, + ).toEqual({ held_by: 'peer', address: 'http://127.0.0.1:58627' }); + expect(sessionSchema.parse(fullSession).ownership).toBeUndefined(); + expect( + sessionSchema.safeParse({ ...fullSession, ownership: { held_by: 'unknown' } }).success, + ).toBe(false); + }); }); describe('sessionCreateSchema', () => { diff --git a/packages/protocol/src/envelope.ts b/packages/protocol/src/envelope.ts index 581087ddd3..56766662e7 100644 --- a/packages/protocol/src/envelope.ts +++ b/packages/protocol/src/envelope.ts @@ -29,12 +29,16 @@ export function okEnvelope(data: T, requestId: string): Envelope { * (or `undefined`) the field is absent and the wire shape stays byte-identical * to the original `{ code, msg, data: null, request_id }` — `JSON.stringify` * drops `undefined` properties, so callers that have no stack are unaffected. + * `details` behaves the same way: when provided it carries a structured, + * client-consumable payload (e.g. `SessionOwnershipDetails` under + * `SESSION_HELD_BY_PEER`); when omitted the field is absent from the wire. */ export function errEnvelope( code: number, msg: string, requestId: string, stack?: string, + details?: unknown, ): Envelope { - return { code, msg, data: null, request_id: requestId, stack }; + return { code, msg, data: null, request_id: requestId, stack, details }; } diff --git a/packages/protocol/src/error-codes.ts b/packages/protocol/src/error-codes.ts index 9c4eaccce1..f3c926ae28 100644 --- a/packages/protocol/src/error-codes.ts +++ b/packages/protocol/src/error-codes.ts @@ -99,6 +99,8 @@ export const ErrorCode = { FS_ALREADY_EXISTS: 40919, /** goal 只允许主 agent 使用 */ GOAL_UNSUPPORTED_AGENT: 40920, + /** session 的 lease 被其他实例持有；`details` 为 SessionOwnershipDetails */ + SESSION_HELD_BY_PEER: 40921, /** approval 60s 超时 */ APPROVAL_EXPIRED: 41001, @@ -194,6 +196,7 @@ export const ErrorCodeReason: Readonly> = { [ErrorCode.GOAL_OBJECTIVE_TOO_LONG]: 'goal.objective_too_long', [ErrorCode.FS_ALREADY_EXISTS]: 'fs.already_exists', [ErrorCode.GOAL_UNSUPPORTED_AGENT]: 'goal.unsupported_agent', + [ErrorCode.SESSION_HELD_BY_PEER]: 'session.held_by_peer', [ErrorCode.APPROVAL_EXPIRED]: 'approval.expired', [ErrorCode.QUESTION_EXPIRED]: 'question.expired', diff --git a/packages/protocol/src/events.ts b/packages/protocol/src/events.ts index 6f9178d145..7cedf42e78 100644 --- a/packages/protocol/src/events.ts +++ b/packages/protocol/src/events.ts @@ -235,6 +235,8 @@ export type KimiErrorCode = | 'session.approval_handler_error' | 'session.question_handler_error' | 'session.init_failed' + | 'session.held_by_peer' + | 'session.lease_lost' | 'agent.not_found' | 'turn.agent_busy' | 'goal.already_exists' @@ -312,6 +314,9 @@ export type KimiErrorCode = | 'os.fs.unknown' | 'os.process.spawn_failed' | 'os.process.kill_failed' + | 'os.lock.held' + | 'os.lock.wait_timeout' + | 'os.lock.io' | 'storage.not_found' | 'storage.decode_failed' | 'storage.corrupted' @@ -505,6 +510,32 @@ export interface SessionCreatedEvent { readonly session: Session; } +/** + * Volatile, payload-less hint that the set of sessions on disk changed + * (design §3.8): a workspace or session directory appeared or vanished + * under the shared `/sessions` tree, + * possibly created by ANOTHER server instance sharing the home. Clients + * should re-pull `GET /sessions` instead of reading anything into the event + * itself — it is fanned out live only (never journaled, never replayed). + */ +export interface SessionListChangedEvent { + readonly type: 'session.list_changed'; +} + +/** + * Volatile per-session hint that the session's skill catalog changed: a skill + * file or directory under one of the watched sources appeared, changed, or + * vanished (mirrors the core `ISessionSkillCatalog.onDidChange` feed, whose + * payload is the changed source id). Fanned out live only (never journaled, + * never replayed) to connections subscribed to that session — clients should + * re-pull `GET /sessions/{sid}/skills` instead of reading anything into the + * hint itself. + */ +export interface SkillCatalogChangedEvent { + readonly type: 'skill_catalog.changed'; + readonly sourceId: string; +} + export interface WorkspaceCreatedEvent { readonly type: 'event.workspace.created'; readonly workspace: Workspace; @@ -907,6 +938,8 @@ export type AgentEvent = | AgentStatusUpdatedEvent | SessionMetaUpdatedEvent | SessionCreatedEvent + | SessionListChangedEvent + | SkillCatalogChangedEvent | WorkspaceCreatedEvent | WorkspaceUpdatedEvent | WorkspaceDeletedEvent diff --git a/packages/protocol/src/fs.ts b/packages/protocol/src/fs.ts index 39e2c9a1be..b54f18fa99 100644 --- a/packages/protocol/src/fs.ts +++ b/packages/protocol/src/fs.ts @@ -79,6 +79,27 @@ export const fsChangeEntrySchema = z.object({ }); export type FsChangeEntry = z.infer; +/** + * Payload of the volatile `event.fs.changed` frame on `/api/v1/ws`: + * + * { type: 'event.fs.changed', seq, session_id, timestamp, payload: FsChangeEvent } + * + * The frames never enter the journal — the stream is volatile and there is no + * catch-up cursor. `seq` is **per-connection monotonic**: each WebSocket + * connection numbers only the frames actually delivered to it (matched change + * frames and `truncated` frames alike), starting at 1 with no gaps; a frame + * dispatched only to other connections does not advance yours. + * + * `truncated: true` (with `count`; `changes` empty) means the coalescing + * window overflowed server-side and that window's entries were dropped + * wholesale — the incremental stream after the previously received `seq` is + * no longer trustworthy. On `truncated`, on a `seq` gap, or on an epoch + * change, the client MUST rebuild its baseline with a full REST listing + * (`POST /sessions/{sid}/fs:list`) and resume applying increments on top; + * partial continuation after a truncation is not defined. Truncated frames + * are broadcast to every connection watching the session, regardless of each + * connection's path set. + */ export const fsChangeEventSchema = z.object({ changes: z.array(fsChangeEntrySchema), coalesced_window_ms: z.number().int().positive(), diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts index 16489267e7..1464498ef5 100644 --- a/packages/protocol/src/index.ts +++ b/packages/protocol/src/index.ts @@ -9,6 +9,7 @@ export * from './ws-control'; export * from './asyncapi'; export * from './session'; +export * from './session-ownership'; export * from './workspace'; export * from './message'; export * from './approval'; diff --git a/packages/protocol/src/rest/snapshot.ts b/packages/protocol/src/rest/snapshot.ts index 02278a69ef..2bac26268c 100644 --- a/packages/protocol/src/rest/snapshot.ts +++ b/packages/protocol/src/rest/snapshot.ts @@ -78,8 +78,13 @@ export type SnapshotSubagent = z.infer; export const sessionSnapshotResponseSchema = z.object({ /** Durable event watermark this snapshot is consistent with. */ as_of_seq: z.number().int().nonnegative(), - /** Journal epoch — pass back via the WS cursor for invalidation detection. */ - epoch: z.string().min(1), + /** + * Journal epoch — pass back via the WS cursor for invalidation detection. + * Absent when the journal has no baseline yet (no durable event written): + * "no baseline" is not "baseline changed" — treat the cursor as fresh, + * not as invalidated. + */ + epoch: z.string().min(1).optional(), session: sessionSchema, /** Most recent messages (chronological ascending), bounded page. */ messages: z.object({ diff --git a/packages/protocol/src/session-ownership.ts b/packages/protocol/src/session-ownership.ts new file mode 100644 index 0000000000..30ae0906eb --- /dev/null +++ b/packages/protocol/src/session-ownership.ts @@ -0,0 +1,37 @@ +/** + * Session-ownership error details — the structured `details` payload carried + * under `SESSION_HELD_BY_PEER` (40921) when an instance receives a request for + * a session whose lease is held by another instance. + * + * Wire semantics: the payload rides the REST envelope `details` field (HTTP + * 200 envelope; the business outcome lives in `code`) and the v2 WS `error` + * frame `details` field, verbatim in both directions. Clients branch on + * `kind`; within `held-by-peer` the `phase` tells the client what to do next: + * + * - creating kernel lock held before owner metadata is visible; retry shortly + * - routable holder is live and registered an address; client may redirect + * - held-by-local-instance holder has no address (local/embedded engine); terminal, do not retry + */ +import { z } from 'zod'; + +export const sessionOwnershipPhaseSchema = z.enum([ + 'creating', + 'routable', + 'held-by-local-instance', +]); +export type SessionOwnershipPhase = z.infer; + +export const heldByPeerDetailsSchema = z.object({ + kind: z.literal('held-by-peer'), + phase: sessionOwnershipPhaseSchema, + /** Present only when phase === 'routable'. */ + address: z.string().optional(), + /** Retry hint (ms) for `creating`. */ + retry_after_ms: z.number().int().nonnegative().optional(), +}); +export type HeldByPeerDetails = z.infer; + +export const sessionOwnershipDetailsSchema = z.discriminatedUnion('kind', [ + heldByPeerDetailsSchema, +]); +export type SessionOwnershipDetails = z.infer; diff --git a/packages/protocol/src/session.ts b/packages/protocol/src/session.ts index 8d10164e8c..c89ea698e5 100644 --- a/packages/protocol/src/session.ts +++ b/packages/protocol/src/session.ts @@ -78,6 +78,21 @@ export type SessionMetadata = z.infer; export const sessionPendingInteractionSchema = z.enum(['none', 'approval', 'question']); export type SessionPendingInteraction = z.infer; +/** + * Per-session holder annotation derived from the kernel lock on + * `session-leases/.lock` plus its optional `.owner.json` routing metadata. + * `self` = this instance holds the lock (the session is materialized here), + * `peer` = another instance holds it (`address` is its advertised base URL), + * `none` = no process holds the lock. Purely display/redirect metadata: the + * live kernel lock is the authority and is re-checked on materialization. + */ +export const sessionOwnershipSchema = z.object({ + held_by: z.enum(['self', 'peer', 'none']), + address: z.string().min(1).optional(), +}); + +export type SessionOwnership = z.infer; + export const sessionSchema = z.object({ id: z.string().min(1), workspace_id: workspaceIdSchema, @@ -102,6 +117,7 @@ export const sessionSchema = z.object({ * reason is cancelled/failed). */ last_turn_reason: z.enum(['completed', 'cancelled', 'failed']).optional(), archived: z.boolean().optional(), + ownership: sessionOwnershipSchema.optional(), current_prompt_id: z.string().min(1).optional(), /** Text of the most recent user prompt, for search/preview. Absent for empty sessions. */ last_prompt: z.string().optional(), diff --git a/packages/protocol/src/ws-control.ts b/packages/protocol/src/ws-control.ts index 19807c9f46..e29179e4ca 100644 --- a/packages/protocol/src/ws-control.ts +++ b/packages/protocol/src/ws-control.ts @@ -178,6 +178,13 @@ export const unsubscribeAckPayloadSchema = subscribeAckPayloadSchema; export const unsubscribeAckMessageSchema = wsAckEnvelopeSchema(unsubscribeAckPayloadSchema); +/** + * Filesystem watch (per connection): `watch_fs_add` / `watch_fs_remove` + * manage this connection's path set for a session; matching changes stream + * back as volatile `event.fs.changed` frames with a per-connection monotonic + * `seq`. See `fsChangeEventSchema` (`fs.ts`) for the full seq / truncated + * contract, including the client's rebuild-baseline obligation. + */ export const watchFsAddPayloadSchema = z.object({ session_id: z.string(), paths: z.array(z.string()), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 251932976a..7151c4e0b8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -73,6 +73,10 @@ importers: version: 4.1.4(@types/node@22.19.17)(@vitest/coverage-v8@4.1.4)(jsdom@25.0.1)(msw@2.15.0(@types/node@22.19.17)(typescript@6.0.2))(vite@8.0.8(@types/node@22.19.17)(esbuild@0.27.7)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.3)) apps/kimi-code: + dependencies: + fs-native-extensions: + specifier: 1.5.0 + version: 1.5.0 devDependencies: '@moonshot-ai/acp-adapter': specifier: workspace:^ @@ -83,6 +87,9 @@ importers: '@moonshot-ai/kap-server': specifier: workspace:^ version: link:../../packages/kap-server + '@moonshot-ai/kernel-file-lock': + specifier: workspace:^ + version: link:../../packages/kernel-file-lock '@moonshot-ai/kimi-code-oauth': specifier: workspace:^ version: link:../../packages/oauth @@ -683,6 +690,9 @@ importers: '@modelcontextprotocol/sdk': specifier: ^1.29.0 version: 1.29.0(zod@4.3.6) + '@moonshot-ai/kernel-file-lock': + specifier: workspace:^ + version: link:../kernel-file-lock '@moonshot-ai/kimi-code-oauth': specifier: workspace:^ version: link:../oauth @@ -870,6 +880,12 @@ importers: specifier: ^4.21.0 version: 4.21.0 + packages/kernel-file-lock: + dependencies: + fs-native-extensions: + specifier: 1.5.0 + version: 1.5.0 + packages/klient: dependencies: '@moonshot-ai/agent-core-v2': @@ -879,6 +895,9 @@ importers: specifier: 'catalog:' version: 4.3.6 devDependencies: + '@moonshot-ai/kap-server': + specifier: workspace:^ + version: link:../kap-server '@moonshot-ai/protocol': specifier: workspace:^ version: link:../protocol @@ -933,7 +952,11 @@ importers: specifier: workspace:^ version: link:../kaos - packages/minidb: {} + packages/minidb: + dependencies: + '@moonshot-ai/kernel-file-lock': + specifier: workspace:^ + version: link:../kernel-file-lock packages/node-sdk: dependencies: @@ -5071,6 +5094,25 @@ packages: resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} engines: {node: 18 || 20 || >=22} + bare-addon-resolve@1.10.1: + resolution: {integrity: sha512-F/SD2du8keuYSb4xipnGz5j2E6yhNdHA8ZVxtHae6h2uOrpBIjjbhXvjzKZbr5XUOzqBzh/i8GVFycj2DlFQIA==} + peerDependencies: + bare-url: '*' + peerDependenciesMeta: + bare-url: + optional: true + + bare-module-resolve@1.12.4: + resolution: {integrity: sha512-xcfgg2u7HqgJiBmah71O9vvdFAgHCvkqC/WSC2O7Bbgosoc1eC/BWe/6IDJ4OsfKlkxuvC/TDWXC+oH5yeW8mA==} + peerDependencies: + bare-url: '*' + peerDependenciesMeta: + bare-url: + optional: true + + bare-semver@1.1.0: + resolution: {integrity: sha512-1Hw5qJ7hXdVt3uPUqjeFTuxyvBUJauvz5A1I2jk8gzjZMHp04n//6nV9MDbG9CMw78JHY2lGV0w6s//LrASm2w==} + base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} @@ -6185,6 +6227,9 @@ packages: resolution: {integrity: sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==} engines: {node: '>=6 <7 || >=8'} + fs-native-extensions@1.5.0: + resolution: {integrity: sha512-nuZLFm9mGCxvyi7Llww/J4OyifKCS21nEUTAmnlTZp3FObPOvA32aCedCmt4Z+8yk+caqfClNaSCfn/P7T7FLQ==} + fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} @@ -8384,6 +8429,10 @@ packages: remark@15.0.1: resolution: {integrity: sha512-Eht5w30ruCXgFmxVUSlNWQ9iiimq07URKeFS3hNc8cUWy1llX4KDWfyEDZRycMc+znsN9Ux5/tJ/BFdgdOwA3A==} + require-addon@1.2.0: + resolution: {integrity: sha512-VNPDZlYgIYQwWp9jMTzljx+k0ZtatKlcvOhktZ/anNPI3dQ9NXk7cq2U4iJ1wd9IrytRnYhyEocFWbkdPb+MYA==} + engines: {bare: '>=1.10.0'} + require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} @@ -9746,6 +9795,9 @@ packages: resolution: {integrity: sha512-K4jVyjnBdgvc86Y6BkaLZEN933SwYOuBFkdmBu9ZfkcAbdVbpITnDmjvZ/aQjRXQrv5EPkTnD1s39GiiqbngCw==} engines: {node: '>= 0.4'} + which-runtime@1.4.0: + resolution: {integrity: sha512-0ugbP4CJW4e2D20jvEcC4973dCgIaHI4Rw1PT+26U9zEve7FyYdWAIwUnoeOYvoCfn+wXHoHTKb1KhkYlb60Pw==} + which-typed-array@1.1.20: resolution: {integrity: sha512-LYfpUkmqwl0h9A2HL09Mms427Q1RZWuOHsukfVcKRq9q95iQxdw0ix1JQrqbcDR9PH1QDwf5Qo8OZb5lksZ8Xg==} engines: {node: '>= 0.4'} @@ -14141,6 +14193,17 @@ snapshots: balanced-match@4.0.4: {} + bare-addon-resolve@1.10.1: + dependencies: + bare-module-resolve: 1.12.4 + bare-semver: 1.1.0 + + bare-module-resolve@1.12.4: + dependencies: + bare-semver: 1.1.0 + + bare-semver@1.1.0: {} + base64-js@1.5.1: {} baseline-browser-mapping@2.10.20: {} @@ -15430,6 +15493,13 @@ snapshots: jsonfile: 4.0.0 universalify: 0.1.2 + fs-native-extensions@1.5.0: + dependencies: + require-addon: 1.2.0 + which-runtime: 1.4.0 + transitivePeerDependencies: + - bare-url + fs.realpath@1.0.0: {} fsevents@2.3.3: @@ -18101,6 +18171,12 @@ snapshots: transitivePeerDependencies: - supports-color + require-addon@1.2.0: + dependencies: + bare-addon-resolve: 1.10.1 + transitivePeerDependencies: + - bare-url + require-directory@2.1.1: {} require-from-string@2.0.2: {} @@ -19596,6 +19672,8 @@ snapshots: is-weakmap: 2.0.2 is-weakset: 2.0.4 + which-runtime@1.4.0: {} + which-typed-array@1.1.20: dependencies: available-typed-arrays: 1.0.7