diff --git a/docs/adr/0021-host-simlock-managed-device-allocation.md b/docs/adr/0021-host-simlock-managed-device-allocation.md index e5f088eca..e0055790a 100644 --- a/docs/adr/0021-host-simlock-managed-device-allocation.md +++ b/docs/adr/0021-host-simlock-managed-device-allocation.md @@ -110,10 +110,12 @@ authorization and attribution around the same boundary. Plain local device selec Before acquisition, agent-device durably records a non-authoritative allocation operation: the logical requester, idempotency key, immutable shape request, deadline, and Host attribution when -applicable. After Simlock responds, it records the allocator handle/outcome and whether Host -published or cleaned it. This journal exists only to recover the Host-to-Simlock handoff. It never -mirrors Simlock's queue, provisioning, lease, cleanup, health, or capacity states, and it never -decides whether a device is reusable. +applicable. After Simlock responds, it records the allocator handle/outcome. Before invoking an +external Host binding publisher, it durably records a pending publication; publication success is +then recorded separately, and recovery conservatively cleans a pending or uncertain binding before +releasing the allocator lease. This journal exists only to recover the Host-to-Simlock handoff. It +never mirrors Simlock's queue, provisioning, lease, cleanup, health, or capacity states, and it +never decides whether a device is reusable. Each logical requester is a restart-stable allocation lane; concurrent leases use distinct lanes. Replaying the same attempt key returns the same durable outcome, including a refusal. Disconnect, diff --git a/packages/capture-kit/src/index.ts b/packages/capture-kit/src/index.ts index 0eb9a8645..a6a9f785e 100644 --- a/packages/capture-kit/src/index.ts +++ b/packages/capture-kit/src/index.ts @@ -16,6 +16,7 @@ export { decodeDurableDescriptor } from './durable-descriptor-codec.ts'; export { createScreenRecordingLiveHandle } from './screen-recording-live-handle.ts'; export { createScreenRecordingCompletion } from './screen-recording-completion.ts'; export { assertScreenRecordingOptionsSupported } from './screen-recording-options.ts'; +export { freezeJsonObject, isBoundedJsonObject } from './durable-json.ts'; export { cleanupManagedAppLogProcess, reattachCleanupOnlyAppLogProcess, diff --git a/packages/host-kit/src/file.ts b/packages/host-kit/src/file.ts index 55a20c5b7..ce6024283 100644 --- a/packages/host-kit/src/file.ts +++ b/packages/host-kit/src/file.ts @@ -1,7 +1,8 @@ export { isAtomicPublishTemporaryPath, + publishDurableFileSync, publishFileSync, - withAtomicPublishTempPathSync, + type DurableFilePublishMode, } from './internal/atomic-file.ts'; export { lstatIfPresent, diff --git a/packages/host-kit/src/internal/atomic-file.ts b/packages/host-kit/src/internal/atomic-file.ts index 3b2ecd097..1d91a615e 100644 --- a/packages/host-kit/src/internal/atomic-file.ts +++ b/packages/host-kit/src/internal/atomic-file.ts @@ -35,6 +35,52 @@ export function publishFileSync(options: { }); } +export type DurableFilePublishMode = 'replace' | 'link-exclusive'; + +/** Publishes complete UTF-8 contents with a durable file and directory fence. */ +export function publishDurableFileSync(options: { + destination: string; + contents: string; + mode?: number; + publish?: DurableFilePublishMode; +}): void { + const directory = path.dirname(options.destination); + withAtomicPublishTempPathSync(options.destination, (temporaryPath) => { + let descriptor: number | undefined; + let failed = false; + let primaryError: unknown; + try { + assertSafeDestination(options.destination); + descriptor = fs.openSync(temporaryPath, 'wx', options.mode ?? 0o600); + fs.writeFileSync(descriptor, options.contents, 'utf8'); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = undefined; + assertSafeDestination(options.destination); + if (options.publish === 'link-exclusive') { + fs.linkSync(temporaryPath, options.destination); + } else { + fs.renameSync(temporaryPath, options.destination); + } + syncDirectoryBestEffort(directory); + } catch (error) { + failed = true; + primaryError = error; + } + if (descriptor !== undefined) { + try { + fs.closeSync(descriptor); + } catch (error) { + if (!failed) { + failed = true; + primaryError = error; + } + } + } + if (failed) throw primaryError; + }); +} + /** * Gives a specialized durable publisher a canonical temp path and cleanup * ownership while it performs its own open/fsync/safety protocol. @@ -63,6 +109,22 @@ export function withAtomicPublishTempPathSync( } } +/** Syncs a containing directory when the host filesystem supports directory fsync. */ +function syncDirectoryBestEffort(directory: string): void { + let descriptor: number | undefined; + try { + descriptor = fs.openSync(directory, 'r'); + fs.fsyncSync(descriptor); + } catch { + } finally { + if (descriptor !== undefined) { + try { + fs.closeSync(descriptor); + } catch {} + } + } +} + /** Returns the canonical same-directory temp path used by atomic publishers. */ function createAtomicPublishTempPath(destination: string): string { return path.join( @@ -78,3 +140,24 @@ export function isAtomicPublishTemporaryPath(value: unknown, destination: string const name = path.basename(value); return name.startsWith(`.${path.basename(destination)}.`) && name.endsWith('.tmp'); } + +function assertSafeDestination(destination: string): void { + const stats = lstatIfPresent(destination); + if (stats?.isSymbolicLink()) { + throw new Error(`Refusing to replace a durable file symbolic link: ${destination}`); + } + if (stats && !stats.isFile()) { + throw new Error( + `Refusing to replace a durable path that is not a regular file: ${destination}`, + ); + } +} + +function lstatIfPresent(pathname: string): fs.Stats | undefined { + try { + return fs.lstatSync(pathname); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } +} diff --git a/packages/host-kit/src/internal/durable-file.test.ts b/packages/host-kit/src/internal/durable-file.test.ts new file mode 100644 index 000000000..778bb46b4 --- /dev/null +++ b/packages/host-kit/src/internal/durable-file.test.ts @@ -0,0 +1,122 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, expect, test, vi } from 'vitest'; +import { isAtomicPublishTemporaryPath, publishDurableFileSync } from './atomic-file.ts'; + +const roots: string[] = []; + +afterEach(() => { + vi.restoreAllMocks(); + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +test('fsyncs complete contents before publication and uses the requested mode', () => { + const root = fixtureRoot('ordering'); + const destination = path.join(root, 'record.json'); + const events: string[] = []; + const realFsync = fs.fsyncSync; + const realRename = fs.renameSync; + vi.spyOn(fs, 'fsyncSync').mockImplementation((descriptor) => { + events.push('fsync'); + return realFsync(descriptor); + }); + vi.spyOn(fs, 'renameSync').mockImplementation((source, target) => { + events.push('publish'); + return realRename(source, target); + }); + + publishDurableFileSync({ destination, contents: 'durable\n', mode: 0o640 }); + + expect(fs.readFileSync(destination, 'utf8')).toBe('durable\n'); + expect(fs.statSync(destination).mode & 0o777).toBe(0o640); + expect(events.indexOf('fsync')).toBeGreaterThanOrEqual(0); + expect(events.indexOf('fsync')).toBeLessThan(events.indexOf('publish')); + expect(temporaryPaths(root, destination)).toEqual([]); +}); + +test.each(['symbolic link', 'non-regular path'] as const)( + 'refuses a final %s and removes the temporary file', + (kind) => { + const root = fixtureRoot(kind); + const destination = path.join(root, 'record.json'); + if (kind === 'symbolic link') { + const outside = path.join(root, 'outside.json'); + fs.writeFileSync(outside, 'outside'); + fs.symlinkSync(outside, destination); + } else { + fs.mkdirSync(destination); + } + + expect(() => publishDurableFileSync({ destination, contents: 'replacement' })).toThrow( + kind === 'symbolic link' ? /symbolic link/ : /not a regular file/, + ); + expect(temporaryPaths(root, destination)).toEqual([]); + }, +); + +test('keeps an existing destination on link-exclusive publication failure', () => { + const root = fixtureRoot('exclusive'); + const destination = path.join(root, 'record.json'); + fs.writeFileSync(destination, 'original'); + + expect(() => + publishDurableFileSync({ + destination, + contents: 'replacement', + publish: 'link-exclusive', + }), + ).toThrow(/EEXIST/); + expect(fs.readFileSync(destination, 'utf8')).toBe('original'); + expect(temporaryPaths(root, destination)).toEqual([]); +}); + +test('preserves the publication error while cleaning the temporary file', () => { + const root = fixtureRoot('publish-error'); + const destination = path.join(root, 'record.json'); + const primary = new Error('publication failed'); + vi.spyOn(fs, 'renameSync').mockImplementation(() => { + throw primary; + }); + + assert.throws( + () => publishDurableFileSync({ destination, contents: 'durable' }), + (error: unknown) => error === primary, + ); + expect(temporaryPaths(root, destination)).toEqual([]); +}); + +test('preserves a file fsync error when descriptor cleanup also fails', () => { + const root = fixtureRoot('close-error'); + const destination = path.join(root, 'record.json'); + const primary = new Error('file fsync failed'); + const secondary = new Error('descriptor close failed'); + const realClose = fs.closeSync; + vi.spyOn(fs, 'fsyncSync').mockImplementation(() => { + throw primary; + }); + vi.spyOn(fs, 'closeSync').mockImplementation((descriptor) => { + realClose(descriptor); + throw secondary; + }); + + assert.throws( + () => publishDurableFileSync({ destination, contents: 'durable' }), + (error: unknown) => error === primary, + ); + expect(temporaryPaths(root, destination)).toEqual([]); +}); + +function fixtureRoot(label: string): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), `agent-device-durable-file-${label}-`)); + roots.push(root); + return root; +} + +function temporaryPaths(root: string, destination: string): string[] { + return fs + .readdirSync(root) + .map((name) => path.join(root, name)) + .filter((pathname) => isAtomicPublishTemporaryPath(pathname, destination)); +} diff --git a/src/daemon/__tests__/atomic-publish-ownership.test.ts b/src/daemon/__tests__/atomic-publish-ownership.test.ts index a5af19cce..2f0dc2c36 100644 --- a/src/daemon/__tests__/atomic-publish-ownership.test.ts +++ b/src/daemon/__tests__/atomic-publish-ownership.test.ts @@ -22,14 +22,18 @@ test('simple same-directory publishers use the shared atomic publish owner', () } }); -test('durable capture publication keeps its specialized fsync and destination checks', () => { - const source = fs.readFileSync( +test('durable publishers share the host-kit durable publication owner', () => { + const sourcePaths = [ new URL('../durable-capture-resource-store.ts', import.meta.url), - 'utf8', - ); - assert.match(source, /withAtomicPublishTempPathSync/); - assert.match(source, /fs\.openSync\([^\n]+['"]wx['"]/); - assert.match(source, /fs\.fsyncSync/); - assert.match(source, /fs\.renameSync/); - assert.match(source, /assertSafeDestination/); + new URL('../managed-device-allocation/store-filesystem.ts', import.meta.url), + ]; + for (const sourcePath of sourcePaths) { + const source = fs.readFileSync(sourcePath, 'utf8'); + assert.match(source, /publishDurableFileSync/); + assert.doesNotMatch( + source, + /fs\.(?:openSync|writeFileSync|fsyncSync|renameSync|linkSync)\s*\(/, + ); + assert.doesNotMatch(source, /assertSafeDestination/); + } }); diff --git a/src/daemon/__tests__/config.test.ts b/src/daemon/__tests__/config.test.ts index a0cffdce9..5278e6fb1 100644 --- a/src/daemon/__tests__/config.test.ts +++ b/src/daemon/__tests__/config.test.ts @@ -10,6 +10,7 @@ test('resolveDaemonPaths keeps explicit state directories authoritative', () => try { const paths = resolveDaemonPaths('~/custom-daemon', { env: { HOME: home } }); assert.equal(paths.baseDir, path.join(home, 'custom-daemon')); + assert.equal(paths.allocationsDir, path.join(home, 'custom-daemon', 'allocations')); } finally { fs.rmSync(home, { recursive: true, force: true }); } diff --git a/src/daemon/client/__tests__/boundary-fault-acceptance.test.ts b/src/daemon/client/__tests__/boundary-fault-acceptance.test.ts index 417607b33..943b1e54f 100644 --- a/src/daemon/client/__tests__/boundary-fault-acceptance.test.ts +++ b/src/daemon/client/__tests__/boundary-fault-acceptance.test.ts @@ -132,6 +132,7 @@ function daemonPaths(baseDir: string): DaemonPaths { infoPath: path.join(baseDir, 'daemon.json'), lockPath: path.join(baseDir, 'daemon.lock'), logPath: path.join(baseDir, 'daemon.log'), + allocationsDir: path.join(baseDir, 'allocations'), sessionsDir: path.join(baseDir, 'sessions'), }; } diff --git a/src/daemon/client/__tests__/boundary-fault-transport.test.ts b/src/daemon/client/__tests__/boundary-fault-transport.test.ts index a0db14ad4..867413af0 100644 --- a/src/daemon/client/__tests__/boundary-fault-transport.test.ts +++ b/src/daemon/client/__tests__/boundary-fault-transport.test.ts @@ -251,6 +251,7 @@ function daemonPaths(): DaemonPaths { infoPath: path.join(baseDir, 'daemon.json'), lockPath: path.join(baseDir, 'daemon.lock'), logPath: path.join(baseDir, 'daemon.log'), + allocationsDir: path.join(baseDir, 'allocations'), sessionsDir: path.join(baseDir, 'sessions'), }; } diff --git a/src/daemon/client/__tests__/daemon-client-timeout-route.test.ts b/src/daemon/client/__tests__/daemon-client-timeout-route.test.ts index f6b574acf..af956466d 100644 --- a/src/daemon/client/__tests__/daemon-client-timeout-route.test.ts +++ b/src/daemon/client/__tests__/daemon-client-timeout-route.test.ts @@ -59,6 +59,7 @@ function dummyStatePaths(): DaemonPaths { infoPath: path.join(baseDir, 'daemon.json'), lockPath: path.join(baseDir, 'daemon.lock'), logPath: path.join(baseDir, 'daemon.log'), + allocationsDir: path.join(baseDir, 'allocations'), sessionsDir: path.join(baseDir, 'sessions'), }; } diff --git a/src/daemon/config.ts b/src/daemon/config.ts index 63a1ceb32..e23b5bdf6 100644 --- a/src/daemon/config.ts +++ b/src/daemon/config.ts @@ -17,6 +17,7 @@ export type DaemonPaths = { infoPath: string; lockPath: string; logPath: string; + allocationsDir: string; sessionsDir: string; }; @@ -35,6 +36,7 @@ export function resolveDaemonPaths( infoPath: path.join(baseDir, 'daemon.json'), lockPath: path.join(baseDir, 'daemon.lock'), logPath: path.join(baseDir, 'daemon.log'), + allocationsDir: path.join(baseDir, 'allocations'), sessionsDir: path.join(baseDir, 'sessions'), }; } diff --git a/src/daemon/durable-capture-resource-store.ts b/src/daemon/durable-capture-resource-store.ts index 0a145b53e..13968e2a6 100644 --- a/src/daemon/durable-capture-resource-store.ts +++ b/src/daemon/durable-capture-resource-store.ts @@ -5,10 +5,7 @@ import type { DurableResourceEnvelope, } from '@agent-device/contracts/durable-resource-envelope'; import { decodeDurableResourceEnvelope } from '@agent-device/capture-kit'; -import { - openVerifiedFileForRead, - withAtomicPublishTempPathSync, -} from '@agent-device/host-kit/file'; +import { openVerifiedFileForRead, publishDurableFileSync } from '@agent-device/host-kit/file'; export type DurableCaptureResourceRecord = | Readonly<{ status: 'missing' }> @@ -60,21 +57,9 @@ export function createDurableCaptureResourceStore( write(resourcePath: string, envelope: DurableResourceEnvelope): void { const directory = path.dirname(resourcePath); fs.mkdirSync(directory, { recursive: true }); - withAtomicPublishTempPathSync(resourcePath, (temporaryPath) => { - let descriptor: number | undefined; - try { - assertSafeDestination(resourcePath, options.displayName); - descriptor = fs.openSync(temporaryPath, 'wx', 0o600); - fs.writeFileSync(descriptor, `${JSON.stringify(envelope)}\n`, 'utf8'); - fs.fsyncSync(descriptor); - fs.closeSync(descriptor); - descriptor = undefined; - assertSafeDestination(resourcePath, options.displayName); - fs.renameSync(temporaryPath, resourcePath); - syncDirectoryBestEffort(directory); - } finally { - if (descriptor !== undefined) fs.closeSync(descriptor); - } + publishDurableFileSync({ + destination: resourcePath, + contents: `${JSON.stringify(envelope)}\n`, }); }, list(sessionsDir: string): string[] { @@ -112,24 +97,6 @@ function assertFileName(fileName: string): void { } } -function assertSafeDestination(resourcePath: string, displayName: string): void { - let destination: fs.Stats; - try { - destination = fs.lstatSync(resourcePath); - } catch (error) { - if (isMissingFile(error)) return; - throw error; - } - if (destination.isSymbolicLink()) { - throw new Error(`Refusing to replace a ${displayName.toLowerCase()} resource symbolic link`); - } - if (!destination.isFile()) { - throw new Error( - `Refusing to replace a ${displayName.toLowerCase()} resource that is not a regular file`, - ); - } -} - function pathEntryExistsWithoutFollowing(resourcePath: string): boolean { try { fs.lstatSync(resourcePath); @@ -143,18 +110,6 @@ function invalidRecord(message: string): DurableCaptureResourc return { status: 'unreattachable', reason: 'descriptor-invalid', message }; } -function syncDirectoryBestEffort(directory: string): void { - let descriptor: number | undefined; - try { - descriptor = fs.openSync(directory, 'r'); - fs.fsyncSync(descriptor); - } catch { - // Atomic rename is the correctness boundary; directory fsync support varies by filesystem. - } finally { - if (descriptor !== undefined) fs.closeSync(descriptor); - } -} - function isMissingFile(error: unknown): boolean { return ( error !== null && diff --git a/src/daemon/managed-device-allocation/__tests__/decision.test.ts b/src/daemon/managed-device-allocation/__tests__/decision.test.ts new file mode 100644 index 000000000..e7879f3b0 --- /dev/null +++ b/src/daemon/managed-device-allocation/__tests__/decision.test.ts @@ -0,0 +1,90 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { newAllocationOperation } from '../record-factory.ts'; +import { applyAllocationTransition } from '../transitions.ts'; +import { decideAllocationAction } from '../decision.ts'; +import { ALLOCATION_LEASE, ALLOCATION_REQUEST } from './fixtures.ts'; + +const NOW = 1_700_000_000_000; + +function requested() { + return newAllocationOperation({ + ...ALLOCATION_REQUEST, + allocatorInstanceId: 'allocator-1', + nowMs: NOW, + }); +} + +function granted() { + const pending = applyAllocationTransition(requested(), { kind: 'request-dispatched' }, NOW + 1); + assert.equal(pending.status, 'applied'); + const result = applyAllocationTransition( + pending.record, + { + kind: 'allocator-outcome', + outcome: { + status: 'granted', + lease: ALLOCATION_LEASE, + identityIncarnationId: 'incarnation-1', + }, + }, + NOW + 2, + ); + assert.equal(result.status, 'applied'); + return result.record; +} + +test('new work may request once, while recovery of intent only looks up the allocator', () => { + assert.equal(decideAllocationAction(requested(), 'new').kind, 'request'); + assert.equal(decideAllocationAction(requested(), 'recover').kind, 'lookup'); + const pending = applyAllocationTransition(requested(), { kind: 'request-dispatched' }, NOW + 1); + assert.equal(pending.status, 'applied'); + assert.equal(decideAllocationAction(pending.record, 'recover').kind, 'lookup'); +}); + +test('an allocator-unknown record never gets an implicit second request', () => { + const record = applyAllocationTransition( + requested(), + { kind: 'allocator-unknown', message: 'response lost' }, + NOW + 1, + ); + assert.equal(record.status, 'applied'); + const action = decideAllocationAction(record.record, 'recover'); + assert.deepEqual(action, { + kind: 'lookup', + ref: { requesterId: 'requester-a', attemptKey: 'attempt-1' }, + }); +}); + +test('grants publish first, and release cleans the binding before calling the allocator', () => { + const record = granted(); + assert.equal(decideAllocationAction(record, 'continue').kind, 'publish'); + const publishPending = applyAllocationTransition( + record, + { kind: 'binding-publish-pending' }, + NOW + 2, + ); + assert.equal(publishPending.status, 'applied'); + assert.equal(decideAllocationAction(publishPending.record, 'continue').kind, 'blocked'); + assert.equal(decideAllocationAction(publishPending.record, 'recover').kind, 'cleanup'); + const published = applyAllocationTransition( + publishPending.record, + { kind: 'binding-published' }, + NOW + 3, + ); + assert.equal(published.status, 'applied'); + const cleanup = decideAllocationAction(published.record, 'release'); + assert.equal(cleanup.kind, 'cleanup'); + const cleaned = applyAllocationTransition(published.record, { kind: 'binding-cleaned' }, NOW + 4); + assert.equal(cleaned.status, 'applied'); + assert.deepEqual(decideAllocationAction(cleaned.record, 'release'), { + kind: 'release', + leaseId: 'lease-1', + }); +}); + +test('cancel and explicit supersession are allocator actions, not local terminal guesses', () => { + assert.equal(decideAllocationAction(requested(), 'cancel').kind, 'cancel'); + assert.equal(decideAllocationAction(requested(), 'supersede').kind, 'supersede'); + assert.equal(decideAllocationAction(granted(), 'cancel').kind, 'blocked'); +}); diff --git a/src/daemon/managed-device-allocation/__tests__/fixtures.ts b/src/daemon/managed-device-allocation/__tests__/fixtures.ts new file mode 100644 index 000000000..0e0ebf5db --- /dev/null +++ b/src/daemon/managed-device-allocation/__tests__/fixtures.ts @@ -0,0 +1,56 @@ +import type { + LeaseRequestInput, + LeaseRequestStatus, + ManagedIdentityRef, + ManagedLease, + ManagedShapeRequest, +} from '@agent-device/contracts/managed-device-allocation'; + +const ALLOCATION_SHAPE: ManagedShapeRequest = { + platform: 'ios', + deviceType: 'iPhone 16', + osVersion: '18.2', +}; + +const ALLOCATION_IDENTITY: ManagedIdentityRef = { + deviceId: 'device-1', + identityIncarnationId: 'incarnation-1', +}; + +export const ALLOCATION_LEASE: ManagedLease = { + id: 'lease-1', + ttlDeadline: 1_700_000_900_000, + device: { address: ALLOCATION_IDENTITY.deviceId }, + environment: { SIMLOCK_IOS_DEVICE_SET: '/managed/set' }, +}; + +export const ALLOCATION_REQUEST: LeaseRequestInput = { + requesterId: 'requester-a', + requestGeneration: 1, + attemptKey: 'attempt-1', + shape: ALLOCATION_SHAPE, + deadlineAtMs: 1_700_000_600_000, + admission: 'fail-fast', + activation: 'direct', + attribution: { tenantId: 'tenant-a' }, +}; + +export const ALLOCATION_PENDING_STATUS: LeaseRequestStatus = { + requesterId: ALLOCATION_REQUEST.requesterId, + requestGeneration: ALLOCATION_REQUEST.requestGeneration, + attemptKey: ALLOCATION_REQUEST.attemptKey, + state: 'pending', +}; + +export const ALLOCATION_GRANTED_STATUS: LeaseRequestStatus = { + ...ALLOCATION_PENDING_STATUS, + identityIncarnationId: ALLOCATION_IDENTITY.identityIncarnationId, + state: 'granted', + lease: ALLOCATION_LEASE, +}; + +export const ALLOCATION_REFUSED_STATUS: LeaseRequestStatus = { + ...ALLOCATION_PENDING_STATUS, + state: 'refused', + refusal: { reason: 'simulator-capacity', retryAfterMs: 30_000 }, +}; diff --git a/src/daemon/managed-device-allocation/__tests__/journal.test.ts b/src/daemon/managed-device-allocation/__tests__/journal.test.ts new file mode 100644 index 000000000..127d74548 --- /dev/null +++ b/src/daemon/managed-device-allocation/__tests__/journal.test.ts @@ -0,0 +1,564 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { test, vi } from 'vitest'; +import type { + LeaseRequestInput, + ManagedDeviceAllocatorPort, + SupersedeLeaseRequestInput, +} from '@agent-device/contracts/managed-device-allocation'; +import type { ScriptedAllocatorMethod } from '../../../__tests__/test-utils/managed-device-allocator.fixtures.ts'; +import { createScriptedManagedDeviceAllocator } from '../../../__tests__/test-utils/managed-device-allocator.fixtures.ts'; +import { mkdtempForTestSync } from '../../../__tests__/test-utils/tmp-dir.ts'; +import { createAllocationOperationJournal, type AllocationBindingHooks } from '../journal.ts'; +import { createAllocationOperationStore, type AllocationOperationStore } from '../store.ts'; +import { + ALLOCATION_GRANTED_STATUS, + ALLOCATION_PENDING_STATUS, + ALLOCATION_REFUSED_STATUS, + ALLOCATION_REQUEST, +} from './fixtures.ts'; + +const NOW = 1_700_000_000_000; + +function setup( + script: Partial> = {}, + hooks?: AllocationBindingHooks, +): { + root: string; + store: ReturnType; + allocator: ReturnType; + journal: ReturnType; +} { + const root = mkdtempForTestSync('allocation-operation-journal-'); + const store = createAllocationOperationStore({ allocationsDir: path.join(root, 'allocations') }); + const allocator = createScriptedManagedDeviceAllocator({ instanceId: 'allocator-1', script }); + const journal = createAllocationOperationJournal({ + store, + allocator, + binding: hooks, + now: () => NOW, + }); + return { root, store, allocator, journal }; +} + +function input(overrides: Partial = {}): LeaseRequestInput { + return { ...ALLOCATION_REQUEST, ...overrides }; +} + +function statusFor(overrides: Partial = {}) { + return { ...ALLOCATION_GRANTED_STATUS, ...overrides }; +} + +function foundRecord( + journal: ReturnType, + request: LeaseRequestInput = ALLOCATION_REQUEST, +) { + const read = journal.read(request); + assert.equal(read.status, 'found'); + return read.record; +} + +function bindingHooks( + journal: ReturnType | null = null, + cleanupImpl?: AllocationBindingHooks['cleanup'], +): AllocationBindingHooks & { published: string[]; cleaned: string[] } { + const published: string[] = []; + const cleaned: string[] = []; + return { + published, + cleaned, + async publish(binding) { + if (journal) { + const read = journal.read(binding.operation); + assert.equal(read.status, 'found'); + assert.equal(read.record.phase.status, 'granted'); + assert.equal(read.record.binding, 'publish-pending'); + } + published.push(binding.lease.id); + }, + async cleanup(binding) { + cleaned.push(binding.lease.id); + if (cleanupImpl) await cleanupImpl(binding); + }, + }; +} + +test('persists intent before request and the allocator outcome before publishing a binding', async () => { + const root = mkdtempForTestSync('allocation-operation-order-'); + const store = createAllocationOperationStore({ allocationsDir: path.join(root, 'allocations') }); + const scripted = createScriptedManagedDeviceAllocator({ + instanceId: 'allocator-1', + script: { requestLease: [ALLOCATION_GRANTED_STATUS] }, + }); + const allocator: ManagedDeviceAllocatorPort = { + ...scripted, + async requestLease(request) { + const read = store.read(request); + assert.equal(read.status, 'found'); + assert.equal(read.record.phase.status, 'pending'); + return scripted.requestLease(request); + }, + }; + const hooks: AllocationBindingHooks = { + async publish(binding) { + const read = store.read(binding.operation); + assert.equal(read.status, 'found'); + assert.equal(read.record.phase.status, 'granted'); + assert.equal(read.record.binding, 'publish-pending'); + }, + async cleanup() {}, + }; + const withHooks = createAllocationOperationJournal({ + store, + allocator, + binding: hooks, + now: () => NOW, + }); + const result = await withHooks.allocate(ALLOCATION_REQUEST); + assert.equal(result.status, 'granted'); + const record = foundRecord(withHooks); + assert.equal(record.binding, 'published'); + assert.deepEqual( + scripted.calls.map((call) => call.method), + ['requestLease'], + ); +}); + +test('cleans a binding after publish succeeds but its durable publication state is lost', async () => { + const root = mkdtempForTestSync('allocation-operation-publish-recovery-'); + const baseStore = createAllocationOperationStore({ + allocationsDir: path.join(root, 'allocations'), + }); + let failPublicationStateWrite = true; + const store: AllocationOperationStore = Object.freeze({ + ...baseStore, + async transition(ref, expectedFence, transitionInput, nowMs) { + if (failPublicationStateWrite && transitionInput.kind === 'binding-published') { + failPublicationStateWrite = false; + throw new Error('binding publication state write lost'); + } + return baseStore.transition(ref, expectedFence, transitionInput, nowMs); + }, + }); + const events: string[] = []; + const hooks: AllocationBindingHooks = { + async publish() { + events.push('publish'); + }, + async cleanup() { + events.push('cleanup'); + }, + }; + const scriptedAllocator = createScriptedManagedDeviceAllocator({ + instanceId: 'allocator-1', + script: { requestLease: [ALLOCATION_GRANTED_STATUS], releaseLease: [undefined] }, + }); + const allocator: ManagedDeviceAllocatorPort = { + ...scriptedAllocator, + async releaseLease(input) { + events.push('release'); + return scriptedAllocator.releaseLease(input); + }, + }; + const journal = createAllocationOperationJournal({ + store, + allocator, + binding: hooks, + now: () => NOW, + }); + + const publishResult = await journal.allocate(ALLOCATION_REQUEST); + assert.equal(publishResult.status, 'blocked'); + assert.equal( + publishResult.status === 'blocked' ? publishResult.reason : undefined, + 'persistence-failed', + ); + assert.equal(foundRecord(journal).binding, 'publish-pending'); + + const released = await journal.release(ALLOCATION_REQUEST); + assert.equal(released.status, 'released'); + assert.deepEqual(events, ['publish', 'cleanup', 'release']); + assert.deepEqual( + scriptedAllocator.calls.map((call) => call.method), + ['requestLease', 'releaseLease'], + ); +}); + +test('reconciles a lost response by lookup after reconstructing the journal', async () => { + const first = setup({ requestLease: [new Error('connection closed')] }); + const uncertain = await first.journal.allocate(ALLOCATION_REQUEST); + assert.equal(uncertain.status, 'uncertain'); + assert.equal(foundRecord(first.journal).phase.status, 'unknown'); + + const restartedAllocator = createScriptedManagedDeviceAllocator({ + instanceId: 'allocator-1', + script: { getLeaseRequestStatus: [ALLOCATION_GRANTED_STATUS] }, + }); + const restarted = createAllocationOperationJournal({ + store: createAllocationOperationStore({ allocationsDir: path.join(first.root, 'allocations') }), + allocator: restartedAllocator, + binding: bindingHooks(), + now: () => NOW, + }); + const recovered = await restarted.recover(ALLOCATION_REQUEST); + assert.equal(recovered.status, 'granted'); + assert.deepEqual( + restartedAllocator.calls.map((call) => call.method), + ['getLeaseRequestStatus'], + ); +}); + +test('does not replay durable work through a different allocator instance', async () => { + const first = setup({ requestLease: [new Error('response lost')] }); + assert.equal((await first.journal.allocate(ALLOCATION_REQUEST)).status, 'uncertain'); + + const allocator = createScriptedManagedDeviceAllocator({ instanceId: 'allocator-2' }); + const restarted = createAllocationOperationJournal({ + store: createAllocationOperationStore({ allocationsDir: path.join(first.root, 'allocations') }), + allocator, + now: () => NOW, + }); + const result = await restarted.recover(ALLOCATION_REQUEST); + assert.equal(result.status, 'blocked'); + assert.equal(result.status === 'blocked' ? result.reason : undefined, 'payload-mismatch'); + assert.deepEqual(allocator.calls, []); +}); + +test('caller abort abandons the wait without durably cancelling allocator work', async () => { + const controller = new AbortController(); + const scripted = createScriptedManagedDeviceAllocator({ + instanceId: 'allocator-1', + script: { requestLease: [new Error('request aborted locally')] }, + }); + const allocator: ManagedDeviceAllocatorPort = { + ...scripted, + async requestLease(request) { + controller.abort(); + return scripted.requestLease(request); + }, + }; + const root = mkdtempForTestSync('allocation-operation-abort-'); + const store = createAllocationOperationStore({ allocationsDir: path.join(root, 'allocations') }); + const journal = createAllocationOperationJournal({ store, allocator, now: () => NOW }); + const result = await journal.allocate(input({ signal: controller.signal })); + assert.equal(result.status, 'abandoned'); + assert.equal(foundRecord(journal).phase.status, 'pending'); + assert.deepEqual( + scripted.calls.map((call) => call.method), + ['requestLease'], + ); + assert.equal( + scripted.calls.some((call) => call.method === 'cancelLeaseRequest'), + false, + ); +}); + +test('replays a durable unresolved request through its original allocator action', async () => { + const controller = new AbortController(); + controller.abort(); + const setupResult = setup({ requestLease: [ALLOCATION_GRANTED_STATUS] }, bindingHooks()); + assert.equal( + (await setupResult.journal.allocate(input({ signal: controller.signal }))).status, + 'abandoned', + ); + assert.equal(foundRecord(setupResult.journal).phase.status, 'unresolved'); + + const replay = await setupResult.journal.allocate(ALLOCATION_REQUEST); + assert.equal(replay.status, 'granted'); + assert.deepEqual( + setupResult.allocator.calls.map((call) => call.method), + ['requestLease'], + ); +}); + +test('same-key replay is stable, while a new lane goes to an allocator decision', async () => { + const secondStatus = statusFor({ + requesterId: 'requester-b', + attemptKey: 'attempt-2', + requestGeneration: 1, + }); + const first = setup( + { + requestLease: [ALLOCATION_GRANTED_STATUS, secondStatus], + }, + bindingHooks(), + ); + const firstResult = await first.journal.allocate(ALLOCATION_REQUEST); + assert.equal(firstResult.status, 'granted'); + const replay = await first.journal.allocate(ALLOCATION_REQUEST); + assert.equal(replay.status, 'granted'); + const second = await first.journal.allocate( + input({ requesterId: 'requester-b', attemptKey: 'attempt-2', requestGeneration: 1 }), + ); + assert.equal(second.status, 'granted'); + assert.deepEqual( + first.allocator.calls.map((call) => call.method), + ['requestLease', 'requestLease'], + ); +}); + +test('a new key cannot replace nonterminal allocator work without explicit supersession', async () => { + const first = setup({ requestLease: [new Error('response lost')] }); + assert.equal((await first.journal.allocate(ALLOCATION_REQUEST)).status, 'uncertain'); + const blocked = await first.journal.allocate( + input({ attemptKey: 'attempt-2', requestGeneration: 2 }), + ); + assert.equal(blocked.status, 'blocked'); + assert.equal(blocked.status === 'blocked' ? blocked.reason : undefined, 'lane-busy'); + assert.deepEqual( + first.allocator.calls.map((call) => call.method), + ['requestLease'], + ); +}); + +test('a new key cannot reuse a stale requester generation after terminal work', async () => { + const setupResult = setup({ + requestLease: [{ ...ALLOCATION_REFUSED_STATUS, attemptKey: 'attempt-2', requestGeneration: 2 }], + }); + assert.equal( + (await setupResult.journal.allocate(input({ attemptKey: 'attempt-2', requestGeneration: 2 }))) + .status, + 'refused', + ); + const stale = await setupResult.journal.allocate( + input({ attemptKey: 'attempt-3', requestGeneration: 1 }), + ); + assert.equal(stale.status, 'blocked'); + assert.equal(stale.status === 'blocked' ? stale.reason : undefined, 'payload-mismatch'); + assert.deepEqual( + setupResult.allocator.calls.map((call) => call.method), + ['requestLease'], + ); +}); + +test('explicit supersession uses the allocator and leaves both attempts durable for reconciliation', async () => { + const replacementStatus = statusFor({ attemptKey: 'attempt-2', requestGeneration: 2 }); + const first = setup( + { + requestLease: [new Error('response lost')], + supersedeLeaseRequest: [replacementStatus], + }, + bindingHooks(), + ); + assert.equal((await first.journal.allocate(ALLOCATION_REQUEST)).status, 'uncertain'); + const replacement: LeaseRequestInput = input({ attemptKey: 'attempt-3', requestGeneration: 3 }); + const supersede: SupersedeLeaseRequestInput = { + requesterId: 'requester-a', + expectedRequestGeneration: 1, + replacement: { ...replacement, attemptKey: 'attempt-2', requestGeneration: 2 }, + }; + const third = await first.journal.supersede(supersede); + assert.equal(third.status, 'granted'); + assert.equal(foundRecord(first.journal).phase.status, 'unknown'); + assert.equal(foundRecord(first.journal, supersede.replacement).phase.status, 'granted'); + assert.deepEqual( + first.allocator.calls.map((call) => call.method), + ['requestLease', 'supersedeLeaseRequest'], + ); +}); + +test('supersession refuses a live binding before asking the allocator to revoke it', async () => { + const first = setup({ requestLease: [ALLOCATION_GRANTED_STATUS] }, bindingHooks()); + assert.equal((await first.journal.allocate(ALLOCATION_REQUEST)).status, 'granted'); + const replacement = input({ attemptKey: 'attempt-2', requestGeneration: 2 }); + const result = await first.journal.supersede({ + requesterId: ALLOCATION_REQUEST.requesterId, + expectedRequestGeneration: ALLOCATION_REQUEST.requestGeneration, + replacement, + }); + assert.equal(result.status, 'blocked'); + assert.equal(result.status === 'blocked' ? result.reason : undefined, 'already-granted'); + assert.deepEqual( + first.allocator.calls.map((call) => call.method), + ['requestLease'], + ); +}); + +test('a terminal refusal replays without re-entering the allocator', async () => { + const setupResult = setup({ requestLease: [ALLOCATION_REFUSED_STATUS] }); + const first = await setupResult.journal.allocate(ALLOCATION_REQUEST); + assert.equal(first.status, 'refused'); + const replay = await setupResult.journal.allocate(ALLOCATION_REQUEST); + assert.equal(replay.status, 'refused'); + assert.deepEqual( + setupResult.allocator.calls.map((call) => call.method), + ['requestLease'], + ); +}); + +test('explicit cancellation is durable and is not confused with caller abandonment', async () => { + const controller = new AbortController(); + controller.abort(); + const setupResult = setup({ + cancelLeaseRequest: [ + { + requesterId: ALLOCATION_REQUEST.requesterId, + requestGeneration: ALLOCATION_REQUEST.requestGeneration, + attemptKey: ALLOCATION_REQUEST.attemptKey, + state: 'cancelled', + }, + ], + }); + const abandonedResult = await setupResult.journal.allocate(input({ signal: controller.signal })); + assert.equal(abandonedResult.status, 'abandoned'); + const cancelled = await setupResult.journal.cancel(ALLOCATION_REQUEST); + assert.equal(cancelled.status, 'cancelled'); + const replay = await setupResult.journal.cancel(ALLOCATION_REQUEST); + assert.equal(replay.status, 'cancelled'); + assert.deepEqual( + setupResult.allocator.calls.map((call) => call.method), + ['cancelLeaseRequest'], + ); +}); + +test('a late grant during explicit cancellation is durable but is never published as a success', async () => { + const hooks = bindingHooks(); + const setupResult = setup( + { + requestLease: [ALLOCATION_PENDING_STATUS], + cancelLeaseRequest: [ALLOCATION_GRANTED_STATUS], + }, + hooks, + ); + assert.equal((await setupResult.journal.allocate(ALLOCATION_REQUEST)).status, 'pending'); + const result = await setupResult.journal.cancel(ALLOCATION_REQUEST); + assert.equal(result.status, 'blocked'); + assert.equal(result.status === 'blocked' ? result.reason : undefined, 'already-granted'); + assert.deepEqual(hooks.published, []); + assert.equal(foundRecord(setupResult.journal).phase.status, 'granted'); +}); + +test('corrupt state is retained as unreadable evidence and never implies an allocator outcome', async () => { + const first = setup(); + const record = first.store.create({ + schemaVersion: 1, + requesterId: ALLOCATION_REQUEST.requesterId, + attemptKey: ALLOCATION_REQUEST.attemptKey, + allocatorInstanceId: 'allocator-1', + shape: ALLOCATION_REQUEST.shape, + deadlineAtMs: ALLOCATION_REQUEST.deadlineAtMs, + requestGeneration: ALLOCATION_REQUEST.requestGeneration, + admission: ALLOCATION_REQUEST.admission, + activation: ALLOCATION_REQUEST.activation, + createdAtMs: NOW, + updatedAtMs: NOW, + fence: { token: JSON.stringify(['requester-a', 'attempt-1']), generation: 0 }, + phase: { status: 'unresolved' }, + binding: 'unpublished', + release: 'not-requested', + }); + assert.equal(record.status, 'created'); + fs.writeFileSync(first.store.resolvePath(ALLOCATION_REQUEST), '{'); + const result = await first.journal.recover(ALLOCATION_REQUEST); + assert.equal(result.status, 'unreadable'); + assert.equal(first.allocator.calls.length, 0); +}); + +test('root journal enumeration failure blocks a new allocator attempt', async () => { + const setupResult = setup({ + requestLease: [new Error('response lost'), ALLOCATION_GRANTED_STATUS], + }); + assert.equal((await setupResult.journal.allocate(ALLOCATION_REQUEST)).status, 'uncertain'); + + const readdir = vi.spyOn(fs, 'readdirSync').mockImplementationOnce(() => { + throw Object.assign(new Error('allocation journal root is unreadable'), { code: 'EACCES' }); + }); + + try { + const result = await setupResult.journal.allocate( + input({ attemptKey: 'attempt-2', requestGeneration: 2 }), + ); + assert.equal(result.status, 'unreadable'); + assert.equal(result.status === 'unreadable' ? result.reason : undefined, 'corrupt'); + assert.deepEqual( + setupResult.allocator.calls.map((call) => call.method), + ['requestLease'], + ); + } finally { + readdir.mockRestore(); + } +}); + +test('cleanup uncertainty blocks release until cleanup is retried, then allocator release is retryable', async () => { + const failing = setup({ requestLease: [ALLOCATION_GRANTED_STATUS] }); + const hooks = bindingHooks(null, async () => { + throw new Error('binding cleanup uncertain'); + }); + const withFailingCleanup = createAllocationOperationJournal({ + store: failing.store, + allocator: failing.allocator, + binding: hooks, + now: () => NOW, + }); + assert.equal((await withFailingCleanup.allocate(ALLOCATION_REQUEST)).status, 'granted'); + const cleanupPending = await withFailingCleanup.release(ALLOCATION_REQUEST); + assert.equal(cleanupPending.status, 'cleanup-pending'); + assert.equal( + failing.allocator.calls.some((call) => call.method === 'releaseLease'), + false, + ); + + const restartedHooks = bindingHooks(); + const restarted = createAllocationOperationJournal({ + store: createAllocationOperationStore({ + allocationsDir: path.join(failing.root, 'allocations'), + }), + allocator: createScriptedManagedDeviceAllocator({ + instanceId: 'allocator-1', + script: { releaseLease: [undefined] }, + }), + binding: restartedHooks, + now: () => NOW, + }); + const released = await restarted.release(ALLOCATION_REQUEST); + assert.equal(released.status, 'released'); + assert.deepEqual(restartedHooks.cleaned, ['lease-1']); +}); + +test('allocator release uncertainty remains pending and is retried after restart', async () => { + const first = setup( + { + requestLease: [ALLOCATION_GRANTED_STATUS], + releaseLease: [new Error('release response lost')], + }, + bindingHooks(), + ); + assert.equal((await first.journal.allocate(ALLOCATION_REQUEST)).status, 'granted'); + const uncertain = await first.journal.release(ALLOCATION_REQUEST); + assert.equal(uncertain.status, 'uncertain'); + assert.equal(foundRecord(first.journal).release, 'pending'); + + const allocator = createScriptedManagedDeviceAllocator({ + instanceId: 'allocator-1', + script: { releaseLease: [undefined] }, + }); + const restarted = createAllocationOperationJournal({ + store: createAllocationOperationStore({ allocationsDir: path.join(first.root, 'allocations') }), + allocator, + binding: bindingHooks(), + now: () => NOW, + }); + assert.equal((await restarted.release(ALLOCATION_REQUEST)).status, 'released'); + assert.deepEqual( + allocator.calls.map((call) => call.method), + ['releaseLease'], + ); +}); + +test('a durable allocator outcome is not replaced or deleted by release replay', async () => { + const first = setup( + { requestLease: [ALLOCATION_GRANTED_STATUS], releaseLease: [undefined] }, + bindingHooks(), + ); + assert.equal((await first.journal.allocate(ALLOCATION_REQUEST)).status, 'granted'); + assert.equal((await first.journal.release(ALLOCATION_REQUEST)).status, 'released'); + const pathOnDisk = first.store.resolvePath(ALLOCATION_REQUEST); + assert.equal(fs.existsSync(pathOnDisk), true); + const replay = await first.journal.release(ALLOCATION_REQUEST); + assert.equal(replay.status, 'released'); + assert.deepEqual( + first.allocator.calls.map((call) => call.method), + ['requestLease', 'releaseLease'], + ); +}); diff --git a/src/daemon/managed-device-allocation/__tests__/record.test.ts b/src/daemon/managed-device-allocation/__tests__/record.test.ts new file mode 100644 index 000000000..50f51cd10 --- /dev/null +++ b/src/daemon/managed-device-allocation/__tests__/record.test.ts @@ -0,0 +1,273 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { LeaseRequestStatus } from '@agent-device/contracts/managed-device-allocation'; +import { managedBindingFence } from '@agent-device/contracts/platform-runtime'; +import { AppError } from '@agent-device/kernel/errors'; +import { ALLOCATION_OPERATION_SCHEMA_VERSION, bindingFenceFor } from '../record.ts'; +import { newAllocationOperation } from '../record-factory.ts'; +import { applyAllocationTransition } from '../transitions.ts'; +import type { AllocationOperationRecord, AllocationTransition } from '../record.ts'; +import { decodeAllocationOperationRecord } from '../record-codec.ts'; +import { + ALLOCATION_GRANTED_STATUS, + ALLOCATION_LEASE, + ALLOCATION_PENDING_STATUS, + ALLOCATION_REFUSED_STATUS, + ALLOCATION_REQUEST, +} from './fixtures.ts'; + +const NOW = 1_700_000_000_000; + +function requested(overrides: Partial[0]> = {}) { + return newAllocationOperation({ + ...ALLOCATION_REQUEST, + allocatorInstanceId: 'allocator-1', + nowMs: NOW, + ...overrides, + }); +} + +function apply( + record: AllocationOperationRecord, + transition: AllocationTransition, + nowMs = NOW + 1, +): AllocationOperationRecord { + const result = applyAllocationTransition(record, transition, nowMs); + assert.equal(result.status, 'applied'); + return result.record; +} + +test('records a grant before binding publication and derives its managed binding fence', () => { + const dispatched = apply(requested(), { kind: 'request-dispatched' }); + const granted = apply(dispatched, { + kind: 'allocator-outcome', + outcome: { + status: 'granted', + lease: ALLOCATION_LEASE, + identityIncarnationId: 'incarnation-1', + }, + }); + + assert.equal(granted.phase.status, 'granted'); + assert.equal(granted.binding, 'unpublished'); + assert.equal(granted.release, 'not-requested'); + assert.deepEqual( + bindingFenceFor(granted), + managedBindingFence({ + requesterId: 'requester-a', + requestGeneration: 1, + identityIncarnationId: 'incarnation-1', + }), + ); + assert.notEqual(granted.fence.generation, dispatched.fence.generation); +}); + +test('exactly replaying a local transition is idempotent while a stale fence is refused by the store seam', () => { + const dispatched = apply(requested(), { kind: 'request-dispatched' }); + const replay = applyAllocationTransition(dispatched, { kind: 'request-dispatched' }, NOW + 2); + assert.equal(replay.status, 'already-applied'); + assert.equal(replay.record, dispatched); + + const granted = apply(dispatched, { + kind: 'allocator-outcome', + outcome: { + status: 'granted', + lease: ALLOCATION_LEASE, + identityIncarnationId: 'incarnation-1', + }, + }); + assert.throws( + () => applyAllocationTransition(dispatched, { kind: 'binding-published' }, NOW + 3), + (error: unknown) => error instanceof AppError && error.details?.reason === 'transition-invalid', + ); + assert.throws( + () => applyAllocationTransition(granted, { kind: 'binding-published' }, NOW + 3), + (error: unknown) => error instanceof AppError && error.details?.reason === 'transition-invalid', + ); + const publishPending = apply(granted, { kind: 'binding-publish-pending' }); + const published = apply(publishPending, { kind: 'binding-published' }); + assert.equal( + applyAllocationTransition(published, { kind: 'binding-published' }, NOW + 4).status, + 'already-applied', + ); +}); + +test('cleanup and allocator release are ordered and remain retryable', () => { + const granted = apply(apply(requested(), { kind: 'request-dispatched' }), { + kind: 'allocator-outcome', + outcome: { + status: 'granted', + lease: ALLOCATION_LEASE, + identityIncarnationId: 'incarnation-1', + }, + }); + const publishPending = apply(granted, { kind: 'binding-publish-pending' }); + const published = apply(publishPending, { kind: 'binding-published' }); + const pendingCleanup = apply(published, { + kind: 'binding-cleanup-pending', + message: 'binding teardown was not confirmed', + }); + const cleaned = apply(pendingCleanup, { kind: 'binding-cleaned' }); + const releasePending = apply(cleaned, { kind: 'release-pending' }); + const released = apply(releasePending, { kind: 'allocator-released' }); + + assert.equal(released.binding, 'cleaned'); + assert.equal(released.release, 'released'); + assert.equal(released.phase.status, 'granted'); +}); + +test('allocator status conversion preserves lookup results and refuses malformed grants', () => { + const dispatched = apply(requested(), { kind: 'request-dispatched' }); + const pending = applyAllocationTransition( + dispatched, + { + kind: 'allocator-status', + status: { ...ALLOCATION_PENDING_STATUS, identityIncarnationId: 'incarnation-1' }, + }, + NOW + 2, + ); + assert.equal(pending.status, 'applied'); + assert.equal(pending.record.phase.status, 'pending'); + assert.equal(pending.record.identityIncarnationId, 'incarnation-1'); + + const identityPreserved = applyAllocationTransition( + pending.record, + { kind: 'allocator-status', status: ALLOCATION_PENDING_STATUS }, + NOW + 2, + ); + assert.equal(identityPreserved.status, 'already-applied'); + assert.equal(identityPreserved.record.identityIncarnationId, 'incarnation-1'); + + const refused = applyAllocationTransition( + pending.record, + { + kind: 'allocator-status', + status: ALLOCATION_REFUSED_STATUS, + }, + NOW + 3, + ); + assert.equal(refused.status, 'applied'); + assert.equal(refused.record.phase.status, 'refused'); + + const malformed = applyAllocationTransition( + dispatched, + { + kind: 'allocator-status', + status: { ...ALLOCATION_GRANTED_STATUS, lease: undefined }, + }, + NOW + 4, + ); + assert.equal(malformed.status, 'applied'); + assert.equal(malformed.record.phase.status, 'ambiguous'); + + const unknownWithGrant = applyAllocationTransition( + dispatched, + { + kind: 'allocator-status', + status: { ...ALLOCATION_GRANTED_STATUS, state: 'unknown' }, + }, + NOW + 5, + ); + assert.equal(unknownWithGrant.status, 'applied'); + assert.equal(unknownWithGrant.record.phase.status, 'ambiguous'); + + const unknownState = applyAllocationTransition( + dispatched, + { + kind: 'allocator-status', + status: { ...ALLOCATION_PENDING_STATUS, state: 'future' } as unknown as LeaseRequestStatus, + }, + NOW + 6, + ); + assert.equal(unknownState.status, 'applied'); + assert.equal(unknownState.record.phase.status, 'ambiguous'); +}); + +test('record decoding fails closed for unsupported, ambiguous, and unfenced state', () => { + const granted = apply(apply(requested(), { kind: 'request-dispatched' }), { + kind: 'allocator-outcome', + outcome: { + status: 'granted', + lease: ALLOCATION_LEASE, + identityIncarnationId: 'incarnation-1', + }, + }); + const stored = JSON.parse(JSON.stringify(granted)) as Record; + + assert.deepEqual(decodeAllocationOperationRecord(stored), { + status: 'decoded', + record: granted, + }); + assert.equal( + unreadableReason( + decodeAllocationOperationRecord({ + ...stored, + schemaVersion: ALLOCATION_OPERATION_SCHEMA_VERSION + 1, + }), + ), + 'unsupported-version', + ); + assert.equal( + unreadableReason( + decodeAllocationOperationRecord({ + ...stored, + phase: { status: 'pending' }, + binding: 'published', + }), + ), + 'ambiguous', + ); + assert.equal( + unreadableReason( + decodeAllocationOperationRecord({ + ...stored, + phase: { status: 'refused', refusal: { reason: 'simulator-capacity' } }, + binding: 'unpublished', + }), + ), + 'ambiguous', + ); + assert.equal( + unreadableReason( + decodeAllocationOperationRecord({ + ...stored, + fence: { token: 'not-canonical', generation: 2 }, + }), + ), + 'unfenced', + ); +}); + +test('an ambiguous transition is a fail-closed terminal fence', () => { + const dispatched = apply(requested(), { kind: 'request-dispatched' }); + const ambiguous = applyAllocationTransition( + dispatched, + { + kind: 'allocator-status', + status: { ...ALLOCATION_GRANTED_STATUS, lease: undefined }, + }, + NOW + 2, + ); + assert.equal(ambiguous.status, 'applied'); + assert.equal(ambiguous.record.phase.status, 'ambiguous'); + const replay = applyAllocationTransition( + ambiguous.record, + { + kind: 'allocator-outcome', + outcome: { + status: 'granted', + lease: ALLOCATION_GRANTED_STATUS.lease!, + identityIncarnationId: 'incarnation-1', + }, + }, + NOW + 3, + ); + assert.equal(replay.status, 'already-terminal'); + assert.equal(replay.record, ambiguous.record); +}); + +function unreadableReason( + result: ReturnType, +): string | undefined { + return result.status === 'unreadable' ? result.reason : undefined; +} diff --git a/src/daemon/managed-device-allocation/__tests__/store.test.ts b/src/daemon/managed-device-allocation/__tests__/store.test.ts new file mode 100644 index 000000000..8b6c54950 --- /dev/null +++ b/src/daemon/managed-device-allocation/__tests__/store.test.ts @@ -0,0 +1,237 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { test, vi } from 'vitest'; +import { acquireProcessLock } from '@agent-device/host-kit/file'; +import { readCurrentOwnerIdentity } from '@agent-device/host-kit/process'; +import { mkdtempForTestSync } from '../../../__tests__/test-utils/tmp-dir.ts'; +import { newAllocationOperation } from '../record-factory.ts'; +import type { AllocationOperationRecord } from '../record.ts'; +import { createAllocationOperationStore, type AllocationOperationStore } from '../store.ts'; +import { ALLOCATION_LEASE, ALLOCATION_REQUEST } from './fixtures.ts'; + +const NOW = 1_700_000_000_000; + +function fixture(): { + root: string; + store: AllocationOperationStore; + record: AllocationOperationRecord; +} { + const root = mkdtempForTestSync('allocation-operation-store-'); + const store = createAllocationOperationStore({ allocationsDir: path.join(root, 'allocations') }); + const record = newAllocationOperation({ + ...ALLOCATION_REQUEST, + allocatorInstanceId: 'allocator-1', + nowMs: NOW, + }); + return { root, store, record }; +} + +test('publishes an operation durably under an independent hashed state directory and reconstructs it after restart', () => { + const { root, store, record } = fixture(); + const created = store.create(record); + assert.equal(created.status, 'created'); + const operationPath = store.resolvePath(record); + assert.equal(path.dirname(path.dirname(operationPath)), path.join(root, 'allocations')); + assert.equal(fs.statSync(operationPath).mode & 0o777, 0o600); + assert.equal(fs.readdirSync(path.dirname(operationPath)).length, 1); + assert.equal(fs.existsSync(path.join(root, 'sessions')), false); + + const restarted = createAllocationOperationStore({ + allocationsDir: path.join(root, 'allocations'), + }); + const read = restarted.read(record); + assert.deepEqual(read, { status: 'found', path: operationPath, record }); + assert.equal(read.status === 'found' ? Object.isFrozen(read.record.phase) : false, true); +}); + +test('uses the operation fence for transitions and refuses a stale writer without changing the file', async () => { + const { store, record } = fixture(); + store.create(record); + const dispatched = await store.transition( + record, + record.fence, + { kind: 'request-dispatched' }, + NOW + 1, + ); + assert.equal(dispatched.status, 'recorded'); + assert.equal(dispatched.status === 'recorded' ? dispatched.record.fence.generation : -1, 1); + + const stale = await store.transition( + record, + record.fence, + { kind: 'allocator-unknown', message: 'lost response' }, + NOW + 2, + ); + assert.equal(stale.status, 'fence-lost'); + assert.equal(stale.status === 'fence-lost' ? stale.current.fence.generation : -1, 1); + + const replay = await store.transition( + record, + dispatched.status === 'recorded' ? dispatched.record.fence : record.fence, + { kind: 'request-dispatched' }, + NOW + 3, + ); + assert.equal(replay.status, 'already-applied'); + assert.equal(replay.status === 'already-applied' ? replay.record.fence.generation : -1, 1); +}); + +test('a concurrent writer waits for the same operation lock before reading and fencing', async () => { + const { store, record } = fixture(); + store.create(record); + const owner = readCurrentOwnerIdentity(); + const operationPath = store.resolvePath(record); + const release = await acquireProcessLock({ + lockDirPath: `${operationPath}.lock`, + owner: { pid: owner.pid, startTime: owner.startTime, acquiredAtMs: Date.now() }, + description: 'allocation operation test holder', + }); + let settled = false; + const pending = store + .transition(record, record.fence, { kind: 'request-dispatched' }, NOW + 1) + .finally(() => { + settled = true; + }); + await new Promise((resolve) => setTimeout(resolve, 150)); + assert.equal(settled, false); + await release(); + assert.equal((await pending).status, 'recorded'); +}); + +test('retains corrupt, unsupported, unfenced, and path-ambiguous records as diagnosable entries', () => { + const { root, store, record } = fixture(); + store.create(record); + const operationPath = store.resolvePath(record); + const stored = JSON.parse(fs.readFileSync(operationPath, 'utf8')) as Record; + const cases = [ + ['corrupt', '{'], + ['unsupported-version', JSON.stringify({ ...stored, schemaVersion: 8 })], + ['unfenced', JSON.stringify({ ...stored, fence: { token: 'wrong', generation: 0 } })], + [ + 'ambiguous', + JSON.stringify({ ...stored, phase: { status: 'pending' }, binding: 'published' }), + ], + ] as const; + for (const [reason, contents] of cases) { + fs.writeFileSync(operationPath, contents); + const read = store.read(record); + assert.equal(read.status, 'unreadable'); + assert.equal(read.status === 'unreadable' ? read.reason : undefined, reason); + } + + fs.writeFileSync(operationPath, JSON.stringify({ ...stored, requesterId: 'other-requester' })); + const pathMismatch = store.read(record); + assert.equal(pathMismatch.status, 'unreadable'); + assert.equal(pathMismatch.status === 'unreadable' ? pathMismatch.reason : undefined, 'ambiguous'); + assert.equal(store.list().filter((entry) => entry.status === 'unreadable').length, 1); + assert.equal(path.dirname(path.dirname(operationPath)), path.join(root, 'allocations')); +}); + +test('does not follow a symbolic-link destination', () => { + const { root, store, record } = fixture(); + const operationPath = store.resolvePath(record); + fs.mkdirSync(path.dirname(operationPath), { recursive: true }); + const outside = path.join(root, 'outside.json'); + fs.writeFileSync(outside, '{}'); + fs.symlinkSync(outside, operationPath); + assert.equal(store.create(record).status, 'unreadable'); + assert.equal(store.read(record).status, 'unreadable'); + assert.equal(fs.readFileSync(outside, 'utf8'), '{}'); +}); + +test('retains a record as unreadable when its identity changes during read', () => { + const { store, record } = fixture(); + assert.equal(store.create(record).status, 'created'); + const operationPath = store.resolvePath(record); + const replacementPath = `${operationPath}.replacement`; + fs.copyFileSync(operationPath, replacementPath); + const realLstatSync = fs.lstatSync; + let operationLstatCalls = 0; + const lstat = vi.spyOn(fs, 'lstatSync').mockImplementation((( + target: fs.PathLike, + options?: unknown, + ) => { + if (target.toString() === operationPath) { + operationLstatCalls += 1; + const resolved = operationLstatCalls === 2 ? replacementPath : target; + return (realLstatSync as (path: fs.PathLike, options?: unknown) => fs.Stats)( + resolved, + options, + ); + } + return (realLstatSync as (path: fs.PathLike, options?: unknown) => fs.Stats)(target, options); + }) as typeof fs.lstatSync); + + try { + const read = store.read(record); + assert.equal(read.status, 'unreadable'); + assert.equal(read.status === 'unreadable' ? read.reason : undefined, 'corrupt'); + assert.match(read.status === 'unreadable' ? read.message : '', /identity changed/); + } finally { + lstat.mockRestore(); + fs.rmSync(replacementPath, { force: true }); + } +}); + +test('does not follow a symbolic-link lane directory', () => { + const { root, store, record } = fixture(); + const operationPath = store.resolvePath(record); + const lanePath = path.dirname(operationPath); + const outsideLane = path.join(root, 'outside-lane'); + fs.mkdirSync(path.dirname(lanePath), { recursive: true }); + fs.mkdirSync(outsideLane); + fs.symlinkSync(outsideLane, lanePath, 'dir'); + + assert.equal(store.create(record).status, 'unreadable'); + assert.equal(store.read(record).status, 'unreadable'); + assert.deepEqual(fs.readdirSync(outsideLane), []); +}); + +test('retains an unreadable lane enumeration instead of omitting its operations', () => { + const { store, record } = fixture(); + assert.equal(store.create(record).status, 'created'); + const lanePath = path.dirname(store.resolvePath(record)); + const originalReaddirSync = fs.readdirSync; + const readdir = vi.spyOn(fs, 'readdirSync').mockImplementation((directory, options) => { + if (directory.toString() === lanePath) { + throw Object.assign(new Error('allocation lane is unreadable'), { code: 'EACCES' }); + } + return originalReaddirSync(directory, options); + }); + + try { + const listed = store.list(); + assert.equal(listed.length, 1); + assert.equal(listed[0]?.status, 'unreadable'); + assert.equal(listed[0]?.status === 'unreadable' ? listed[0].reason : undefined, 'corrupt'); + } finally { + readdir.mockRestore(); + } +}); + +test('a terminal allocator outcome is retained rather than deleted', async () => { + const { store, record } = fixture(); + store.create(record); + const dispatched = await store.transition( + record, + record.fence, + { kind: 'request-dispatched' }, + NOW + 1, + ); + assert.equal(dispatched.status, 'recorded'); + const granted = await store.transition( + record, + dispatched.status === 'recorded' ? dispatched.record.fence : record.fence, + { + kind: 'allocator-outcome', + outcome: { + status: 'granted', + lease: ALLOCATION_LEASE, + identityIncarnationId: 'incarnation-1', + }, + }, + NOW + 2, + ); + assert.equal(granted.status, 'recorded'); + assert.equal(fs.existsSync(store.resolvePath(record)), true); +}); diff --git a/src/daemon/managed-device-allocation/decision.ts b/src/daemon/managed-device-allocation/decision.ts new file mode 100644 index 000000000..109ccae00 --- /dev/null +++ b/src/daemon/managed-device-allocation/decision.ts @@ -0,0 +1,142 @@ +import type { + LeaseRequestRef, + ManagedLease, +} from '@agent-device/contracts/managed-device-allocation'; +import type { AllocationOperationRecord, AllocationOperationRef } from './record.ts'; +import { bindingFenceFor } from './record.ts'; +import type { ResourceOwnershipFence } from '@agent-device/contracts/platform-runtime'; + +export type AllocationDecisionMode = + | 'new' + | 'recover' + | 'continue' + | 'cancel' + | 'release' + | 'supersede'; + +export type AllocationBinding = Readonly<{ + operation: AllocationOperationRef; + identityIncarnationId: string; + lease: ManagedLease; + fence: ResourceOwnershipFence; +}>; + +export type AllocationAction = + | Readonly<{ kind: 'request'; ref: LeaseRequestRef }> + | Readonly<{ kind: 'supersede'; ref: LeaseRequestRef }> + | Readonly<{ kind: 'lookup'; ref: LeaseRequestRef }> + | Readonly<{ kind: 'cancel'; ref: LeaseRequestRef }> + | Readonly<{ kind: 'publish'; binding: AllocationBinding }> + | Readonly<{ kind: 'cleanup'; binding: AllocationBinding }> + | Readonly<{ kind: 'release'; leaseId: string }> + | Readonly<{ kind: 'pending' }> + | Readonly<{ kind: 'terminal' }> + | Readonly<{ + kind: 'blocked'; + reason: 'ambiguous-state' | 'already-granted' | 'binding-unavailable' | 'not-releasable'; + message: string; + }>; + +export function decideAllocationAction( + record: AllocationOperationRecord, + mode: AllocationDecisionMode, +): AllocationAction { + const ref = { requesterId: record.requesterId, attemptKey: record.attemptKey }; + + if (record.phase.status === 'ambiguous') { + return blocked('ambiguous-state', 'allocation operation state is ambiguous'); + } + if (record.phase.status === 'granted') { + return decideGrantedAction(record, mode); + } + if ( + record.phase.status === 'refused' || + record.phase.status === 'superseded' || + record.phase.status === 'cancelled' + ) { + return { kind: 'terminal' }; + } + if (mode === 'cancel') return { kind: 'cancel', ref }; + if (mode === 'new' && record.phase.status === 'unresolved') return { kind: 'request', ref }; + if (mode === 'supersede' && record.phase.status === 'unresolved') + return { kind: 'supersede', ref }; + if (record.phase.status === 'unknown') return { kind: 'lookup', ref }; + if (record.phase.status === 'pending') { + return mode === 'continue' ? { kind: 'pending' } : { kind: 'lookup', ref }; + } + return { kind: 'lookup', ref }; +} + +function decideGrantedAction( + record: AllocationOperationRecord, + mode: AllocationDecisionMode, +): AllocationAction { + if (record.phase.status !== 'granted') { + return blocked('ambiguous-state', 'granted action received a non-granted record'); + } + const binding = toBinding(record); + if (!binding) return blocked('ambiguous-state', 'granted allocation has no binding identity'); + if (mode === 'cancel') return blocked('already-granted', 'allocation is already granted'); + const cleanup = decideGrantedCleanup(record, mode, binding); + if (cleanup) return cleanup; + const leaseId = record.phase.lease.id; + if (mode === 'release') return decideGrantedRelease(record, leaseId); + return decideGrantedBinding(record, binding, leaseId); +} + +function decideGrantedCleanup( + record: AllocationOperationRecord, + mode: AllocationDecisionMode, + binding: AllocationBinding, +): AllocationAction | undefined { + const recoveryCleanup = + mode === 'recover' && + (record.binding === 'publish-pending' || record.binding === 'cleanup-pending'); + if ((mode !== 'release' && !recoveryCleanup) || record.binding === 'cleaned') return undefined; + return record.release === 'not-requested' ? { kind: 'cleanup', binding } : undefined; +} + +function decideGrantedRelease( + record: AllocationOperationRecord, + leaseId: string, +): AllocationAction { + return record.release === 'released' ? { kind: 'terminal' } : { kind: 'release', leaseId }; +} + +function decideGrantedBinding( + record: AllocationOperationRecord, + binding: AllocationBinding, + leaseId: string, +): AllocationAction { + if (record.binding === 'unpublished') return { kind: 'publish', binding }; + if (record.binding === 'published') return { kind: 'terminal' }; + if (record.binding === 'publish-pending') { + return blocked( + 'not-releasable', + 'allocation binding publication is uncertain and requires explicit release recovery', + ); + } + if (record.binding === 'cleanup-pending') { + return blocked('not-releasable', 'allocation binding requires explicit release recovery'); + } + return record.release === 'released' ? { kind: 'terminal' } : { kind: 'release', leaseId }; +} + +function toBinding(record: AllocationOperationRecord): AllocationBinding | null { + if (record.phase.status !== 'granted' || record.identityIncarnationId === undefined) return null; + const fence = bindingFenceFor(record); + if (!fence) return null; + return Object.freeze({ + operation: Object.freeze({ requesterId: record.requesterId, attemptKey: record.attemptKey }), + identityIncarnationId: record.identityIncarnationId, + lease: record.phase.lease, + fence, + }); +} + +function blocked( + reason: Extract['reason'], + message: string, +): AllocationAction { + return { kind: 'blocked', reason, message }; +} diff --git a/src/daemon/managed-device-allocation/fence.ts b/src/daemon/managed-device-allocation/fence.ts new file mode 100644 index 000000000..8aab8ffd1 --- /dev/null +++ b/src/daemon/managed-device-allocation/fence.ts @@ -0,0 +1,22 @@ +import type { ResourceOwnershipFence } from '@agent-device/contracts/platform-runtime'; +import type { AllocationOperationRef } from './record.ts'; +import { isFenceGeneration, isVerbatimId } from './record-validation.ts'; + +export function allocationOperationFence( + ref: AllocationOperationRef, + generation: number, +): ResourceOwnershipFence { + if ( + !isVerbatimId(ref.requesterId) || + !isVerbatimId(ref.attemptKey) || + !isFenceGeneration(generation) + ) { + throw new TypeError( + 'Allocation operation fence requires canonical ids and a non-negative generation', + ); + } + return Object.freeze({ + token: JSON.stringify([ref.requesterId, ref.attemptKey]), + generation, + }); +} diff --git a/src/daemon/managed-device-allocation/journal-action-binding.ts b/src/daemon/managed-device-allocation/journal-action-binding.ts new file mode 100644 index 000000000..df83c18f6 --- /dev/null +++ b/src/daemon/managed-device-allocation/journal-action-binding.ts @@ -0,0 +1,83 @@ +import type { AllocationAction } from './decision.ts'; +import type { AllocationOperationRecord } from './record.ts'; +import type { AllocationJournalResult } from './journal-types.ts'; +import { abandoned, blocked, cleanupPending, errorMessage, uncertain } from './journal-results.ts'; +import type { AllocationActionRunnerContext } from './journal-action-context.ts'; +import { transition } from './journal-action-persistence.ts'; + +export async function publishBinding( + options: AllocationActionRunnerContext, + record: AllocationOperationRecord, + action: Extract, + signal?: AbortSignal, +): Promise { + if (!options.binding) { + return blocked('binding-unavailable', 'managed binding publisher is not configured', record); + } + if (record.binding !== 'unpublished') return options.project(record); + const pending = await transition(options, record, { kind: 'binding-publish-pending' }); + if (pending.status !== 'stored') return pending; + if (signal?.aborted) return abandoned(pending.record); + try { + await options.binding.publish(action.binding); + } catch (error) { + return persistCleanupFailure(options, pending.record, error); + } + const published = await transition(options, pending.record, { kind: 'binding-published' }); + if (published.status !== 'stored') return published; + return signal?.aborted ? abandoned(published.record) : options.project(published.record); +} + +export async function cleanupBinding( + options: AllocationActionRunnerContext, + record: AllocationOperationRecord, + action: Extract, +): Promise { + if (record.binding === 'unpublished') { + const cleaned = await transition(options, record, { kind: 'binding-cleaned' }); + return cleaned.status === 'stored' ? releaseLease(options, cleaned.record) : cleaned; + } + if (!options.binding) { + return blocked('binding-unavailable', 'managed binding cleaner is not configured', record); + } + try { + await options.binding.cleanup(action.binding); + } catch (error) { + return persistCleanupFailure(options, record, error); + } + const cleaned = await transition(options, record, { kind: 'binding-cleaned' }); + return cleaned.status === 'stored' ? releaseLease(options, cleaned.record) : cleaned; +} + +export async function releaseLease( + options: AllocationActionRunnerContext, + record: AllocationOperationRecord, +): Promise { + if (record.phase.status !== 'granted') return options.project(record); + const pending = await transition(options, record, { kind: 'release-pending' }); + if (pending.status !== 'stored') return pending; + if (pending.record.phase.status !== 'granted') { + return blocked( + 'invalid-transition', + 'allocator release lost its granted lease', + pending.record, + ); + } + try { + await options.allocator.releaseLease({ leaseId: pending.record.phase.lease.id }); + } catch (error) { + return uncertain(pending.record, 'release-uncertain', errorMessage(error)); + } + const released = await transition(options, pending.record, { kind: 'allocator-released' }); + return released.status === 'stored' ? options.project(released.record) : released; +} + +async function persistCleanupFailure( + options: AllocationActionRunnerContext, + record: AllocationOperationRecord, + error: unknown, +): Promise { + const message = errorMessage(error); + const pending = await transition(options, record, { kind: 'binding-cleanup-pending', message }); + return pending.status === 'stored' ? cleanupPending(pending.record, message) : pending; +} diff --git a/src/daemon/managed-device-allocation/journal-action-context.ts b/src/daemon/managed-device-allocation/journal-action-context.ts new file mode 100644 index 000000000..a85a8b70b --- /dev/null +++ b/src/daemon/managed-device-allocation/journal-action-context.ts @@ -0,0 +1,22 @@ +import type { ManagedDeviceAllocatorPort } from '@agent-device/contracts/managed-device-allocation'; +import type { AllocationDecisionMode } from './decision.ts'; +import type { AllocationOperationRecord } from './record.ts'; +import type { + AllocationBindingHooks, + AllocationJournalResult, + JournalContext, +} from './journal-types.ts'; +import type { AllocationOperationStore } from './store.ts'; + +export type AllocationActionRunnerContext = Readonly<{ + allocator: ManagedDeviceAllocatorPort; + binding?: AllocationBindingHooks; + store: AllocationOperationStore; + now: () => number; + execute( + record: AllocationOperationRecord, + mode: AllocationDecisionMode, + context: JournalContext, + ): Promise; + project(record: AllocationOperationRecord): AllocationJournalResult; +}>; diff --git a/src/daemon/managed-device-allocation/journal-action-persistence.ts b/src/daemon/managed-device-allocation/journal-action-persistence.ts new file mode 100644 index 000000000..77d352c39 --- /dev/null +++ b/src/daemon/managed-device-allocation/journal-action-persistence.ts @@ -0,0 +1,69 @@ +import { AppError } from '@agent-device/kernel/errors'; +import type { AllocationOperationRecord, AllocationTransition } from './record.ts'; +import type { AllocationJournalResult, PersistedTransition } from './journal-types.ts'; +import type { AllocationOperationStore } from './store.ts'; +import { blocked, errorMessage, uncertain, unreadableResult } from './journal-results.ts'; +import type { AllocationActionRunnerContext } from './journal-action-context.ts'; + +export async function transition( + options: AllocationActionRunnerContext, + record: AllocationOperationRecord, + transitionInput: AllocationTransition, +): Promise { + try { + const result = await options.store.transition( + { requesterId: record.requesterId, attemptKey: record.attemptKey }, + record.fence, + transitionInput, + options.now(), + ); + return mapTransitionResult(result, record); + } catch (error) { + return transitionError(record, error); + } +} + +function mapTransitionResult( + result: Awaited>, + record: AllocationOperationRecord, +): PersistedTransition { + if ( + result.status === 'recorded' || + result.status === 'already-applied' || + result.status === 'already-terminal' + ) { + return { status: 'stored', record: result.record }; + } + if (result.status === 'fence-lost') { + return blocked( + 'fence-lost', + 'allocation operation was changed by another writer', + result.current, + ); + } + if (result.status === 'missing') { + return blocked('operation-missing', 'allocation operation record is missing', record); + } + if (result.status === 'unreadable') return unreadableResult(result); + return blocked('persistence-failed', 'allocation transition did not produce a result', record); +} + +function transitionError(record: AllocationOperationRecord, error: unknown): PersistedTransition { + if (error instanceof AppError && error.details?.reason === 'transition-invalid') { + return blocked('invalid-transition', error.message, record); + } + return blocked('persistence-failed', errorMessage(error), record); +} + +export async function unknownAfterError( + options: AllocationActionRunnerContext, + record: AllocationOperationRecord, + error: unknown, +): Promise { + const message = errorMessage(error); + const persisted = await transition(options, record, { kind: 'allocator-unknown', message }); + if (persisted.status !== 'stored') return persisted; + return persisted.record.phase.status === 'unknown' + ? uncertain(persisted.record, 'allocator-uncertain', message) + : options.project(persisted.record); +} diff --git a/src/daemon/managed-device-allocation/journal-action-runtime.ts b/src/daemon/managed-device-allocation/journal-action-runtime.ts new file mode 100644 index 000000000..031170311 --- /dev/null +++ b/src/daemon/managed-device-allocation/journal-action-runtime.ts @@ -0,0 +1,159 @@ +import type { LeaseRequestStatus } from '@agent-device/contracts/managed-device-allocation'; +import type { AllocationDecisionMode } from './decision.ts'; +import type { AllocationOperationRecord } from './record.ts'; +import type { + AllocationJournalActionRunner, + AllocationJournalResult, + JournalContext, +} from './journal-types.ts'; +import { abandoned, blocked, uncertain } from './journal-results.ts'; +import type { AllocationActionRunnerContext } from './journal-action-context.ts'; +import { cleanupBinding, publishBinding, releaseLease } from './journal-action-binding.ts'; +import { transition, unknownAfterError } from './journal-action-persistence.ts'; + +type ActionRunnerOptions = AllocationActionRunnerContext; + +export function createAllocationJournalActionRunner( + options: ActionRunnerOptions, +): AllocationJournalActionRunner { + return Object.freeze({ + request: (record, context) => requestAllocation(options, record, context), + supersede: (record, context) => supersedeAllocation(options, record, context), + lookup: (record, mode, signal) => lookupAllocation(options, record, mode, signal), + cancel: (record) => cancelAllocation(options, record), + publish: (record, action, signal) => publishBinding(options, record, action, signal), + cleanup: (record, action) => cleanupBinding(options, record, action), + release: (record) => releaseLease(options, record), + }); +} + +function requestAllocation( + options: ActionRunnerOptions, + record: AllocationOperationRecord, + context: JournalContext, +): Promise { + const input = context.requestInput; + return input + ? dispatchRequest(options, record, context.signal, () => options.allocator.requestLease(input)) + : Promise.resolve( + blocked('invalid-transition', 'allocation request input is unavailable', record), + ); +} + +function supersedeAllocation( + options: ActionRunnerOptions, + record: AllocationOperationRecord, + context: JournalContext, +): Promise { + const input = context.supersedeInput; + return input + ? dispatchRequest(options, record, context.signal, () => + options.allocator.supersedeLeaseRequest(input), + ) + : Promise.resolve(blocked('invalid-transition', 'supersession input is unavailable', record)); +} + +function lookupAllocation( + options: ActionRunnerOptions, + record: AllocationOperationRecord, + mode: AllocationDecisionMode, + signal?: AbortSignal, +): Promise { + return options.allocator + .getLeaseRequestStatus({ requesterId: record.requesterId, attemptKey: record.attemptKey }) + .then((status) => + settleAllocatorStatus( + options, + record, + status, + mode === 'release' ? 'release' : 'recover', + signal, + ), + ) + .catch((error) => allocatorError(options, record, error, signal)); +} + +function cancelAllocation( + options: ActionRunnerOptions, + record: AllocationOperationRecord, +): Promise { + return options.allocator + .cancelLeaseRequest({ requesterId: record.requesterId, attemptKey: record.attemptKey }) + .then((status) => settleAllocatorStatus(options, record, status, 'cancel')) + .catch((error) => unknownAfterError(options, record, error)); +} + +function dispatchRequest( + options: ActionRunnerOptions, + record: AllocationOperationRecord, + signal: AbortSignal | undefined, + dispatch: () => Promise, +): Promise { + return transition(options, record, { kind: 'request-dispatched' }).then((dispatched) => { + if (dispatched.status !== 'stored') return dispatched; + if (signal?.aborted) return abandoned(dispatched.record); + return dispatch() + .then((status) => + settleAllocatorStatus(options, dispatched.record, status, 'continue', signal), + ) + .catch((error) => allocatorError(options, dispatched.record, error, signal)); + }); +} + +function allocatorError( + options: ActionRunnerOptions, + record: AllocationOperationRecord, + error: unknown, + signal?: AbortSignal, +): Promise { + return signal?.aborted + ? Promise.resolve(abandoned(record)) + : unknownAfterError(options, record, error); +} + +async function settleAllocatorStatus( + options: ActionRunnerOptions, + record: AllocationOperationRecord, + status: LeaseRequestStatus, + followUp: 'continue' | 'recover' | 'release' | 'cancel', + signal?: AbortSignal, +): Promise { + const persisted = await transition(options, record, { kind: 'allocator-status', status }); + if (persisted.status !== 'stored') return persisted; + return settlePersistedStatus(options, persisted.record, followUp, signal); +} + +function settlePersistedStatus( + options: ActionRunnerOptions, + record: AllocationOperationRecord, + followUp: 'continue' | 'recover' | 'release' | 'cancel', + signal?: AbortSignal, +): Promise { + if (signal?.aborted) return Promise.resolve(abandoned(record)); + if (record.phase.status === 'unknown') { + return Promise.resolve(uncertain(record, 'allocator-uncertain', 'allocator status is unknown')); + } + if (record.phase.status === 'ambiguous') { + return Promise.resolve( + blocked('ambiguous-state', 'allocator status could not be matched to this operation', record), + ); + } + if (followUp === 'cancel') return settleCancellation(options, record); + if (record.phase.status === 'pending') return Promise.resolve(options.project(record)); + return options.execute(record, followUp === 'release' ? 'release' : 'continue', { signal }); +} + +function settleCancellation( + options: ActionRunnerOptions, + record: AllocationOperationRecord, +): Promise { + return record.phase.status === 'granted' + ? Promise.resolve( + blocked( + 'already-granted', + 'allocator cancellation arrived after the lease was granted', + record, + ), + ) + : Promise.resolve(options.project(record)); +} diff --git a/src/daemon/managed-device-allocation/journal-lane.ts b/src/daemon/managed-device-allocation/journal-lane.ts new file mode 100644 index 000000000..2af40900f --- /dev/null +++ b/src/daemon/managed-device-allocation/journal-lane.ts @@ -0,0 +1,273 @@ +import { isDeepStrictEqual } from 'node:util'; +import type { + LeaseRequestInput, + ManagedDeviceAllocatorPort, + SupersedeLeaseRequestInput, +} from '@agent-device/contracts/managed-device-allocation'; +import { newAllocationOperation } from './record-factory.ts'; +import type { AllocationOperationRecord } from './record.ts'; +import type { AllocationJournalResult } from './journal-types.ts'; +import { blocked, errorMessage, unreadableResult } from './journal-results.ts'; +import type { AllocationOperationStore } from './store.ts'; + +export type AllocationJournalLane = Readonly<{ + open( + input: LeaseRequestInput, + allowedNonterminalLane?: AllocationOperationRecord, + ): Readonly<{ record: AllocationOperationRecord; created: boolean }> | AllocationJournalResult; + expectedPrior( + input: SupersedeLeaseRequestInput, + ): Readonly<{ record: AllocationOperationRecord }> | AllocationJournalResult; + withLock( + requesterId: string, + task: () => Promise, + ): Promise; +}>; + +type LaneDependencies = Readonly<{ + allocator: ManagedDeviceAllocatorPort; + store: AllocationOperationStore; + now: () => number; +}>; + +export function createAllocationJournalLane(options: LaneDependencies): AllocationJournalLane { + return Object.freeze({ + open: createOpen(options), + expectedPrior: createExpectedPrior(options.store), + withLock: createLaneLock(options.store), + }); +} + +function createOpen(dependencies: LaneDependencies): AllocationJournalLane['open'] { + return (input, allowedNonterminalLane) => + openLaneOperation(dependencies, input, allowedNonterminalLane); +} + +function createExpectedPrior( + store: AllocationOperationStore, +): AllocationJournalLane['expectedPrior'] { + return (input) => expectedPriorOperation(store, input); +} + +function createLaneLock(store: AllocationOperationStore): AllocationJournalLane['withLock'] { + return (requesterId, task) => + store.withLaneLock(requesterId, task).catch((error) => { + return blocked('persistence-failed', errorMessage(error)); + }); +} + +function openLaneOperation( + dependencies: LaneDependencies, + input: LeaseRequestInput, + allowedNonterminalLane?: AllocationOperationRecord, +): Readonly<{ record: AllocationOperationRecord; created: boolean }> | AllocationJournalResult { + const candidate = buildCandidate(dependencies, input); + if (isJournalResult(candidate)) return candidate; + const existing = resolveExistingCandidate( + dependencies.store.read(candidate), + input, + dependencies.allocator.instanceId, + ); + if (existing) return existing; + const conflict = laneConflict( + dependencies.store, + input.requesterId, + input.requestGeneration, + allowedNonterminalLane, + ); + if (conflict) return conflict; + return persistCandidate(dependencies.store, candidate, input, dependencies.allocator.instanceId); +} + +function buildCandidate( + dependencies: LaneDependencies, + input: LeaseRequestInput, +): AllocationOperationRecord | AllocationJournalResult { + try { + return newAllocationOperation({ + requesterId: input.requesterId, + requestGeneration: input.requestGeneration, + attemptKey: input.attemptKey, + allocatorInstanceId: dependencies.allocator.instanceId, + shape: input.shape, + deadlineAtMs: input.deadlineAtMs, + admission: input.admission, + activation: input.activation, + ...(input.attribution === undefined ? {} : { attribution: input.attribution }), + nowMs: dependencies.now(), + }); + } catch (error) { + return blocked('payload-mismatch', errorMessage(error)); + } +} + +function resolveExistingCandidate( + existing: ReturnType, + input: LeaseRequestInput, + allocatorInstanceId: string, +): + | Readonly<{ record: AllocationOperationRecord; created: boolean }> + | AllocationJournalResult + | undefined { + if (existing.status === 'unreadable') return unreadableResult(existing); + if (existing.status === 'missing') return undefined; + return sameRequest(existing.record, input, allocatorInstanceId) + ? { record: existing.record, created: false } + : mismatchedCandidate(existing.record); +} + +function persistCandidate( + store: AllocationOperationStore, + candidate: AllocationOperationRecord, + input: LeaseRequestInput, + allocatorInstanceId: string, +): Readonly<{ record: AllocationOperationRecord; created: boolean }> | AllocationJournalResult { + let created: ReturnType; + try { + created = store.create(candidate); + } catch (error) { + return blocked('persistence-failed', errorMessage(error)); + } + if (created.status === 'unreadable') return unreadableResult(created); + return sameRequest(created.record, input, allocatorInstanceId) + ? { record: created.record, created: created.status === 'created' } + : mismatchedCandidate(created.record); +} + +function mismatchedCandidate(record: AllocationOperationRecord): AllocationJournalResult { + return blocked( + 'payload-mismatch', + 'allocation attempt key already names a different request', + record, + ); +} + +function expectedPriorOperation( + store: AllocationOperationStore, + input: SupersedeLeaseRequestInput, +): Readonly<{ record: AllocationOperationRecord }> | AllocationJournalResult { + if (!Number.isInteger(input.expectedRequestGeneration) || input.expectedRequestGeneration < 1) { + return blocked('payload-mismatch', 'supersession generation is invalid'); + } + const matches = findExpectedPrior(store, input); + if (!Array.isArray(matches)) return matches; + return resolveExpectedPrior(matches); +} + +function findExpectedPrior( + store: AllocationOperationStore, + input: SupersedeLeaseRequestInput, +): AllocationOperationRecord[] | AllocationJournalResult { + const matches: AllocationOperationRecord[] = []; + for (const entry of store.list()) { + if (entry.status === 'unreadable') return unreadableResult(entry); + if ( + entry.status === 'found' && + entry.record.requesterId === input.requesterId && + entry.record.requestGeneration === input.expectedRequestGeneration + ) { + matches.push(entry.record); + } + } + return matches; +} + +function resolveExpectedPrior( + matches: AllocationOperationRecord[], +): Readonly<{ record: AllocationOperationRecord }> | AllocationJournalResult { + if (matches.length === 0) { + return blocked( + 'operation-missing', + 'supersession expected generation has no durable operation', + ); + } + if (matches.length > 1) { + return blocked('ambiguous-state', 'supersession expected generation names multiple operations'); + } + const record = matches[0]!; + return holdsAllocationLane(record) + ? { record } + : blocked('invalid-transition', 'supersession expected generation is terminal', record); +} + +type LaneEntryInspection = + | Readonly<{ highestGeneration: number; conflict?: AllocationOperationRecord }> + | AllocationJournalResult; + +function laneConflict( + store: AllocationOperationStore, + requesterId: string, + requestGeneration: number, + allowedNonterminalLane?: AllocationOperationRecord, +): AllocationJournalResult | undefined { + let highestGeneration = 0; + for (const entry of store.list()) { + const inspection = inspectLaneEntry(entry, requesterId, allowedNonterminalLane); + if (isJournalResult(inspection)) return inspection; + highestGeneration = Math.max(highestGeneration, inspection.highestGeneration); + if (inspection.conflict) return laneBusy(inspection.conflict); + } + return requestGeneration <= highestGeneration + ? blocked( + 'payload-mismatch', + 'allocation request generation is not newer than durable lane history', + ) + : undefined; +} + +function inspectLaneEntry( + entry: ReturnType[number], + requesterId: string, + allowedNonterminalLane?: AllocationOperationRecord, +): LaneEntryInspection { + if (entry.status === 'unreadable') return unreadableResult(entry); + if (entry.status === 'missing' || entry.record.requesterId !== requesterId) { + return { highestGeneration: 0 }; + } + const allowed = allowedNonterminalLane?.attemptKey === entry.record.attemptKey; + return { + highestGeneration: entry.record.requestGeneration, + ...(holdsAllocationLane(entry.record) && !allowed ? { conflict: entry.record } : {}), + }; +} + +function laneBusy(record: AllocationOperationRecord): AllocationJournalResult { + return blocked( + 'lane-busy', + 'requester lane has an allocation operation that may still mutate the allocator', + record, + ); +} + +function isJournalResult(value: unknown): value is AllocationJournalResult { + return typeof value === 'object' && value !== null && 'status' in value; +} + +function sameRequest( + record: AllocationOperationRecord, + input: LeaseRequestInput, + allocatorInstanceId: string, +): boolean { + return ( + record.allocatorInstanceId === allocatorInstanceId && + record.requesterId === input.requesterId && + record.attemptKey === input.attemptKey && + record.requestGeneration === input.requestGeneration && + record.deadlineAtMs === input.deadlineAtMs && + record.admission === input.admission && + record.activation === input.activation && + isDeepStrictEqual(record.shape, input.shape) && + isDeepStrictEqual(record.attribution, input.attribution) + ); +} + +function holdsAllocationLane(record: AllocationOperationRecord): boolean { + if (record.phase.status !== 'granted') { + return ( + record.phase.status === 'unresolved' || + record.phase.status === 'pending' || + record.phase.status === 'unknown' + ); + } + return record.release !== 'released'; +} diff --git a/src/daemon/managed-device-allocation/journal-projection.ts b/src/daemon/managed-device-allocation/journal-projection.ts new file mode 100644 index 000000000..0da3e57fc --- /dev/null +++ b/src/daemon/managed-device-allocation/journal-projection.ts @@ -0,0 +1,42 @@ +import type { AllocationOperationRecord } from './record.ts'; +import type { AllocationJournalResult } from './journal-types.ts'; +import { blocked, uncertain } from './journal-results.ts'; + +export function projectAllocationRecord( + record: AllocationOperationRecord, +): AllocationJournalResult { + if (record.phase.status === 'unresolved' || record.phase.status === 'pending') { + return { status: 'pending', record }; + } + if (record.phase.status === 'unknown') { + return uncertain(record, 'allocator-uncertain', record.phase.message); + } + if (record.phase.status === 'ambiguous') { + return blocked('ambiguous-state', record.phase.message, record); + } + if (record.phase.status === 'granted') return projectGrantedRecord(record); + return projectTerminalRecord(record); +} + +function projectGrantedRecord(record: AllocationOperationRecord): AllocationJournalResult { + return record.release === 'released' + ? { status: 'released', record } + : { status: 'granted', record }; +} + +function projectTerminalRecord(record: AllocationOperationRecord): AllocationJournalResult { + switch (record.phase.status) { + case 'refused': + return { status: 'refused', record }; + case 'superseded': + return { status: 'superseded', record }; + case 'cancelled': + return { status: 'cancelled', record }; + default: + return blocked( + 'ambiguous-state', + 'allocation operation has an unsupported terminal state', + record, + ); + } +} diff --git a/src/daemon/managed-device-allocation/journal-results.ts b/src/daemon/managed-device-allocation/journal-results.ts new file mode 100644 index 000000000..89fbbcd8f --- /dev/null +++ b/src/daemon/managed-device-allocation/journal-results.ts @@ -0,0 +1,48 @@ +import type { AllocationOperationRecord } from './record.ts'; +import type { AllocationOperationUnreadable } from './store.ts'; +import type { + AllocationJournalBlockReason, + AllocationJournalReason, + AllocationJournalResult, +} from './journal-types.ts'; + +export function blocked( + reason: AllocationJournalBlockReason, + message: string, + record?: AllocationOperationRecord, +): AllocationJournalResult { + return { status: 'blocked', reason, message, ...(record === undefined ? {} : { record }) }; +} + +export function abandoned(record: AllocationOperationRecord): AllocationJournalResult { + return { status: 'abandoned', record }; +} + +export function uncertain( + record: AllocationOperationRecord, + reason: AllocationJournalReason, + message: string, +): AllocationJournalResult { + return { status: 'uncertain', record, reason, message }; +} + +export function cleanupPending( + record: AllocationOperationRecord, + message: string, +): AllocationJournalResult { + return { status: 'cleanup-pending', record, reason: 'cleanup-uncertain', message }; +} + +export function unreadableResult(result: AllocationOperationUnreadable): AllocationJournalResult { + return { + status: 'unreadable', + path: result.path, + reason: result.reason, + message: result.message, + ...(result.version === undefined ? {} : { version: result.version }), + }; +} + +export function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/src/daemon/managed-device-allocation/journal-types.ts b/src/daemon/managed-device-allocation/journal-types.ts new file mode 100644 index 000000000..21c7acb0a --- /dev/null +++ b/src/daemon/managed-device-allocation/journal-types.ts @@ -0,0 +1,96 @@ +import type { + LeaseRequestInput, + SupersedeLeaseRequestInput, +} from '@agent-device/contracts/managed-device-allocation'; +import type { AllocationAction, AllocationBinding, AllocationDecisionMode } from './decision.ts'; +import type { AllocationOperationRecord } from './record.ts'; +import type { AllocationOperationUnreadable } from './store.ts'; + +export type AllocationBindingHooks = Readonly<{ + publish(binding: AllocationBinding): Promise; + cleanup(binding: AllocationBinding): Promise; +}>; + +export type AllocationJournalBlockReason = + | 'ambiguous-state' + | 'already-granted' + | 'binding-unavailable' + | 'fence-lost' + | 'invalid-transition' + | 'operation-missing' + | 'payload-mismatch' + | 'persistence-failed' + | 'lane-busy' + | 'not-releasable'; + +export type AllocationJournalReason = + | 'allocator-uncertain' + | 'cleanup-uncertain' + | 'release-uncertain'; + +export type AllocationJournalResult = + | Readonly<{ + status: + | 'pending' + | 'abandoned' + | 'uncertain' + | 'granted' + | 'refused' + | 'superseded' + | 'cancelled' + | 'cleanup-pending' + | 'released'; + record: AllocationOperationRecord; + reason?: AllocationJournalReason; + message?: string; + }> + | Readonly<{ + status: 'blocked'; + record?: AllocationOperationRecord; + reason: AllocationJournalBlockReason; + message: string; + }> + | Readonly<{ + status: 'unreadable'; + path: string; + reason: AllocationOperationUnreadable['reason']; + message: string; + version?: number; + }>; + +export type JournalContext = Readonly<{ + signal?: AbortSignal; + requestInput?: LeaseRequestInput; + supersedeInput?: SupersedeLeaseRequestInput; +}>; + +export type PersistedTransition = + | Readonly<{ status: 'stored'; record: AllocationOperationRecord }> + | AllocationJournalResult; + +export type AllocationJournalActionRunner = Readonly<{ + request( + record: AllocationOperationRecord, + context: JournalContext, + ): Promise; + supersede( + record: AllocationOperationRecord, + context: JournalContext, + ): Promise; + lookup( + record: AllocationOperationRecord, + mode: AllocationDecisionMode, + signal?: AbortSignal, + ): Promise; + cancel(record: AllocationOperationRecord): Promise; + publish( + record: AllocationOperationRecord, + action: Extract, + signal?: AbortSignal, + ): Promise; + cleanup( + record: AllocationOperationRecord, + action: Extract, + ): Promise; + release(record: AllocationOperationRecord): Promise; +}>; diff --git a/src/daemon/managed-device-allocation/journal.ts b/src/daemon/managed-device-allocation/journal.ts new file mode 100644 index 000000000..e71818bc1 --- /dev/null +++ b/src/daemon/managed-device-allocation/journal.ts @@ -0,0 +1,290 @@ +import type { + LeaseRequestInput, + ManagedDeviceAllocatorPort, + SupersedeLeaseRequestInput, +} from '@agent-device/contracts/managed-device-allocation'; +import type { AllocationAction, AllocationDecisionMode } from './decision.ts'; +import { decideAllocationAction } from './decision.ts'; +import type { AllocationOperationRecord, AllocationOperationRef } from './record.ts'; +import type { AllocationJournalLane } from './journal-lane.ts'; +import { createAllocationJournalLane } from './journal-lane.ts'; +import { createAllocationJournalActionRunner } from './journal-action-runtime.ts'; +import { projectAllocationRecord } from './journal-projection.ts'; +import type { + AllocationJournalActionRunner, + AllocationBindingHooks, + AllocationJournalResult, + JournalContext, +} from './journal-types.ts'; +import type { AllocationOperationRead, AllocationOperationStore } from './store.ts'; +import { abandoned, blocked, unreadableResult } from './journal-results.ts'; + +export type { AllocationBindingHooks, AllocationJournalResult } from './journal-types.ts'; + +export type AllocationOperationJournal = Readonly<{ + allocate(input: LeaseRequestInput): Promise; + recover(ref: AllocationOperationRef): Promise; + cancel(ref: AllocationOperationRef): Promise; + release(ref: AllocationOperationRef): Promise; + supersede(input: SupersedeLeaseRequestInput): Promise; + read(ref: AllocationOperationRef): AllocationOperationRead; + list(): AllocationOperationRead[]; +}>; + +type JournalSetup = Readonly<{ + store: AllocationOperationStore; + allocator: ManagedDeviceAllocatorPort; + binding?: AllocationBindingHooks; + now: () => number; + lane: AllocationJournalLane; +}>; + +type ExecuteOperation = ( + record: AllocationOperationRecord, + mode: AllocationDecisionMode, + context: JournalContext, +) => Promise; + +type JournalServices = JournalSetup & + Readonly<{ + actions: AllocationJournalActionRunner; + execute: ExecuteOperation; + project(record: AllocationOperationRecord): AllocationJournalResult; + }>; + +export function createAllocationOperationJournal( + options: Readonly<{ + store: AllocationOperationStore; + allocator: ManagedDeviceAllocatorPort; + binding?: AllocationBindingHooks; + now?: () => number; + }>, +): AllocationOperationJournal { + const setup: JournalSetup = { + ...options, + now: options.now ?? Date.now, + lane: createAllocationJournalLane({ + allocator: options.allocator, + store: options.store, + now: options.now ?? Date.now, + }), + }; + const project = projectAllocationRecord; + function execute( + record: AllocationOperationRecord, + mode: AllocationDecisionMode, + context: JournalContext, + ): Promise { + return executeOperation({ ...setup, actions, execute, project }, record, mode, context); + } + + const actions: AllocationJournalActionRunner = createAllocationJournalActionRunner({ + allocator: setup.allocator, + binding: setup.binding, + store: setup.store, + now: setup.now, + execute, + project, + }); + return createJournalServices({ ...setup, actions, execute, project }); +} + +function createJournalServices(services: JournalServices): AllocationOperationJournal { + return Object.freeze({ + allocate: (input) => allocateOperation(services, input), + recover: (ref) => referenceOperation(services, ref, 'recover'), + cancel: (ref) => referenceOperation(services, ref, 'cancel'), + release: (ref) => referenceOperation(services, ref, 'release'), + supersede: (input) => supersedeOperation(services, input), + read: (ref) => services.store.read(ref), + list: () => services.store.list(), + }); +} + +function allocateOperation( + services: JournalServices, + input: LeaseRequestInput, +): Promise { + return services.lane.withLock(input.requesterId, async () => { + const opened = services.lane.open(input); + if ('status' in opened) return opened; + return services.execute(opened.record, 'new', { + signal: input.signal, + requestInput: input, + }); + }); +} + +function referenceOperation( + services: JournalServices, + ref: AllocationOperationRef, + mode: Extract, +): Promise { + return services.lane.withLock(ref.requesterId, async () => { + const current = readRequired(services.store, ref); + if ('status' in current) return current; + return services.execute(current.record, mode, {}); + }); +} + +function supersedeOperation( + services: JournalServices, + input: SupersedeLeaseRequestInput, +): Promise { + const invalid = validateSupersessionInput(input); + if (invalid) return Promise.resolve(invalid); + return services.lane.withLock(input.requesterId, async () => { + const prior = services.lane.expectedPrior(input); + if ('status' in prior) return prior; + const allocatorCheck = checkAllocatorInstance(services.allocator, prior.record); + if (allocatorCheck) return allocatorCheck; + const liveBinding = liveBindingSupersession(prior.record, input); + if (liveBinding) return liveBinding; + const opened = services.lane.open(input.replacement, prior.record); + if ('status' in opened) return opened; + return services.execute(opened.record, 'supersede', { + signal: input.replacement.signal, + supersedeInput: input, + }); + }); +} + +function validateSupersessionInput( + input: SupersedeLeaseRequestInput, +): AllocationJournalResult | undefined { + if (input.replacement.requesterId !== input.requesterId) { + return blocked('payload-mismatch', 'supersession requester does not match its replacement'); + } + return input.replacement.requestGeneration > input.expectedRequestGeneration && + Number.isInteger(input.replacement.requestGeneration) + ? undefined + : blocked('payload-mismatch', 'supersession replacement generation is not newer'); +} + +function liveBindingSupersession( + prior: AllocationOperationRecord, + input: SupersedeLeaseRequestInput, +): AllocationJournalResult | undefined { + if (prior.attemptKey === input.replacement.attemptKey) { + return blocked('payload-mismatch', 'supersession must use a new attempt key', prior); + } + if (prior.phase.status !== 'granted') return undefined; + return blocked( + prior.release === 'released' ? 'invalid-transition' : 'already-granted', + 'supersession cannot revoke a live managed binding in this slice', + prior, + ); +} + +function executeOperation( + services: JournalServices, + record: AllocationOperationRecord, + mode: AllocationDecisionMode, + context: JournalContext, +): Promise { + if (context.signal?.aborted) return Promise.resolve(abandoned(record)); + const allocatorCheck = checkAllocatorInstance(services.allocator, record); + if (allocatorCheck) return Promise.resolve(allocatorCheck); + return executeAction( + services.actions, + services.project, + record, + decideAllocationAction(record, mode), + context, + mode, + ); +} + +type AllocatorAction = Extract< + AllocationAction, + { kind: 'request' | 'supersede' | 'lookup' | 'cancel' } +>; +type BindingAction = Extract; + +function executeAction( + actions: AllocationJournalActionRunner, + project: (record: AllocationOperationRecord) => AllocationJournalResult, + record: AllocationOperationRecord, + action: AllocationAction, + context: JournalContext, + mode: AllocationDecisionMode, +): Promise { + if (isAllocatorAction(action)) + return executeAllocatorAction(actions, record, action, context, mode); + if (isBindingAction(action)) return executeBindingAction(actions, record, action, context); + if (action.kind === 'pending' || action.kind === 'terminal') + return Promise.resolve(project(record)); + return Promise.resolve(blocked(action.reason, action.message, record)); +} + +function executeAllocatorAction( + actions: AllocationJournalActionRunner, + record: AllocationOperationRecord, + action: AllocatorAction, + context: JournalContext, + mode: AllocationDecisionMode, +): Promise { + switch (action.kind) { + case 'request': + return actions.request(record, context); + case 'supersede': + return actions.supersede(record, context); + case 'lookup': + return actions.lookup(record, mode, context.signal); + case 'cancel': + return actions.cancel(record); + } +} + +function executeBindingAction( + actions: AllocationJournalActionRunner, + record: AllocationOperationRecord, + action: BindingAction, + context: JournalContext, +): Promise { + switch (action.kind) { + case 'publish': + return actions.publish(record, action, context.signal); + case 'cleanup': + return actions.cleanup(record, action); + case 'release': + return actions.release(record); + } +} + +function isAllocatorAction(action: AllocationAction): action is AllocatorAction { + return ( + action.kind === 'request' || + action.kind === 'supersede' || + action.kind === 'lookup' || + action.kind === 'cancel' + ); +} + +function isBindingAction(action: AllocationAction): action is BindingAction { + return action.kind === 'publish' || action.kind === 'cleanup' || action.kind === 'release'; +} + +function checkAllocatorInstance( + allocator: ManagedDeviceAllocatorPort, + record: AllocationOperationRecord, +): AllocationJournalResult | undefined { + return record.allocatorInstanceId === allocator.instanceId + ? undefined + : blocked( + 'payload-mismatch', + 'allocation operation belongs to a different allocator instance', + record, + ); +} + +function readRequired( + store: AllocationOperationStore, + ref: AllocationOperationRef, +): Readonly<{ record: AllocationOperationRecord }> | AllocationJournalResult { + const read = store.read(ref); + if (read.status === 'found') return { record: read.record }; + if (read.status === 'missing') + return blocked('operation-missing', 'allocation operation record is missing'); + return unreadableResult(read); +} diff --git a/src/daemon/managed-device-allocation/record-codec-state.ts b/src/daemon/managed-device-allocation/record-codec-state.ts new file mode 100644 index 000000000..aff7f9179 --- /dev/null +++ b/src/daemon/managed-device-allocation/record-codec-state.ts @@ -0,0 +1,158 @@ +import type { + LeaseRefusal, + ManagedLease, + ManagedShapeRequest, +} from '@agent-device/contracts/managed-device-allocation'; +import type { AllocationOperationPhase, AllocationOperationRecord } from './record.ts'; +import { + isFiniteNumber, + isNonEmptyString, + isPlainObject, + isVerbatimId, +} from './record-validation.ts'; + +export function decodePhase(value: unknown): AllocationOperationPhase | null { + if (!isPlainObject(value)) return null; + const simple = decodeSimplePhase(value.status); + if (simple) return simple; + if (value.status === 'unknown') return decodeMessagePhase('unknown', value.message); + if (value.status === 'ambiguous') return decodeMessagePhase('ambiguous', value.message); + if (value.status === 'granted') return decodeGrantedPhase(value.lease); + if (value.status === 'refused') return decodeRefusedPhase(value.refusal); + return null; +} + +function decodeSimplePhase(value: unknown): AllocationOperationPhase | null { + if (value === 'unresolved') return { status: 'unresolved' }; + if (value === 'pending') return { status: 'pending' }; + if (value === 'superseded') return { status: 'superseded' }; + if (value === 'cancelled') return { status: 'cancelled' }; + return null; +} + +function decodeMessagePhase( + status: 'unknown' | 'ambiguous', + message: unknown, +): AllocationOperationPhase | null { + return isNonEmptyString(message) ? { status, message } : null; +} + +function decodeGrantedPhase(value: unknown): AllocationOperationPhase | null { + const lease = decodeLease(value); + return lease ? { status: 'granted', lease } : null; +} + +function decodeRefusedPhase(value: unknown): AllocationOperationPhase | null { + const refusal = decodeRefusal(value); + return refusal ? { status: 'refused', refusal } : null; +} + +type ConsistencyFields = { + phase: AllocationOperationPhase; + identityIncarnationId?: string; + binding: AllocationOperationRecord['binding']; + release: AllocationOperationRecord['release']; +}; + +export function checkConsistency(fields: ConsistencyFields): string | undefined { + return fields.phase.status === 'granted' + ? checkGrantedConsistency(fields) + : checkNonGrantedConsistency(fields); +} + +function checkGrantedConsistency(fields: ConsistencyFields): string | undefined { + if (fields.identityIncarnationId === undefined) { + return 'granted allocation record has no identity incarnation'; + } + if (fields.binding === 'not-applicable') return 'granted allocation record has no binding state'; + if ( + (fields.binding === 'published' || fields.binding === 'cleanup-pending') && + fields.release !== 'not-requested' + ) { + return 'allocator release is recorded before binding cleanup'; + } + if (fields.release !== 'not-requested' && fields.binding !== 'cleaned') { + return 'allocator release is recorded before binding cleanup'; + } + return undefined; +} + +function checkNonGrantedConsistency(fields: ConsistencyFields): string | undefined { + const expectedBinding = terminalPhase(fields.phase.status) ? 'not-applicable' : 'unpublished'; + if (fields.binding !== expectedBinding) return 'allocation record binding state is inconsistent'; + return fields.release === 'not-requested' + ? undefined + : 'non-granted allocation record claims allocator release'; +} + +function terminalPhase( + status: AllocationOperationPhase['status'], +): status is 'refused' | 'superseded' | 'cancelled' { + return status === 'refused' || status === 'superseded' || status === 'cancelled'; +} + +export function decodeShape(value: unknown): ManagedShapeRequest | null { + if (!isPlainObject(value)) return null; + if (value.platform !== 'ios' && value.platform !== 'android') return null; + if (!isNonEmptyString(value.deviceType)) return null; + if (value.osVersion !== undefined && !isNonEmptyString(value.osVersion)) return null; + return Object.freeze({ + platform: value.platform, + deviceType: value.deviceType, + ...(value.osVersion === undefined ? {} : { osVersion: value.osVersion }), + }); +} + +function decodeLease(value: unknown): ManagedLease | null { + if (!isPlainObject(value)) return null; + if (!isVerbatimId(value.id) || !isFiniteNumber(value.ttlDeadline)) return null; + if (!isPlainObject(value.device) || !isVerbatimId(value.device.address)) return null; + if (!isPlainObject(value.environment)) return null; + if (Object.values(value.environment).some((item) => typeof item !== 'string')) return null; + return Object.freeze({ + id: value.id, + ttlDeadline: value.ttlDeadline, + device: Object.freeze({ address: value.device.address }), + environment: Object.freeze({ ...value.environment }) as Record, + }); +} + +function decodeRefusal(value: unknown): LeaseRefusal | null { + if (!isPlainObject(value)) return null; + if ( + ![ + 'simulator-capacity', + 'disk-low', + 'invalid-shape', + 'preparation-required', + 'requester-busy', + ].includes(String(value.reason)) + ) + return null; + if (value.retryAfterMs !== undefined && !isFiniteNumber(value.retryAfterMs)) return null; + if (value.message !== undefined && !isNonEmptyString(value.message)) return null; + return Object.freeze({ + reason: value.reason, + ...(value.retryAfterMs === undefined ? {} : { retryAfterMs: value.retryAfterMs }), + ...(value.message === undefined ? {} : { message: value.message }), + }) as LeaseRefusal; +} + +export function decodeBinding(value: unknown): AllocationOperationRecord['binding'] | null { + return value === 'unpublished' || + value === 'publish-pending' || + value === 'published' || + value === 'cleanup-pending' || + value === 'cleaned' || + value === 'not-applicable' + ? value + : null; +} + +export function decodeRelease(value: unknown): AllocationOperationRecord['release'] | null { + return value === 'not-requested' || value === 'pending' || value === 'released' ? value : null; +} + +export function isActivation(value: unknown): value is 'direct' | 'external-fence' { + return value === 'direct' || value === 'external-fence'; +} diff --git a/src/daemon/managed-device-allocation/record-codec.ts b/src/daemon/managed-device-allocation/record-codec.ts new file mode 100644 index 000000000..443355a65 --- /dev/null +++ b/src/daemon/managed-device-allocation/record-codec.ts @@ -0,0 +1,240 @@ +import type { ManagedShapeRequest } from '@agent-device/contracts/managed-device-allocation'; +import type { JsonObject } from '@agent-device/contracts/client'; +import type { AllocationOperationPhase, AllocationOperationRecord } from './record.ts'; +import { allocationOperationFence } from './fence.ts'; +import { ALLOCATION_OPERATION_SCHEMA_VERSION } from './schema.ts'; +import { + isFenceGeneration, + isFiniteNumber, + isPlainObject, + isRequestGeneration, + isVerbatimId, +} from './record-validation.ts'; +import { decodeAllocationAttribution } from './record-json.ts'; +import { + checkConsistency, + decodeBinding, + decodePhase, + decodeRelease, + decodeShape, + isActivation, +} from './record-codec-state.ts'; + +export type AllocationOperationDecoding = + | Readonly<{ status: 'decoded'; record: AllocationOperationRecord }> + | AllocationOperationUnreadable; + +export type AllocationOperationUnreadable = Readonly<{ + status: 'unreadable'; + reason: AllocationOperationUnreadableReason; + message: string; + version?: number; +}>; + +export type AllocationOperationUnreadableReason = + | 'corrupt' + | 'unsupported-version' + | 'ambiguous' + | 'unfenced'; + +type DecodeResult = T | AllocationOperationUnreadable; +type RawAllocationRecord = Record; + +type DecodedBaseFields = Readonly<{ + requesterId: string; + attemptKey: string; + allocatorInstanceId: string; + shape: ManagedShapeRequest; + deadlineAtMs: number; + requestGeneration: number; + activation: 'direct' | 'external-fence'; + createdAtMs: number; + updatedAtMs: number; + attribution?: JsonObject; + identityIncarnationId?: string; +}>; + +type DecodedStateFields = Readonly<{ + phase: AllocationOperationPhase; + binding: AllocationOperationRecord['binding']; + release: AllocationOperationRecord['release']; +}>; + +export function decodeAllocationOperationRecord(value: unknown): AllocationOperationDecoding { + const object = decodeObject(value); + if (isUnreadable(object)) return object; + const version = decodeVersion(object); + if (version) return version; + const base = decodeBaseFields(object); + if (isUnreadable(base)) return base; + const fence = decodeFence(base, object.fence); + if (isUnreadable(fence)) return fence; + const state = decodeStateFields(object, base.identityIncarnationId); + if (isUnreadable(state)) return state; + return createDecodedRecord(base, fence.fence, state); +} + +function decodeObject(value: unknown): DecodeResult { + return isPlainObject(value) ? value : unreadable('corrupt', 'allocation record is not an object'); +} + +function decodeVersion(value: RawAllocationRecord): AllocationOperationUnreadable | undefined { + if (value.schemaVersion === ALLOCATION_OPERATION_SCHEMA_VERSION) return undefined; + return unreadable( + 'unsupported-version', + `unsupported allocation record version ${String(value.schemaVersion)}`, + typeof value.schemaVersion === 'number' ? value.schemaVersion : undefined, + ); +} + +function decodeBaseFields(value: RawAllocationRecord): DecodeResult { + const identifiers = decodeIdentifiers(value); + if (isUnreadable(identifiers)) return identifiers; + const request = decodeRequestFields(value); + if (isUnreadable(request)) return request; + const times = decodeTimes(value); + if (isUnreadable(times)) return times; + const optional = decodeOptionalFields(value); + if (isUnreadable(optional)) return optional; + return { ...identifiers, ...request, ...times, ...optional }; +} + +function decodeIdentifiers( + value: RawAllocationRecord, +): DecodeResult> { + const requesterId = isVerbatimId(value.requesterId) ? value.requesterId : undefined; + const attemptKey = isVerbatimId(value.attemptKey) ? value.attemptKey : undefined; + const allocatorInstanceId = isVerbatimId(value.allocatorInstanceId) + ? value.allocatorInstanceId + : undefined; + return requesterId && attemptKey && allocatorInstanceId + ? { requesterId, attemptKey, allocatorInstanceId } + : unreadable('corrupt', 'allocation record identifiers are invalid'); +} + +function decodeRequestFields( + value: RawAllocationRecord, +): DecodeResult> { + const shape = decodeShape(value.shape); + const requestGeneration = isRequestGeneration(value.requestGeneration) + ? value.requestGeneration + : undefined; + if (!shape) return unreadable('corrupt', 'allocation record shape is invalid'); + if (value.admission !== 'fail-fast' || !isActivation(value.activation)) { + return unreadable('corrupt', 'allocation record request policy is invalid'); + } + if (requestGeneration === undefined) { + return unreadable('corrupt', 'allocation record request generation is invalid'); + } + return { shape, requestGeneration, activation: value.activation }; +} + +function decodeTimes( + value: RawAllocationRecord, +): DecodeResult> { + const deadlineAtMs = isFiniteNumber(value.deadlineAtMs) ? value.deadlineAtMs : undefined; + const createdAtMs = isFiniteNumber(value.createdAtMs) ? value.createdAtMs : undefined; + const updatedAtMs = isFiniteNumber(value.updatedAtMs) ? value.updatedAtMs : undefined; + if (deadlineAtMs === undefined || createdAtMs === undefined || updatedAtMs === undefined) { + return unreadable('corrupt', 'allocation record timestamps are invalid'); + } + if (updatedAtMs < createdAtMs) { + return unreadable('ambiguous', 'allocation record update precedes its creation'); + } + return { deadlineAtMs, createdAtMs, updatedAtMs }; +} + +function decodeOptionalFields( + value: RawAllocationRecord, +): DecodeResult> { + const identityIncarnationId = decodeIdentity(value.identityIncarnationId); + if (value.identityIncarnationId !== undefined && identityIncarnationId === undefined) { + return unreadable('corrupt', 'allocation record identity incarnation is invalid'); + } + const attribution = decodeAllocationAttribution(value.attribution); + if (value.attribution !== undefined && attribution === undefined) { + return unreadable('corrupt', 'allocation record attribution is invalid'); + } + return { + ...(identityIncarnationId === undefined ? {} : { identityIncarnationId }), + ...(attribution === undefined ? {} : { attribution }), + }; +} + +function decodeIdentity(value: unknown): string | undefined { + return value === undefined ? undefined : isVerbatimId(value) ? value : undefined; +} + +function decodeStateFields( + value: RawAllocationRecord, + identityIncarnationId: string | undefined, +): DecodeResult { + const phase = decodePhase(value.phase); + const binding = decodeBinding(value.binding); + const release = decodeRelease(value.release); + if (!phase || !binding || !release) { + return unreadable('corrupt', 'allocation record cleanup state is invalid'); + } + const consistency = checkConsistency({ phase, identityIncarnationId, binding, release }); + return consistency ? unreadable('ambiguous', consistency) : { phase, binding, release }; +} + +function createDecodedRecord( + base: DecodedBaseFields, + fence: ReturnType, + state: DecodedStateFields, +): AllocationOperationDecoding { + return { + status: 'decoded', + record: Object.freeze({ + schemaVersion: ALLOCATION_OPERATION_SCHEMA_VERSION, + ...base, + admission: 'fail-fast' as const, + fence, + phase: Object.freeze(state.phase), + binding: state.binding, + release: state.release, + }), + }; +} + +function isUnreadable(value: DecodeResult): value is AllocationOperationUnreadable { + return ( + typeof value === 'object' && + value !== null && + 'status' in value && + value.status === 'unreadable' + ); +} + +function decodeFence( + ref: Pick, + value: unknown, +): + | Readonly<{ status: 'decoded'; fence: ReturnType }> + | AllocationOperationUnreadable { + if ( + !isPlainObject(value) || + typeof value.token !== 'string' || + !isFenceGeneration(value.generation) + ) { + return unreadable('unfenced', 'allocation record has no valid transition fence'); + } + let expected: ReturnType; + try { + expected = allocationOperationFence(ref, value.generation); + } catch { + return unreadable('unfenced', 'allocation record transition fence cannot be reconstructed'); + } + return expected.token === value.token + ? { status: 'decoded', fence: expected } + : unreadable('unfenced', 'allocation record transition fence does not name its operation'); +} + +function unreadable( + reason: AllocationOperationUnreadableReason, + message: string, + version?: number, +): AllocationOperationUnreadable { + return { status: 'unreadable', reason, message, ...(version === undefined ? {} : { version }) }; +} diff --git a/src/daemon/managed-device-allocation/record-factory.ts b/src/daemon/managed-device-allocation/record-factory.ts new file mode 100644 index 000000000..1a41188db --- /dev/null +++ b/src/daemon/managed-device-allocation/record-factory.ts @@ -0,0 +1,37 @@ +import { AppError } from '@agent-device/kernel/errors'; +import { freezeJsonObject } from '@agent-device/capture-kit'; +import { allocationOperationFence } from './fence.ts'; +import { ALLOCATION_OPERATION_SCHEMA_VERSION } from './schema.ts'; +import { isAllocationRequest } from './record-validation.ts'; +import type { AllocationOperationRecord, NewAllocationOperationInput } from './record.ts'; + +export function newAllocationOperation( + fields: NewAllocationOperationInput, +): AllocationOperationRecord { + if (!isAllocationRequest(fields)) { + throw new AppError('COMMAND_FAILED', 'Allocation operation request is invalid', { + reason: 'allocation-request-invalid', + retriable: false, + }); + } + return Object.freeze({ + schemaVersion: ALLOCATION_OPERATION_SCHEMA_VERSION, + requesterId: fields.requesterId, + attemptKey: fields.attemptKey, + allocatorInstanceId: fields.allocatorInstanceId, + shape: Object.freeze({ ...fields.shape }), + deadlineAtMs: fields.deadlineAtMs, + requestGeneration: fields.requestGeneration, + admission: fields.admission, + activation: fields.activation, + ...(fields.attribution === undefined + ? {} + : { attribution: freezeJsonObject(fields.attribution) }), + createdAtMs: fields.nowMs, + updatedAtMs: fields.nowMs, + fence: allocationOperationFence(fields, 0), + phase: Object.freeze({ status: 'unresolved' as const }), + binding: 'unpublished', + release: 'not-requested', + }); +} diff --git a/src/daemon/managed-device-allocation/record-json.ts b/src/daemon/managed-device-allocation/record-json.ts new file mode 100644 index 000000000..774d66f5f --- /dev/null +++ b/src/daemon/managed-device-allocation/record-json.ts @@ -0,0 +1,7 @@ +import type { JsonObject } from '@agent-device/contracts/client'; +import { freezeJsonObject, isBoundedJsonObject } from '@agent-device/capture-kit'; + +export function decodeAllocationAttribution(value: unknown): JsonObject | undefined { + if (value === undefined) return undefined; + return isBoundedJsonObject(value) ? freezeJsonObject(value) : undefined; +} diff --git a/src/daemon/managed-device-allocation/record-validation.ts b/src/daemon/managed-device-allocation/record-validation.ts new file mode 100644 index 000000000..fbbcc706e --- /dev/null +++ b/src/daemon/managed-device-allocation/record-validation.ts @@ -0,0 +1,95 @@ +import type { + LeaseRefusal, + ManagedLease, + ManagedShapeRequest, +} from '@agent-device/contracts/managed-device-allocation'; +import { isBoundedJsonObject } from '@agent-device/capture-kit'; +import type { NewAllocationOperationInput } from './record.ts'; + +export function isAllocationRequest(fields: NewAllocationOperationInput): boolean { + return ( + isVerbatimId(fields.requesterId) && + isVerbatimId(fields.attemptKey) && + isVerbatimId(fields.allocatorInstanceId) && + isRequestGeneration(fields.requestGeneration) && + isFiniteNumber(fields.deadlineAtMs) && + isFiniteNumber(fields.nowMs) && + fields.admission === 'fail-fast' && + (fields.activation === 'direct' || fields.activation === 'external-fence') && + isValidShape(fields.shape) && + (fields.attribution === undefined || isBoundedJsonObject(fields.attribution)) + ); +} + +function isValidShape(shape: ManagedShapeRequest): boolean { + return ( + isPlainObject(shape) && + (shape.platform === 'ios' || shape.platform === 'android') && + isNonEmptyString(shape.deviceType) && + (shape.osVersion === undefined || isNonEmptyString(shape.osVersion)) + ); +} + +export function isValidLease(lease: ManagedLease): boolean { + return ( + isPlainObject(lease) && + isVerbatimId(lease.id) && + isFiniteNumber(lease.ttlDeadline) && + isPlainObject(lease.device) && + isVerbatimId(lease.device.address) && + isPlainObject(lease.environment) && + Object.values(lease.environment).every((value) => typeof value === 'string') + ); +} + +export function isValidRefusal(refusal: LeaseRefusal): boolean { + return ( + isPlainObject(refusal) && + [ + 'simulator-capacity', + 'disk-low', + 'invalid-shape', + 'preparation-required', + 'requester-busy', + ].includes(refusal.reason) && + (refusal.retryAfterMs === undefined || isFiniteNumber(refusal.retryAfterMs)) && + (refusal.message === undefined || isNonEmptyString(refusal.message)) + ); +} + +export function freezeLease(lease: ManagedLease): ManagedLease { + return Object.freeze({ + id: lease.id, + ttlDeadline: lease.ttlDeadline, + device: Object.freeze({ address: lease.device.address }), + environment: Object.freeze({ ...lease.environment }), + }); +} + +export function freezeRefusal(refusal: LeaseRefusal): LeaseRefusal { + return Object.freeze({ ...refusal }); +} + +export function isPlainObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +export function isVerbatimId(value: unknown): value is string { + return typeof value === 'string' && value.length > 0 && value.trim() === value; +} + +export function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0; +} + +export function isFiniteNumber(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value); +} + +export function isRequestGeneration(value: unknown): value is number { + return typeof value === 'number' && Number.isInteger(value) && value >= 1; +} + +export function isFenceGeneration(value: unknown): value is number { + return typeof value === 'number' && Number.isInteger(value) && value >= 0; +} diff --git a/src/daemon/managed-device-allocation/record.ts b/src/daemon/managed-device-allocation/record.ts new file mode 100644 index 000000000..c63d71160 --- /dev/null +++ b/src/daemon/managed-device-allocation/record.ts @@ -0,0 +1,106 @@ +import type { JsonObject } from '@agent-device/contracts/client'; +import type { + LeaseRequestStatus, + LeaseRefusal, + ManagedLease, + ManagedShapeRequest, +} from '@agent-device/contracts/managed-device-allocation'; +import { + managedBindingFence, + type ResourceOwnershipFence, +} from '@agent-device/contracts/platform-runtime'; +import { ALLOCATION_OPERATION_SCHEMA_VERSION } from './schema.ts'; + +export { ALLOCATION_OPERATION_SCHEMA_VERSION }; + +export type AllocationOperationRef = Readonly<{ + requesterId: string; + attemptKey: string; +}>; + +export type AllocationOperationBinding = + | 'unpublished' + | 'publish-pending' + | 'published' + | 'cleanup-pending' + | 'cleaned' + | 'not-applicable'; + +export type AllocationOperationRelease = 'not-requested' | 'pending' | 'released'; + +export type AllocationOperationPhase = + | Readonly<{ status: 'unresolved' }> + | Readonly<{ status: 'pending' }> + | Readonly<{ status: 'unknown'; message: string }> + | Readonly<{ status: 'ambiguous'; message: string }> + | Readonly<{ status: 'granted'; lease: ManagedLease }> + | Readonly<{ status: 'refused'; refusal: LeaseRefusal }> + | Readonly<{ status: 'superseded' }> + | Readonly<{ status: 'cancelled' }>; + +export type AllocationOperationRecord = Readonly<{ + schemaVersion: typeof ALLOCATION_OPERATION_SCHEMA_VERSION; + requesterId: string; + attemptKey: string; + allocatorInstanceId: string; + shape: ManagedShapeRequest; + deadlineAtMs: number; + requestGeneration: number; + admission: 'fail-fast'; + activation: 'direct' | 'external-fence'; + attribution?: JsonObject; + identityIncarnationId?: string; + createdAtMs: number; + updatedAtMs: number; + fence: ResourceOwnershipFence; + phase: AllocationOperationPhase; + binding: AllocationOperationBinding; + release: AllocationOperationRelease; +}>; + +export type NewAllocationOperationInput = Readonly< + Omit< + AllocationOperationRecord, + 'schemaVersion' | 'createdAtMs' | 'updatedAtMs' | 'fence' | 'phase' | 'binding' | 'release' + > & { + nowMs: number; + } +>; + +export type AllocationAllocatorOutcome = + | Readonly<{ status: 'pending'; identityIncarnationId?: string }> + | Readonly<{ + status: 'granted'; + lease: ManagedLease; + identityIncarnationId: string; + }> + | Readonly<{ status: 'refused'; refusal: LeaseRefusal; identityIncarnationId?: string }> + | Readonly<{ status: 'superseded' }> + | Readonly<{ status: 'cancelled' }>; + +export type AllocationTransition = + | Readonly<{ kind: 'request-dispatched' }> + | Readonly<{ kind: 'allocator-outcome'; outcome: AllocationAllocatorOutcome }> + | Readonly<{ kind: 'allocator-status'; status: LeaseRequestStatus }> + | Readonly<{ kind: 'allocator-unknown'; message: string }> + | Readonly<{ kind: 'allocator-ambiguous'; message: string }> + | Readonly<{ kind: 'binding-publish-pending' }> + | Readonly<{ kind: 'binding-published' }> + | Readonly<{ kind: 'binding-cleanup-pending'; message?: string }> + | Readonly<{ kind: 'binding-cleaned' }> + | Readonly<{ kind: 'release-pending' }> + | Readonly<{ kind: 'allocator-released' }>; + +export type AllocationTransitionResult = + | Readonly<{ status: 'applied'; record: AllocationOperationRecord }> + | Readonly<{ status: 'already-applied'; record: AllocationOperationRecord }> + | Readonly<{ status: 'already-terminal'; record: AllocationOperationRecord }>; + +export function bindingFenceFor(record: AllocationOperationRecord): ResourceOwnershipFence | null { + if (record.phase.status !== 'granted' || record.identityIncarnationId === undefined) return null; + return managedBindingFence({ + requesterId: record.requesterId, + requestGeneration: record.requestGeneration, + identityIncarnationId: record.identityIncarnationId, + }); +} diff --git a/src/daemon/managed-device-allocation/schema.ts b/src/daemon/managed-device-allocation/schema.ts new file mode 100644 index 000000000..ef1cda5bc --- /dev/null +++ b/src/daemon/managed-device-allocation/schema.ts @@ -0,0 +1 @@ +export const ALLOCATION_OPERATION_SCHEMA_VERSION = 1 as const; diff --git a/src/daemon/managed-device-allocation/status.ts b/src/daemon/managed-device-allocation/status.ts new file mode 100644 index 000000000..e017fd09a --- /dev/null +++ b/src/daemon/managed-device-allocation/status.ts @@ -0,0 +1,128 @@ +import type { LeaseRequestStatus } from '@agent-device/contracts/managed-device-allocation'; +import { isRequestGeneration, isVerbatimId } from './record-validation.ts'; +import type { AllocationOperationRecord, AllocationTransition } from './record.ts'; + +type TerminalLeaseRequestStatus = LeaseRequestStatus & + Readonly<{ state: 'superseded' | 'cancelled' }>; + +export function transitionFromAllocatorStatus( + record: AllocationOperationRecord, + status: LeaseRequestStatus, +): AllocationTransition { + if (!matchesOperation(record, status)) { + return { + kind: 'allocator-ambiguous', + message: 'allocator status does not identify this operation', + }; + } + if (status.identityIncarnationId !== undefined && !isVerbatimId(status.identityIncarnationId)) { + return { + kind: 'allocator-ambiguous', + message: 'allocator status has an invalid identity incarnation', + }; + } + return transitionForKnownStatus(status); +} + +function matchesOperation(record: AllocationOperationRecord, status: LeaseRequestStatus): boolean { + return ( + isVerbatimId(status.requesterId) && + isVerbatimId(status.attemptKey) && + status.requesterId === record.requesterId && + status.attemptKey === record.attemptKey && + isRequestGeneration(status.requestGeneration) && + status.requestGeneration === record.requestGeneration + ); +} + +function transitionForKnownStatus(status: LeaseRequestStatus): AllocationTransition { + switch (status.state) { + case 'unknown': + return transitionForUnknown(status); + case 'pending': + return transitionForPending(status); + case 'granted': + return transitionForGranted(status); + case 'refused': + return transitionForRefused(status); + case 'superseded': + case 'cancelled': + return transitionForTerminal(status as TerminalLeaseRequestStatus); + default: + return { kind: 'allocator-ambiguous', message: 'allocator status has an unknown state' }; + } +} + +function transitionForUnknown(status: LeaseRequestStatus): AllocationTransition { + return hasTerminalData(status) + ? { kind: 'allocator-ambiguous', message: 'allocator unknown status carried terminal data' } + : { kind: 'allocator-unknown', message: 'allocator status is unknown' }; +} + +function transitionForPending(status: LeaseRequestStatus): AllocationTransition { + if (hasTerminalData(status)) { + return { + kind: 'allocator-ambiguous', + message: 'allocator pending status carried terminal data', + }; + } + return { + kind: 'allocator-outcome', + outcome: { + status: 'pending', + ...(status.identityIncarnationId === undefined + ? {} + : { identityIncarnationId: status.identityIncarnationId }), + }, + }; +} + +function transitionForGranted(status: LeaseRequestStatus): AllocationTransition { + if ( + status.lease === undefined || + status.identityIncarnationId === undefined || + status.refusal !== undefined + ) { + return { + kind: 'allocator-ambiguous', + message: 'allocator grant omitted its lease or identity incarnation', + }; + } + return { + kind: 'allocator-outcome', + outcome: { + status: 'granted', + lease: status.lease, + identityIncarnationId: status.identityIncarnationId, + }, + }; +} + +function transitionForRefused(status: LeaseRequestStatus): AllocationTransition { + if (status.refusal === undefined || status.lease !== undefined) { + return { + kind: 'allocator-ambiguous', + message: 'allocator refusal carried incomplete or conflicting data', + }; + } + return { + kind: 'allocator-outcome', + outcome: { + status: 'refused', + refusal: status.refusal, + ...(status.identityIncarnationId === undefined + ? {} + : { identityIncarnationId: status.identityIncarnationId }), + }, + }; +} + +function transitionForTerminal(status: TerminalLeaseRequestStatus): AllocationTransition { + return hasTerminalData(status) + ? { kind: 'allocator-ambiguous', message: 'allocator terminal status carried conflicting data' } + : { kind: 'allocator-outcome', outcome: { status: status.state } }; +} + +function hasTerminalData(status: LeaseRequestStatus): boolean { + return status.lease !== undefined || status.refusal !== undefined; +} diff --git a/src/daemon/managed-device-allocation/store-creation.ts b/src/daemon/managed-device-allocation/store-creation.ts new file mode 100644 index 000000000..c1d06a2f2 --- /dev/null +++ b/src/daemon/managed-device-allocation/store-creation.ts @@ -0,0 +1,90 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import type { AllocationOperationRecord, AllocationOperationRef } from './record.ts'; +import type { + AllocationOperationCreate, + AllocationOperationRead, + AllocationOperationUnreadable, +} from './store.ts'; +import { decodeAllocationOperationRecord } from './record-codec.ts'; +import { + assertSafeAllocationDirectory, + errorMessage, + isAlreadyExists, + publishAllocationRecord, +} from './store-filesystem.ts'; +import { hash } from './store-lock.ts'; +import { allocationOperationUnreadable } from './store-results.ts'; + +type ReadOperation = ( + recordPath: string, + expectedRef?: AllocationOperationRef, +) => AllocationOperationRead; + +export function createAllocationRecord( + options: Readonly<{ + allocationsDir: string; + record: AllocationOperationRecord; + read: ReadOperation; + }>, +): AllocationOperationCreate { + const recordPath = path.join( + options.allocationsDir, + hash(options.record.requesterId), + `${hash(options.record.attemptKey)}.json`, + ); + const decoded = decodeAllocationOperationRecord(options.record); + if (decoded.status !== 'decoded') return allocationOperationUnreadable(recordPath, decoded); + const directoryError = prepareRecordDirectory(options.allocationsDir, recordPath); + if (directoryError) return directoryError; + const existing = options.read(recordPath, options.record); + if (existing.status === 'found') + return { status: 'exists', path: recordPath, record: existing.record }; + if (existing.status === 'unreadable') return existing; + return publishNewRecord(recordPath, decoded.record, options.read); +} + +function prepareRecordDirectory( + allocationsDir: string, + recordPath: string, +): AllocationOperationUnreadable | undefined { + try { + assertSafeAllocationDirectory(allocationsDir); + fs.mkdirSync(path.dirname(recordPath), { recursive: true, mode: 0o700 }); + assertSafeAllocationDirectory(path.dirname(recordPath)); + return undefined; + } catch (error) { + return allocationOperationUnreadable(recordPath, { + status: 'unreadable', + reason: 'corrupt', + message: errorMessage(error), + }); + } +} + +function publishNewRecord( + recordPath: string, + record: AllocationOperationRecord, + read: ReadOperation, +): AllocationOperationCreate { + try { + publishAllocationRecord(recordPath, record, 'link-exclusive'); + return { status: 'created', path: recordPath, record }; + } catch (error) { + if (!isAlreadyExists(error)) throw error; + return resolveCreateRace(recordPath, read); + } +} + +function resolveCreateRace(recordPath: string, read: ReadOperation): AllocationOperationCreate { + const existing = read(recordPath); + if (existing.status === 'found') { + return { status: 'exists', path: recordPath, record: existing.record }; + } + if (existing.status === 'unreadable') return existing; + return allocationOperationUnreadable(recordPath, { + status: 'unreadable', + reason: 'corrupt', + message: 'allocation operation path disappeared during creation', + }); +} diff --git a/src/daemon/managed-device-allocation/store-filesystem.ts b/src/daemon/managed-device-allocation/store-filesystem.ts new file mode 100644 index 000000000..191981c19 --- /dev/null +++ b/src/daemon/managed-device-allocation/store-filesystem.ts @@ -0,0 +1,139 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { publishDurableFileSync } from '@agent-device/host-kit/file'; +import type { AllocationOperationRecord } from './record.ts'; + +export type AllocationOperationPath = + | Readonly<{ status: 'path'; path: string }> + | Readonly<{ status: 'unreadable'; path: string; message: string }>; + +export function publishAllocationRecord( + recordPath: string, + record: AllocationOperationRecord, + mode: 'replace' | 'link-exclusive', +): void { + publishDurableFileSync({ + destination: recordPath, + contents: `${JSON.stringify(record)}\n`, + publish: mode, + }); +} + +export function listAllocationOperationPaths(allocationsDir: string): AllocationOperationPath[] { + const rootState = inspectDirectory(allocationsDir); + if (rootState === 'missing') return []; + if (rootState !== null) return [unreadablePath(allocationsDir, rootState)]; + + const lanes = readDirectory(allocationsDir); + if (lanes.status === 'missing') return []; + if (lanes.status === 'unreadable') return [unreadablePath(allocationsDir, lanes.message)]; + return lanes.entries + .flatMap((lane) => listLanePaths(allocationsDir, lane)) + .sort((left, right) => left.path.localeCompare(right.path)); +} + +export function isAlreadyExists(error: unknown): boolean { + return ( + error !== null && + typeof error === 'object' && + 'code' in error && + (error as { code?: unknown }).code === 'EEXIST' + ); +} + +export function isMissingFile(error: unknown): boolean { + return ( + error !== null && + typeof error === 'object' && + 'code' in error && + (error as { code?: unknown }).code === 'ENOENT' + ); +} + +export function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : 'allocation operation record is unreadable'; +} + +export function assertSafeAllocationDirectory(directory: string): void { + let stats: fs.Stats; + try { + stats = fs.lstatSync(directory); + } catch (error) { + if (isMissingFile(error)) return; + throw error; + } + if (stats.isSymbolicLink() || !stats.isDirectory()) { + throw new Error(`Refusing unsafe allocation operation directory ${directory}`); + } +} + +export function inspectAllocationOperationDirectories( + recordPath: string, +): 'missing' | string | null { + const laneDirectory = path.dirname(recordPath); + const directories = [path.dirname(laneDirectory), laneDirectory]; + for (const directory of directories) { + let stats: fs.Stats; + try { + stats = fs.lstatSync(directory); + } catch (error) { + if (isMissingFile(error)) return 'missing'; + return errorMessage(error); + } + if (stats.isSymbolicLink()) return 'allocation operation directory is a symbolic link'; + if (!stats.isDirectory()) return 'allocation operation directory is not a directory'; + } + return null; +} + +function inspectDirectory(directory: string): 'missing' | string | null { + let stats: fs.Stats; + try { + stats = fs.lstatSync(directory); + } catch (error) { + return isMissingFile(error) ? 'missing' : errorMessage(error); + } + if (stats.isSymbolicLink()) return 'allocation operation directory is a symbolic link'; + if (!stats.isDirectory()) return 'allocation operation directory is not a directory'; + return null; +} + +type DirectoryRead = + | Readonly<{ status: 'entries'; entries: fs.Dirent[] }> + | Readonly<{ status: 'missing' }> + | Readonly<{ status: 'unreadable'; message: string }>; + +function readDirectory(directory: string): DirectoryRead { + try { + return { status: 'entries', entries: fs.readdirSync(directory, { withFileTypes: true }) }; + } catch (error) { + return isMissingFile(error) + ? { status: 'missing' } + : { status: 'unreadable', message: errorMessage(error) }; + } +} + +function listLanePaths(allocationsDir: string, lane: fs.Dirent): AllocationOperationPath[] { + if (lane.name.endsWith('.lane.lock')) return []; + const lanePath = path.join(allocationsDir, lane.name); + if (!lane.isDirectory()) { + return [ + unreadablePath( + lanePath, + lane.isSymbolicLink() + ? 'allocation operation lane is a symbolic link' + : 'allocation operation lane is not a directory', + ), + ]; + } + const entries = readDirectory(lanePath); + if (entries.status === 'missing') return []; + if (entries.status === 'unreadable') return [unreadablePath(lanePath, entries.message)]; + return entries.entries + .filter((entry) => entry.name.endsWith('.json')) + .map((entry) => ({ status: 'path', path: path.join(lanePath, entry.name) })); +} + +function unreadablePath(pathname: string, message: string): AllocationOperationPath { + return { status: 'unreadable', path: pathname, message }; +} diff --git a/src/daemon/managed-device-allocation/store-lock.ts b/src/daemon/managed-device-allocation/store-lock.ts new file mode 100644 index 000000000..8c0ce0bda --- /dev/null +++ b/src/daemon/managed-device-allocation/store-lock.ts @@ -0,0 +1,51 @@ +import crypto from 'node:crypto'; +import path from 'node:path'; +import { acquireProcessLock } from '@agent-device/host-kit/file'; +import { readCurrentOwnerIdentity } from '@agent-device/host-kit/process'; +import type { AllocationOperationStore } from './store.ts'; + +const LOCK_TIMEOUT_MS = 30_000; + +export function createAllocationStoreLaneLock( + allocationsDir: string, +): AllocationOperationStore['withLaneLock'] { + return (requesterId: string, task: () => Promise) => + withAllocationLaneLock(allocationsDir, requesterId, task); +} + +async function withAllocationLaneLock( + allocationsDir: string, + requesterId: string, + task: () => Promise, +): Promise { + const release = await acquireAllocationStoreLock( + path.join(allocationsDir, `${hash(requesterId)}.lane.lock`), + `allocation lane ${requesterId}`, + ); + try { + return await task(); + } finally { + await release(); + } +} + +export function acquireAllocationStoreLock( + lockDirPath: string, + description: string, +): Promise<() => Promise> { + const owner = readCurrentOwnerIdentity(); + return acquireProcessLock({ + lockDirPath, + owner: { + pid: owner.pid, + startTime: owner.startTime, + acquiredAtMs: Date.now(), + }, + timeoutMs: LOCK_TIMEOUT_MS, + description, + }); +} + +export function hash(value: string): string { + return crypto.createHash('sha256').update(value).digest('hex'); +} diff --git a/src/daemon/managed-device-allocation/store-results.ts b/src/daemon/managed-device-allocation/store-results.ts new file mode 100644 index 000000000..25844e586 --- /dev/null +++ b/src/daemon/managed-device-allocation/store-results.ts @@ -0,0 +1,30 @@ +import type { AllocationOperationRecord } from './record.ts'; +import type { AllocationOperationUnreadable, AllocationOperationWrite } from './store.ts'; +import { decodeAllocationOperationRecord } from './record-codec.ts'; + +export function allocationOperationUnreadable( + recordPath: string, + result: Extract, { status: 'unreadable' }>, +): AllocationOperationUnreadable { + return { + status: 'unreadable', + path: recordPath, + reason: result.reason, + message: result.message, + ...(result.version === undefined ? {} : { version: result.version }), + }; +} + +export type StoredAllocationTransition = Readonly<{ + status: 'applied' | 'already-applied' | 'already-terminal'; + record: AllocationOperationRecord; +}>; + +export function allocationOperationWriteResult( + recordPath: string, + result: StoredAllocationTransition, +): AllocationOperationWrite { + return result.status === 'applied' + ? { status: 'recorded', path: recordPath, record: result.record } + : { status: result.status, path: recordPath, record: result.record }; +} diff --git a/src/daemon/managed-device-allocation/store.ts b/src/daemon/managed-device-allocation/store.ts new file mode 100644 index 000000000..ab3abd0eb --- /dev/null +++ b/src/daemon/managed-device-allocation/store.ts @@ -0,0 +1,218 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { openVerifiedFileForRead } from '@agent-device/host-kit/file'; +import type { + AllocationOperationRecord, + AllocationOperationRef, + AllocationTransition, +} from './record.ts'; +import { applyAllocationTransition } from './transitions.ts'; +import { decodeAllocationOperationRecord } from './record-codec.ts'; +import { createAllocationRecord } from './store-creation.ts'; +import { + errorMessage, + inspectAllocationOperationDirectories, + isMissingFile, + listAllocationOperationPaths, + publishAllocationRecord, +} from './store-filesystem.ts'; +import { acquireAllocationStoreLock, createAllocationStoreLaneLock, hash } from './store-lock.ts'; +import { allocationOperationUnreadable, allocationOperationWriteResult } from './store-results.ts'; +import type { ResourceOwnershipFence } from '@agent-device/contracts/platform-runtime'; + +export type AllocationOperationUnreadable = Readonly<{ + status: 'unreadable'; + path: string; + reason: 'corrupt' | 'unsupported-version' | 'ambiguous' | 'unfenced'; + message: string; + version?: number; +}>; + +export type AllocationOperationRead = + | Readonly<{ status: 'missing'; path: string }> + | Readonly<{ status: 'found'; path: string; record: AllocationOperationRecord }> + | AllocationOperationUnreadable; + +export type AllocationOperationCreate = + | Readonly<{ status: 'created'; path: string; record: AllocationOperationRecord }> + | Readonly<{ status: 'exists'; path: string; record: AllocationOperationRecord }> + | AllocationOperationUnreadable; + +export type AllocationOperationWrite = + | Readonly<{ status: 'recorded'; path: string; record: AllocationOperationRecord }> + | Readonly<{ status: 'already-applied'; path: string; record: AllocationOperationRecord }> + | Readonly<{ status: 'already-terminal'; path: string; record: AllocationOperationRecord }> + | Readonly<{ status: 'fence-lost'; path: string; current: AllocationOperationRecord }> + | AllocationOperationRead; + +export type AllocationOperationStore = Readonly<{ + allocationsDir: string; + resolvePath(ref: AllocationOperationRef): string; + create(record: AllocationOperationRecord): AllocationOperationCreate; + read(ref: AllocationOperationRef): AllocationOperationRead; + withLaneLock(requesterId: string, task: () => Promise): Promise; + transition( + ref: AllocationOperationRef, + expectedFence: ResourceOwnershipFence, + transition: AllocationTransition, + nowMs: number, + ): Promise; + list(): AllocationOperationRead[]; +}>; + +export function createAllocationOperationStore(options: { + allocationsDir: string; +}): AllocationOperationStore { + const allocationsDir = path.resolve(options.allocationsDir); + + return Object.freeze({ + allocationsDir, + resolvePath: (ref) => operationPath(allocationsDir, ref), + create: (record) => createAllocationRecord({ allocationsDir, record, read: readPath }), + read: (ref) => readPath(operationPath(allocationsDir, ref), ref), + withLaneLock: createAllocationStoreLaneLock(allocationsDir), + transition: (ref, expectedFence, transition, nowMs) => + transitionRecord(allocationsDir, ref, expectedFence, transition, nowMs), + list: () => listRecords(allocationsDir), + }); +} + +async function transitionRecord( + allocationsDir: string, + ref: AllocationOperationRef, + expectedFence: ResourceOwnershipFence, + transition: AllocationTransition, + nowMs: number, +): Promise { + const recordPath = operationPath(allocationsDir, ref); + const release = await acquireAllocationStoreLock( + `${recordPath}.lock`, + `allocation operation ${ref.requesterId}/${ref.attemptKey}`, + ); + try { + return applyStoredTransition(recordPath, ref, expectedFence, transition, nowMs); + } finally { + await release(); + } +} + +function applyStoredTransition( + recordPath: string, + ref: AllocationOperationRef, + expectedFence: ResourceOwnershipFence, + transition: AllocationTransition, + nowMs: number, +): AllocationOperationWrite { + const current = readPath(recordPath, ref); + if (current.status !== 'found') return current; + if (!sameFence(current.record.fence, expectedFence)) { + return { status: 'fence-lost', path: recordPath, current: current.record }; + } + const result = applyAllocationTransition(current.record, transition, nowMs); + if (result.status === 'applied') publishAllocationRecord(recordPath, result.record, 'replace'); + return allocationOperationWriteResult(recordPath, result); +} + +function listRecords(allocationsDir: string): AllocationOperationRead[] { + return listAllocationOperationPaths(allocationsDir).map((entry) => + entry.status === 'unreadable' ? corruptRecord(entry.path, entry.message) : readPath(entry.path), + ); +} + +function operationPath(allocationsDir: string, ref: AllocationOperationRef): string { + return path.join(allocationsDir, hash(ref.requesterId), `${hash(ref.attemptKey)}.json`); +} + +type RawRecordRead = + | Readonly<{ status: 'missing' }> + | Readonly<{ status: 'unreadable'; message: string }> + | Readonly<{ status: 'value'; value: unknown }>; + +function readPath( + recordPath: string, + expectedRef?: AllocationOperationRef, +): AllocationOperationRead { + const parentState = inspectAllocationOperationDirectories(recordPath); + if (parentState === 'missing') return { status: 'missing', path: recordPath }; + if (parentState !== null) return corruptRecord(recordPath, parentState); + const raw = readRawRecord(recordPath); + if (raw.status === 'missing') return { status: 'missing', path: recordPath }; + if (raw.status === 'unreadable') return corruptRecord(recordPath, raw.message); + const referenceError = validateRawReference(raw.value, expectedRef); + if (referenceError) return allocationOperationUnreadable(recordPath, referenceError); + const decoded = decodeAllocationOperationRecord(raw.value); + if (decoded.status !== 'decoded') return allocationOperationUnreadable(recordPath, decoded); + return validateDecodedRecord(recordPath, decoded.record, expectedRef); +} + +function readRawRecord(recordPath: string): RawRecordRead { + let descriptor: number | undefined; + try { + descriptor = openVerifiedFileForRead(recordPath); + if (descriptor === undefined) return { status: 'missing' }; + return { status: 'value', value: JSON.parse(fs.readFileSync(descriptor, 'utf8')) as unknown }; + } catch (error) { + return isMissingFile(error) + ? { status: 'missing' } + : { status: 'unreadable', message: errorMessage(error) }; + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + } +} + +function validateRawReference( + value: unknown, + expectedRef: AllocationOperationRef | undefined, +): { status: 'unreadable'; reason: 'ambiguous'; message: string } | undefined { + if (!expectedRef || value === null || typeof value !== 'object') return undefined; + const raw = value as Record; + if ( + isVerbatimId(raw.requesterId) && + isVerbatimId(raw.attemptKey) && + (raw.requesterId !== expectedRef.requesterId || raw.attemptKey !== expectedRef.attemptKey) + ) { + return { + reason: 'ambiguous', + status: 'unreadable', + message: 'allocation operation record does not match its requested reference', + }; + } + return undefined; +} + +function validateDecodedRecord( + recordPath: string, + record: AllocationOperationRecord, + expectedRef: AllocationOperationRef | undefined, +): AllocationOperationRead { + const ref = expectedRef ?? record; + const expectedPath = operationPath(path.dirname(path.dirname(recordPath)), ref); + if ( + record.requesterId !== ref.requesterId || + record.attemptKey !== ref.attemptKey || + recordPath !== expectedPath + ) { + return allocationOperationUnreadable(recordPath, { + status: 'unreadable', + reason: 'ambiguous', + message: 'allocation operation record does not match its durable path', + }); + } + return { status: 'found', path: recordPath, record }; +} + +function corruptRecord(recordPath: string, message: string): AllocationOperationUnreadable { + return allocationOperationUnreadable(recordPath, { + status: 'unreadable', + reason: 'corrupt', + message, + }); +} + +function sameFence(left: ResourceOwnershipFence, right: ResourceOwnershipFence): boolean { + return left.token === right.token && left.generation === right.generation; +} + +function isVerbatimId(value: unknown): value is string { + return typeof value === 'string' && value.length > 0 && value.trim() === value; +} diff --git a/src/daemon/managed-device-allocation/transition-outcome.ts b/src/daemon/managed-device-allocation/transition-outcome.ts new file mode 100644 index 000000000..f423dede7 --- /dev/null +++ b/src/daemon/managed-device-allocation/transition-outcome.ts @@ -0,0 +1,151 @@ +import { isDeepStrictEqual } from 'node:util'; +import { + freezeLease, + freezeRefusal, + isValidLease, + isValidRefusal, + isVerbatimId, +} from './record-validation.ts'; +import type { + AllocationAllocatorOutcome, + AllocationOperationRecord, + AllocationTransitionResult, +} from './record.ts'; +import { + alreadyApplied, + ambiguous, + applied, + isTransitionTerminal, + transitionInvalid, +} from './transition-support.ts'; + +export function applyAllocatorOutcome( + record: AllocationOperationRecord, + outcome: AllocationAllocatorOutcome, + nowMs: number, +): AllocationTransitionResult { + if (isTransitionTerminal(record.phase)) { + if (record.phase.status !== 'ambiguous' && sameAllocatorOutcome(record, outcome)) { + return alreadyApplied(record); + } + return { status: 'already-terminal', record }; + } + const identity = outcomeIdentity(outcome); + const validation = validateAllocatorOutcome(record, outcome, identity); + if (validation) return ambiguous(record, validation, nowMs); + if (isPendingReplay(record, outcome, identity)) return alreadyApplied(record); + return applyKnownAllocatorOutcome(record, outcome, identity, nowMs); +} + +function outcomeIdentity(outcome: AllocationAllocatorOutcome): string | undefined { + return outcome.status === 'granted' || + outcome.status === 'pending' || + outcome.status === 'refused' + ? outcome.identityIncarnationId + : undefined; +} + +function validateAllocatorOutcome( + record: AllocationOperationRecord, + outcome: AllocationAllocatorOutcome, + identity: string | undefined, +): string | undefined { + if (identity !== undefined && !isVerbatimId(identity)) { + return 'allocator returned an invalid identity incarnation'; + } + if ( + record.identityIncarnationId !== undefined && + identity !== undefined && + record.identityIncarnationId !== identity + ) { + return 'allocator changed the identity incarnation for one operation'; + } + return validateAllocatorPayload(outcome); +} + +function validateAllocatorPayload(outcome: AllocationAllocatorOutcome): string | undefined { + if (outcome.status === 'granted') { + return isVerbatimId(outcome.identityIncarnationId) && isValidLease(outcome.lease) + ? undefined + : 'allocator grant did not contain a valid lease and identity'; + } + if (outcome.status === 'refused' && !isValidRefusal(outcome.refusal)) { + return 'allocator refusal did not contain a valid refusal'; + } + return undefined; +} + +function isPendingReplay( + record: AllocationOperationRecord, + outcome: AllocationAllocatorOutcome, + identity: string | undefined, +): boolean { + return ( + outcome.status === 'pending' && + record.phase.status === 'pending' && + (identity === undefined || record.identityIncarnationId === identity) + ); +} + +function applyKnownAllocatorOutcome( + record: AllocationOperationRecord, + outcome: AllocationAllocatorOutcome, + identity: string | undefined, + nowMs: number, +): AllocationTransitionResult { + switch (outcome.status) { + case 'pending': + return applied( + record, + { + phase: { status: 'pending' }, + ...(identity === undefined ? {} : { identityIncarnationId: identity }), + }, + nowMs, + ); + case 'granted': + return applied( + record, + { + phase: { status: 'granted', lease: freezeLease(outcome.lease) }, + identityIncarnationId: outcome.identityIncarnationId, + binding: 'unpublished', + release: 'not-requested', + }, + nowMs, + ); + case 'refused': + return applied( + record, + { + phase: { status: 'refused', refusal: freezeRefusal(outcome.refusal) }, + ...(identity === undefined ? {} : { identityIncarnationId: identity }), + binding: 'not-applicable', + }, + nowMs, + ); + case 'superseded': + return applied(record, { phase: { status: 'superseded' }, binding: 'not-applicable' }, nowMs); + case 'cancelled': + return applied(record, { phase: { status: 'cancelled' }, binding: 'not-applicable' }, nowMs); + default: + return transitionInvalid('allocator-outcome', 'unsupported outcome'); + } +} + +function sameAllocatorOutcome( + record: AllocationOperationRecord, + outcome: AllocationAllocatorOutcome, +): boolean { + if (outcome.status === 'granted') + return ( + record.phase.status === 'granted' && + record.identityIncarnationId === outcome.identityIncarnationId && + isDeepStrictEqual(record.phase.lease, outcome.lease) + ); + if (outcome.status === 'refused') + return ( + record.phase.status === 'refused' && isDeepStrictEqual(record.phase.refusal, outcome.refusal) + ); + return record.phase.status === outcome.status; +} diff --git a/src/daemon/managed-device-allocation/transition-support.ts b/src/daemon/managed-device-allocation/transition-support.ts new file mode 100644 index 000000000..0ce4eb970 --- /dev/null +++ b/src/daemon/managed-device-allocation/transition-support.ts @@ -0,0 +1,63 @@ +import { AppError } from '@agent-device/kernel/errors'; +import { allocationOperationFence } from './fence.ts'; +import type { + AllocationOperationPhase, + AllocationOperationRecord, + AllocationTransitionResult, +} from './record.ts'; + +export function applied( + record: AllocationOperationRecord, + updates: Partial< + Pick + >, + nowMs: number, +): AllocationTransitionResult { + const phase = updates.phase === undefined ? record.phase : Object.freeze(updates.phase); + return { + status: 'applied', + record: Object.freeze({ + ...record, + ...updates, + phase, + updatedAtMs: nowMs, + fence: allocationOperationFence(record, record.fence.generation + 1), + }), + }; +} + +export function alreadyApplied(record: AllocationOperationRecord): AllocationTransitionResult { + return { status: 'already-applied', record }; +} + +export function terminalOrInvalid( + record: AllocationOperationRecord, + transition: string, +): AllocationTransitionResult { + return isTransitionTerminal(record.phase) + ? { status: 'already-terminal', record } + : transitionInvalid(transition, record.phase.status); +} + +export function ambiguous( + record: AllocationOperationRecord, + message: string, + nowMs: number, +): AllocationTransitionResult { + return applied(record, { phase: { status: 'ambiguous', message } }, nowMs); +} + +export function transitionInvalid(transition: string, state: unknown): never { + throw allocationError( + 'transition-invalid', + `Allocation transition '${transition}' is invalid from ${String(state)}`, + ); +} + +export function isTransitionTerminal(phase: AllocationOperationPhase): boolean { + return ['granted', 'refused', 'superseded', 'cancelled', 'ambiguous'].includes(phase.status); +} + +function allocationError(reason: string, message: string): AppError { + return new AppError('COMMAND_FAILED', message, { reason, retriable: false }); +} diff --git a/src/daemon/managed-device-allocation/transitions.ts b/src/daemon/managed-device-allocation/transitions.ts new file mode 100644 index 000000000..b3dde14e5 --- /dev/null +++ b/src/daemon/managed-device-allocation/transitions.ts @@ -0,0 +1,213 @@ +import { isFiniteNumber } from './record-validation.ts'; +import { transitionFromAllocatorStatus } from './status.ts'; +import { applyAllocatorOutcome } from './transition-outcome.ts'; +import { + alreadyApplied, + applied, + isTransitionTerminal, + terminalOrInvalid, + transitionInvalid, +} from './transition-support.ts'; +import type { + AllocationOperationRecord, + AllocationTransition, + AllocationTransitionResult, +} from './record.ts'; + +export function applyAllocationTransition( + record: AllocationOperationRecord, + transition: AllocationTransition, + nowMs: number, +): AllocationTransitionResult { + if (!isFiniteNumber(nowMs)) throw new TypeError('Allocation transition time must be finite'); + const resolved = + transition.kind === 'allocator-status' + ? transitionFromAllocatorStatus(record, transition.status) + : transition; + if (resolved.kind === 'allocator-status') + throw new TypeError('Allocator status was not normalized'); + return applyResolvedTransition(record, resolved, nowMs); +} + +type ResolvedAllocationTransition = Exclude; + +function applyResolvedTransition( + record: AllocationOperationRecord, + resolved: ResolvedAllocationTransition, + nowMs: number, +): AllocationTransitionResult { + if (resolved.kind === 'request-dispatched') return applyRequestDispatch(record, nowMs); + if (resolved.kind === 'allocator-outcome') { + return applyAllocatorOutcome(record, resolved.outcome, nowMs); + } + if (isAllocatorUncertaintyTransition(resolved)) { + return applyAllocatorUncertainty(record, resolved, nowMs); + } + if (isBindingTransition(resolved)) { + return applyBindingTransition(record, resolved, nowMs); + } + if (isReleaseTransition(resolved)) { + return applyReleaseTransition(record, resolved, nowMs); + } + return transitionInvalid('unknown', 'unsupported transition'); +} + +function isAllocatorUncertaintyTransition( + transition: ResolvedAllocationTransition, +): transition is Extract< + ResolvedAllocationTransition, + { kind: 'allocator-unknown' | 'allocator-ambiguous' } +> { + return transition.kind === 'allocator-unknown' || transition.kind === 'allocator-ambiguous'; +} + +function isBindingTransition(transition: ResolvedAllocationTransition): transition is Extract< + ResolvedAllocationTransition, + { + kind: + | 'binding-publish-pending' + | 'binding-published' + | 'binding-cleanup-pending' + | 'binding-cleaned'; + } +> { + return ( + transition.kind === 'binding-publish-pending' || + transition.kind === 'binding-published' || + transition.kind === 'binding-cleanup-pending' || + transition.kind === 'binding-cleaned' + ); +} + +function isReleaseTransition( + transition: ResolvedAllocationTransition, +): transition is Extract< + ResolvedAllocationTransition, + { kind: 'release-pending' | 'allocator-released' } +> { + return transition.kind === 'release-pending' || transition.kind === 'allocator-released'; +} + +function applyRequestDispatch( + record: AllocationOperationRecord, + nowMs: number, +): AllocationTransitionResult { + if (record.phase.status === 'unresolved') { + return applied(record, { phase: { status: 'pending' } }, nowMs); + } + if (record.phase.status === 'pending' || record.phase.status === 'unknown') { + return alreadyApplied(record); + } + return terminalOrInvalid(record, 'request-dispatched'); +} + +function applyAllocatorUncertainty( + record: AllocationOperationRecord, + transition: Extract< + ResolvedAllocationTransition, + { kind: 'allocator-unknown' | 'allocator-ambiguous' } + >, + nowMs: number, +): AllocationTransitionResult { + if (isTransitionTerminal(record.phase)) return terminalOrInvalid(record, transition.kind); + const status = transition.kind === 'allocator-unknown' ? 'unknown' : 'ambiguous'; + if (record.phase.status === status && record.phase.message === transition.message) { + return alreadyApplied(record); + } + return applied(record, { phase: { status, message: transition.message } }, nowMs); +} + +function applyBindingTransition( + record: AllocationOperationRecord, + transition: Extract< + ResolvedAllocationTransition, + { + kind: + | 'binding-publish-pending' + | 'binding-published' + | 'binding-cleanup-pending' + | 'binding-cleaned'; + } + >, + nowMs: number, +): AllocationTransitionResult { + if (record.phase.status !== 'granted') return terminalOrInvalid(record, transition.kind); + if (transition.kind === 'binding-publish-pending') { + return applyPublishPendingBinding(record, nowMs); + } + if (transition.kind === 'binding-published') return applyPublishedBinding(record, nowMs); + if (transition.kind === 'binding-cleaned') return applyCleanedBinding(record, nowMs); + return applyCleanupPendingBinding(record, nowMs); +} + +function applyPublishPendingBinding( + record: AllocationOperationRecord, + nowMs: number, +): AllocationTransitionResult { + if (record.binding === 'publish-pending') return alreadyApplied(record); + if (record.binding !== 'unpublished' || record.release !== 'not-requested') { + return transitionInvalid('binding-publish-pending', record.binding); + } + return applied(record, { binding: 'publish-pending' }, nowMs); +} + +function applyPublishedBinding( + record: AllocationOperationRecord, + nowMs: number, +): AllocationTransitionResult { + if (record.binding === 'published') return alreadyApplied(record); + if (record.binding !== 'publish-pending') + return transitionInvalid('binding-published', record.binding); + return applied(record, { binding: 'published' }, nowMs); +} + +function applyCleanupPendingBinding( + record: AllocationOperationRecord, + nowMs: number, +): AllocationTransitionResult { + if (record.binding === 'cleanup-pending' && record.release === 'not-requested') { + return alreadyApplied(record); + } + if (record.binding === 'cleaned' || record.release !== 'not-requested') { + return transitionInvalid('binding-cleanup-pending', record.binding); + } + if (record.binding !== 'publish-pending' && record.binding !== 'published') { + return transitionInvalid('binding-cleanup-pending', record.binding); + } + return applied(record, { binding: 'cleanup-pending' }, nowMs); +} + +function applyCleanedBinding( + record: AllocationOperationRecord, + nowMs: number, +): AllocationTransitionResult { + if (record.binding === 'cleaned') return alreadyApplied(record); + if ( + !['unpublished', 'publish-pending', 'published', 'cleanup-pending'].includes(record.binding) + ) { + return transitionInvalid('binding-cleaned', record.binding); + } + return applied(record, { binding: 'cleaned' }, nowMs); +} + +function applyReleaseTransition( + record: AllocationOperationRecord, + transition: Extract< + ResolvedAllocationTransition, + { kind: 'release-pending' | 'allocator-released' } + >, + nowMs: number, +): AllocationTransitionResult { + if (record.phase.status !== 'granted') return terminalOrInvalid(record, transition.kind); + if (transition.kind === 'release-pending') { + if (record.release === 'pending' || record.release === 'released') + return alreadyApplied(record); + if (record.binding !== 'cleaned') return transitionInvalid('release-pending', record.binding); + return applied(record, { release: 'pending' }, nowMs); + } + if (record.release === 'released') return alreadyApplied(record); + if (record.release !== 'pending' || record.binding !== 'cleaned') { + return transitionInvalid('allocator-released', record.release); + } + return applied(record, { release: 'released' }, nowMs); +}