diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index cbc6d9822..2518d0c56 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -115,7 +115,7 @@ jobs: - batch: graphile-unit packages: 'graphile/graphile-plugin-utils graphile/graphile-realtime-subscriptions graphile/graphile-sql-expression-validator graphile/graphile-upload-plugin graphile/graphile-storage-registry' - batch: agentic - packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/chat agentic/cli agentic/db-tools agentic/pi agentic/dsh agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama agentic/run-log agentic/metering agentic/agent-conversation agentic/pi-host agentic/run-log-client' + packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/chat agentic/cli agentic/db-tools agentic/pi agentic/dsh agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama agentic/run-log agentic/metering agentic/agent-conversation agentic/pi-host agentic/run-log-client agentic/run-log-gate' - batch: pgpm-unit packages: 'pgpm/types pgpm/naming-spec pgpm/diff pgpm/import pgpm/slice pgpm/transform' - batch: pglite diff --git a/agentic/run-log-gate/README.md b/agentic/run-log-gate/README.md new file mode 100644 index 000000000..3727f8d09 --- /dev/null +++ b/agentic/run-log-gate/README.md @@ -0,0 +1,39 @@ +# @agentic-kit/run-log-gate + +Tool approvals over the run log. The request, the human's answer and every +settled gate decision are entries in the run's log — `agent_event` rows once +the store is the platform's — so a desktop run, a cloud pi Job and the agentic +lane all ask in one place and are answered from any surface that projects the +log (`RunsPanel`, `pendingApprovals` → `resolveApproval`). + +## Three pieces + +- `runLogApprovals` — an `ApprovalChannel` for `@agentic-kit/harness`'s + `RunGate`. Submits a `constructive.approval.request` entry and polls the log + for the matching `constructive.approval.resolution`. The request entry's id + is derived from the tool call id, so a retried submit is the same entry and + the store's idempotency drops it instead of asking a human twice. + `resolveApproval` is the other end for a local run and the tests. +- `runLogGateDecisions` — an `onDecision` sink that appends every settled + decision as a `constructive.gate.decision` entry (audit: which calls the gate + refused, on whose authority) and a `flush()` that rethrows the first write + that failed. +- `createRunLogGateHost` — the harness's `GateHost` (what `createConfirmGate` + and `createGatedToolset` drive) built on the two above: `confirmTool` asks in + the log and returns what the log settled on, `notifyToolSkipped` files the + skipped repeat as a denial, `drain()` surfaces a lost write. + +Nothing here executes on the strength of a click: the decision is the record +that was persisted and read back. A host may mirror a concise notice into a +conversation thread, but the thread is never the authority. + +## Consumers + +In constructive-db: `pi-job-host` (the cloud pi Job — `RunGate` over +`runLogApprovals` + `runLogGateDecisions`), the `code_task` agentic lane and +`ui/desktop`'s capability gate (`createRunLogGateHost`). + +## Testing + +`pnpm test` — the store (`MemoryRunLogStore`), the clock and the sleep are all +injectable, so every path is covered without a database or a network. diff --git a/agentic/run-log-gate/__tests__/approvals.test.ts b/agentic/run-log-gate/__tests__/approvals.test.ts new file mode 100644 index 000000000..d3e5f210b --- /dev/null +++ b/agentic/run-log-gate/__tests__/approvals.test.ts @@ -0,0 +1,154 @@ +import type { ApprovalRequest } from '@agentic-kit/harness'; +import type { PiSessionEntry } from '@agentic-kit/run-log'; +import { MemoryRunLogStore, projectToolState, readAll } from '@agentic-kit/run-log'; + +import { approvalEntryId, resolveApproval, runLogApprovals } from '../src/approvals'; + +const request = (overrides: Partial = {}): ApprovalRequest => ({ + runId: 'run-1', + toolCallId: 'call-1', + toolName: 'bash', + input: { command: 'rm -rf /' }, + reason: 'policy asks about destructive commands', + requestedAt: '2024-01-01T00:00:00.000Z', + ...overrides +}); + +/** Let every pending read/append settle — the submit is several awaits deep. */ +const flush = (): Promise => new Promise((resolve) => setImmediate(resolve)); + +const message = (id: string, text: string): PiSessionEntry => + ({ + type: 'message', + id, + parentId: null, + timestamp: '2024-01-01T00:00:00Z', + message: { role: 'assistant', content: [{ type: 'text', text }] } + }) as PiSessionEntry; + +describe('runLogApprovals', () => { + let store: MemoryRunLogStore; + + beforeEach(() => { + store = new MemoryRunLogStore(); + }); + + const channel = (overrides: Partial[0]> = {}) => + runLogApprovals({ + store, + runId: 'run-1', + intervalMs: 0, + // A macrotask rather than a microtask: a poll loop that only ever yields to + // the microtask queue starves the run's timers for as long as it spins. + sleep: () => new Promise((resolve) => setImmediate(resolve)), + now: () => new Date('2024-01-01T00:00:00.000Z'), + ...overrides + }); + + it('writes the request into the log where a projector finds it pending', async () => { + const pending = channel().request(request()); + // Let the submit land before answering, the way a human would. + await flush(); + + const projected = projectToolState(await readAll(store, 'run-1')); + expect(projected.pendingApprovals).toEqual([ + expect.objectContaining({ + toolCallId: 'call-1', + prompt: 'bash: policy asks about destructive commands' + }) + ]); + + await resolveApproval(store, 'run-1', { toolCallId: 'call-1', decision: 'approved' }); + await expect(pending).resolves.toEqual({ decision: 'allow' }); + }); + + it('carries the resolution\'s reason and actor back to the gate', async () => { + const pending = channel().request(request()); + await flush(); + await resolveApproval(store, 'run-1', { + toolCallId: 'call-1', + decision: 'rejected', + reason: 'not on production', + actorId: 'user-7' + }); + + await expect(pending).resolves.toEqual({ + decision: 'deny', + reason: 'not on production', + actorId: 'user-7' + }); + }); + + it('sees a resolution appended after unrelated entries in a later page', async () => { + const pending = channel({ pageLimit: 1 }).request(request()); + await flush(); + await store.append('run-1', [message('chatter-1', 'still thinking')]); + await store.append('run-1', [message('chatter-2', 'still thinking')]); + await resolveApproval(store, 'run-1', { toolCallId: 'call-1', decision: 'approved' }); + + await expect(pending).resolves.toEqual({ decision: 'allow' }); + }); + + it('asks once when a submit is retried, because the entry id is the tool call', async () => { + const first = channel().request(request()); + await flush(); + const second = channel().request(request()); + await flush(); + + await resolveApproval(store, 'run-1', { toolCallId: 'call-1', decision: 'approved' }); + await expect(first).resolves.toEqual({ decision: 'allow' }); + await expect(second).resolves.toEqual({ decision: 'allow' }); + + const records = await readAll(store, 'run-1'); + const requests = records.filter( + (candidate) => (candidate.entry as { id?: string }).id === approvalEntryId('call-1') + ); + expect(requests).toHaveLength(1); + }); + + it('hangs the request off the log\'s leaf so the session stays one tree', async () => { + await store.append('run-1', [message('leaf', 'about to run a tool')]); + // Bounded so the request settles instead of polling past the assertion. + const pending = channel({ timeoutMs: 0 }).request(request()); + await pending; + + const records = await readAll(store, 'run-1'); + const submitted = records[records.length - 1].entry as { parentId?: string | null }; + expect(submitted.parentId).toBe('leaf'); + }); + + it('denies on timeout rather than proceeding unapproved', async () => { + let clock = 0; + const outcome = await runLogApprovals({ + store, + runId: 'run-1', + intervalMs: 1, + timeoutMs: 10, + sleep: () => { + clock += 10; + return Promise.resolve(); + }, + now: () => new Date(clock) + }).request(request()); + + expect(outcome).toEqual({ decision: 'deny', reason: 'gate: no decision within 10ms' }); + }); + + it('can be told to allow on timeout', async () => { + let clock = 0; + const outcome = await runLogApprovals({ + store, + runId: 'run-1', + intervalMs: 1, + timeoutMs: 10, + onTimeout: 'allow', + sleep: () => { + clock += 10; + return Promise.resolve(); + }, + now: () => new Date(clock) + }).request(request()); + + expect(outcome.decision).toBe('allow'); + }); +}); diff --git a/agentic/run-log-gate/__tests__/gate-decisions.test.ts b/agentic/run-log-gate/__tests__/gate-decisions.test.ts new file mode 100644 index 000000000..47de57d4e --- /dev/null +++ b/agentic/run-log-gate/__tests__/gate-decisions.test.ts @@ -0,0 +1,119 @@ +import type { RunGateDecisionRecord } from '@agentic-kit/harness'; +import type { PiSessionEntry } from '@agentic-kit/run-log'; +import { MemoryRunLogStore, projectToolState, readAll } from '@agentic-kit/run-log'; + +import { gateDecisionEntryId, runLogGateDecisions } from '../src/gate-decisions'; + +const decision = (overrides: Partial = {}): RunGateDecisionRecord => ({ + runId: 'run-1', + toolCallId: 'call-1', + toolName: 'bash', + input: { command: 'rm -rf /' }, + verdict: { decision: 'deny', reason: 'destructive commands are not permitted' }, + decision: 'deny', + reason: 'destructive commands are not permitted', + decidedAt: '2024-01-01T00:00:00.000Z', + ...overrides +}); + +const message = (id: string, text: string): PiSessionEntry => + ({ + type: 'message', + id, + parentId: null, + timestamp: '2024-01-01T00:00:00Z', + message: { role: 'assistant', content: [{ type: 'text', text }] } + }) as PiSessionEntry; + +describe('runLogGateDecisions', () => { + let store: MemoryRunLogStore; + + beforeEach(() => { + store = new MemoryRunLogStore(); + }); + + const recorder = () => runLogGateDecisions({ store, runId: 'run-1' }); + + it('records a policy denial where a projector finds it', async () => { + const decisions = recorder(); + decisions.onDecision(decision()); + await decisions.flush(); + + const projected = projectToolState(await readAll(store, 'run-1')); + expect(projected.gateDecisions['call-1']).toEqual( + expect.objectContaining({ + toolCallId: 'call-1', + toolName: 'bash', + verdict: 'deny', + decision: 'deny', + reason: 'destructive commands are not permitted', + decidedAt: '2024-01-01T00:00:00.000Z' + }) + ); + }); + + it('records the human who settled an asked-for call', async () => { + const decisions = recorder(); + decisions.onDecision( + decision({ + verdict: { decision: 'ask' }, + decision: 'allow', + reason: 'looks fine', + actorId: 'actor-7' + }) + ); + await decisions.flush(); + + expect(projectToolState(await readAll(store, 'run-1')).gateDecisions['call-1']).toEqual( + expect.objectContaining({ verdict: 'ask', decision: 'allow', actorId: 'actor-7' }) + ); + }); + + it('hangs the entry off the log\'s leaf so the session stays one tree', async () => { + await store.append('run-1', [message('entry-1', 'thinking'), message('entry-2', 'calling bash')]); + const decisions = recorder(); + decisions.onDecision(decision()); + await decisions.flush(); + + const records = await readAll(store, 'run-1'); + const entry = records[records.length - 1].entry; + expect(entry).toEqual(expect.objectContaining({ id: gateDecisionEntryId('call-1'), parentId: 'entry-2' })); + }); + + it('writes one entry when the same call is decided twice', async () => { + const decisions = recorder(); + decisions.onDecision(decision()); + decisions.onDecision(decision({ reason: 're-evaluated after a retry' })); + await decisions.flush(); + + // Deterministic ids plus the store's idempotency: a retried execution must + // not double-log its refusals. + expect(await readAll(store, 'run-1')).toHaveLength(1); + }); + + it('keeps the log in decision order', async () => { + const decisions = recorder(); + decisions.onDecision(decision()); + decisions.onDecision(decision({ toolCallId: 'call-2', toolName: 'write_file' })); + await decisions.flush(); + + const records = await readAll(store, 'run-1'); + expect(records.map((record) => record.entry.id)).toEqual([ + gateDecisionEntryId('call-1'), + gateDecisionEntryId('call-2') + ]); + }); + + it('throws the append failure out of flush instead of losing it', async () => { + const failing = runLogGateDecisions({ + store: { + append: () => Promise.reject(new Error('the log is unreachable')), + read: store.read.bind(store) + } as unknown as MemoryRunLogStore, + runId: 'run-1' + }); + failing.onDecision(decision()); + + await expect(failing.flush()).rejects.toThrow('the log is unreachable'); + }); +}); diff --git a/agentic/run-log-gate/__tests__/gate-host.test.ts b/agentic/run-log-gate/__tests__/gate-host.test.ts new file mode 100644 index 000000000..4183da711 --- /dev/null +++ b/agentic/run-log-gate/__tests__/gate-host.test.ts @@ -0,0 +1,119 @@ +import type { RunLogStore } from '@agentic-kit/run-log'; +import { MemoryRunLogStore, projectToolState, readAll } from '@agentic-kit/run-log'; + +import { resolveApproval } from '../src/approvals'; +import { createRunLogGateHost, SKIPPED_REASON } from '../src/gate-host'; + +/** Let every pending read/append settle — the submit is several awaits deep. */ +const flush = (): Promise => new Promise((resolve) => setImmediate(resolve)); + +describe('createRunLogGateHost', () => { + let store: MemoryRunLogStore; + + beforeEach(() => { + store = new MemoryRunLogStore(); + }); + + const host = (overrides: Partial[0]> = {}) => + createRunLogGateHost({ + store, + runId: 'run-1', + intervalMs: 0, + sleep: () => new Promise((resolve) => setImmediate(resolve)), + now: () => new Date('2024-01-01T00:00:00.000Z'), + ...overrides + }); + + it('has a UI: the run log is where the question is asked', () => { + expect(host().hasUI).toBe(true); + }); + + it('asks in the log, with the title and message a human reads', async () => { + const requested: unknown[] = []; + const gate = host({ onRequested: (request) => requested.push(request) }); + const pending = gate.confirmTool('call-1', 'Allow bash?', 'The agent wants to run `ls`.'); + await flush(); + + expect(requested).toEqual([ + { toolCallId: 'call-1', title: 'Allow bash?', message: 'The agent wants to run `ls`.' } + ]); + const records = await readAll(store, 'run-1'); + expect(records.map((record) => record.entry.id)).toEqual(['approval-call-1']); + expect(projectToolState(records).pendingApprovals).toEqual([ + expect.objectContaining({ + toolCallId: 'call-1', + prompt: 'Allow bash?\n\nThe agent wants to run `ls`.' + }) + ]); + + await resolveApproval(store, 'run-1', { toolCallId: 'call-1', decision: 'approved' }); + await expect(pending).resolves.toBe(true); + }); + + it('settles on the resolution a human wrote and files it as the gate decision', async () => { + const decided: unknown[] = []; + const gate = host({ onDecided: (decision) => decided.push(decision) }); + const pending = gate.confirmTool('call-1', 'Allow bash?', 'rm -rf /'); + await flush(); + await resolveApproval(store, 'run-1', { + toolCallId: 'call-1', + decision: 'rejected', + reason: 'not on a Friday', + actorId: 'alice' + }); + + await expect(pending).resolves.toBe(false); + await gate.drain(); + + expect(decided).toEqual([ + { toolCallId: 'call-1', allowed: false, reason: 'not on a Friday', actorId: 'alice' } + ]); + const projected = projectToolState(await readAll(store, 'run-1')); + expect(projected.pendingApprovals).toEqual([]); + expect(projected.gateDecisions['call-1']).toEqual( + expect.objectContaining({ + toolCallId: 'call-1', + toolName: 'Allow bash?', + verdict: 'ask', + decision: 'deny', + reason: 'not on a Friday', + actorId: 'alice', + decidedAt: '2024-01-01T00:00:00.000Z' + }) + ); + }); + + it('denies when nobody answers in time', async () => { + const gate = host({ timeoutMs: 0 }); + await expect(gate.confirmTool('call-1', 'Allow bash?', 'ls')).resolves.toBe(false); + await gate.drain(); + + const projected = projectToolState(await readAll(store, 'run-1')); + expect(projected.gateDecisions['call-1']).toEqual( + expect.objectContaining({ decision: 'deny', reason: expect.stringContaining('no decision') }) + ); + }); + + it('records a skipped repeat as a denial the projector settles the call with', async () => { + const gate = host(); + gate.notifyToolSkipped('call-9'); + await gate.drain(); + + const projected = projectToolState(await readAll(store, 'run-1')); + expect(projected.gateDecisions['call-9']).toEqual( + expect.objectContaining({ toolCallId: 'call-9', decision: 'deny', reason: SKIPPED_REASON }) + ); + }); + + it('rethrows a decision write that failed from drain rather than losing it', async () => { + const failing: RunLogStore = { + read: (runId, cursor, limit) => store.read(runId, cursor, limit), + append: async () => { + throw new Error('log unavailable'); + } + }; + const gate = host({ store: failing }); + gate.notifyToolSkipped('call-9'); + await expect(gate.drain()).rejects.toThrow('log unavailable'); + }); +}); diff --git a/agentic/run-log-gate/jest.config.js b/agentic/run-log-gate/jest.config.js new file mode 100644 index 000000000..8a26efd6d --- /dev/null +++ b/agentic/run-log-gate/jest.config.js @@ -0,0 +1,21 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': [ + 'ts-jest', + { + babelConfig: false, + tsconfig: 'tsconfig.json', + }, + ], + }, + transformIgnorePatterns: [`/node_modules/*`], + testRegex: '(/__tests__/.*\\.(test|spec))\\.(jsx?|tsx?)$', + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], + modulePathIgnorePatterns: ['dist/*'], + moduleNameMapper: { + '^(\\.{1,2}/.*)\\.js$': '$1', + }, +}; diff --git a/agentic/run-log-gate/package.json b/agentic/run-log-gate/package.json new file mode 100644 index 000000000..84a8c3dc6 --- /dev/null +++ b/agentic/run-log-gate/package.json @@ -0,0 +1,43 @@ +{ + "name": "@agentic-kit/run-log-gate", + "version": "0.1.0", + "author": "Constructive ", + "description": "Tool approvals over the run log — the request, the human's answer and every settled gate decision are run-log entries, so desktop, cloud pi and agentic hosts ask and are answered on one surface", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", + "homepage": "https://github.com/constructive-io/constructive", + "license": "SEE LICENSE IN LICENSE", + "publishConfig": { + "access": "public", + "directory": "dist" + }, + "repository": { + "type": "git", + "url": "https://github.com/constructive-io/constructive" + }, + "bugs": { + "url": "https://github.com/constructive-io/constructive/issues" + }, + "scripts": { + "clean": "makage clean", + "prepack": "npm run build", + "build": "makage build", + "build:dev": "makage build --dev", + "lint": "eslint . --fix", + "test": "jest", + "test:watch": "jest --watch" + }, + "keywords": [ + "agentic-kit", + "pi", + "coding-agent", + "run-log", + "approvals", + "constructive" + ], + "peerDependencies": { + "@agentic-kit/harness": "workspace:^", + "@agentic-kit/run-log": "workspace:^" + } +} diff --git a/agentic/run-log-gate/src/approvals.ts b/agentic/run-log-gate/src/approvals.ts new file mode 100644 index 000000000..edd5d40b8 --- /dev/null +++ b/agentic/run-log-gate/src/approvals.ts @@ -0,0 +1,204 @@ +// Approvals over the run log. +// +// The gate needs a human, and a Job has no socket to one: it writes the request +// into the log and reads the answer back out of it. Both directions are rows, so +// a browser hours later still sees the pending request — the log *is* the +// approval UI's state — and the run resumes the moment the resolution lands. +// +// `@agentic-kit/harness` already owns the waiting (`pollingApprovalChannel`); +// this is only its two ends bound to a `RunLogStore`. + +import type { + ApprovalChannel, + ApprovalOutcome, + ApprovalRequest, + PollingApprovalChannelOptions +} from '@agentic-kit/harness'; +import { pollingApprovalChannel } from '@agentic-kit/harness'; +import type { + ApprovalResolutionInput, + PiSessionEntry, + RunEventRecord, + RunLogCursor, + RunLogStore +} from '@agentic-kit/run-log'; +import { + APPROVAL_RESOLUTION_TYPE, + approvalRequestMessage, + approvalResolutionMessage, + assertPiSessionEntry, + isPiMessageEntry, + PI_TRANSCRIPT_FORMAT, + readAll, + START +} from '@agentic-kit/run-log'; + +export interface RunLogApprovalsOptions { + store: RunLogStore; + runId: string; + /** Poll delay while a request is outstanding. */ + intervalMs?: number; + /** Give up after this long; omit to wait as long as a human takes. */ + timeoutMs?: number; + /** Verdict on timeout. The gate defaults it to `deny`. */ + onTimeout?: 'deny' | 'allow'; + /** Records per read while polling. */ + pageLimit?: number; + /** + * The text a human is shown for a request. Defaults to the tool's name and + * the policy's reason; a host that already composed a title and a message + * for the call passes them through here. + */ + prompt?: (request: ApprovalRequest) => string; + now?: () => Date; + sleep?: (ms: number) => Promise; +} + +/** + * The entry id an approval request is written under. + * + * Deterministic on the tool call, so a retried submit is the same entry and the + * store's own idempotency drops the duplicate instead of asking a human twice. + */ +export const approvalEntryId = (toolCallId: string): string => `approval-${toolCallId}`; + +/** + * An `ApprovalChannel` whose request and answer are both records in the run's + * log. + */ +export function runLogApprovals(options: RunLogApprovalsOptions): ApprovalChannel { + const { store, runId } = options; + const now = options.now ?? (() => new Date()); + const prompt = options.prompt ?? defaultPrompt; + // Where reading resumed from last time: an approval may take hours, and + // re-reading the whole run per poll would grow with the transcript. + let cursor: RunLogCursor = START; + // Resolutions seen so far, kept across pages: an answer can land many pages + // after the request it answers. + const resolved = new Map(); + + const refresh = async (): Promise => { + const page = await store.read(runId, cursor, options.pageLimit); + if (page.records.length === 0) return; + cursor = page.cursor; + for (const record of page.records) { + const outcome = resolutionOf(record); + if (outcome) resolved.set(outcome.toolCallId, outcome.outcome); + } + }; + + const channelOptions: PollingApprovalChannelOptions = { + submit: async (request) => { + const parentId = await leafId(store, runId, options.pageLimit); + const entry = { + type: 'message', + id: approvalEntryId(request.toolCallId), + parentId, + timestamp: now().toISOString(), + message: approvalRequestMessage({ + toolCallId: request.toolCallId, + prompt: prompt(request) + }) + } as PiSessionEntry; + await store.append(runId, [entry]); + }, + poll: async (request) => { + await refresh(); + return resolved.get(request.toolCallId); + }, + // One clock for both the entry timestamps and the timeout, so a suite that + // fakes time fakes all of it. + now: () => now().getTime(), + ...(options.intervalMs === undefined ? {} : { intervalMs: options.intervalMs }), + ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }), + ...(options.onTimeout === undefined ? {} : { onTimeout: options.onTimeout }), + ...(options.sleep === undefined ? {} : { sleep: options.sleep }) + }; + + return pollingApprovalChannel(channelOptions); +} + +/** + * Answer a pending request by appending the resolution. + * + * The UI writes this through the API; a local run and the tests write it here. + * Either way it is one more entry in the same log, which is what makes "approve a + * cloud agent from a browser tab" nothing more than an insert. + */ +export async function resolveApproval( + store: RunLogStore, + runId: string, + input: ApprovalResolutionInput, + clock: { now?: () => Date } = {} +): Promise { + const now = clock.now ?? (() => new Date()); + const parentId = await leafId(store, runId); + const entry = { + type: 'message', + id: `resolution-${input.toolCallId}`, + parentId, + timestamp: now().toISOString(), + message: approvalResolutionMessage(input) + } as PiSessionEntry; + await store.append(runId, [entry]); +} + +/** + * A record read as an approval answer, or `undefined` when it is anything else. + * + * The projectors surface a resolution only alongside the tool call it belongs + * to, and a Job polling for its own answer has not necessarily logged that call + * yet, so the answer is read straight off the record. + */ +function resolutionOf( + record: RunEventRecord +): { toolCallId: string; outcome: ApprovalOutcome } | undefined { + // This gate answers a pi session, so a record another harness wrote is not + // its answer to read. + if (record.transcriptFormat !== PI_TRANSCRIPT_FORMAT) return undefined; + const entry = assertPiSessionEntry(record.entry); + if (!isPiMessageEntry(entry)) return undefined; + const message = entry.message; + if (message.role !== 'custom' || message.customType !== APPROVAL_RESOLUTION_TYPE) { + return undefined; + } + const details: Record = + typeof message.details === 'object' && message.details !== null + ? (message.details as Record) + : {}; + const toolCallId = details.toolCallId; + if (typeof toolCallId !== 'string' || toolCallId.length === 0) { + throw new Error( + `approval resolution at seq ${String(record.seq)} carries no toolCallId — the ` + + 'gate cannot tell which tool call it answers' + ); + } + const { reason, actorId } = details; + return { + toolCallId, + outcome: { + decision: details.decision === 'approved' ? 'allow' : 'deny', + ...(typeof reason === 'string' ? { reason } : {}), + ...(typeof actorId === 'string' ? { actorId } : {}) + } + }; +} + +const defaultPrompt = ({ toolName, reason }: ApprovalRequest): string => + reason ? `${toolName}: ${reason}` : `${toolName} needs approval`; + +/** + * The log's last entry id — the parent an out-of-band entry hangs from, so the + * rehydrated session file stays one tree rather than gaining a second root. + * + * Exported for the other writers of out-of-band entries (gate decisions), not + * for the package's consumers. + */ +export async function leafId(store: RunLogStore, runId: string, pageLimit?: number): Promise { + const records = await readAll(store, runId, START, pageLimit); + for (let index = records.length - 1; index >= 0; index -= 1) { + const entry = records[index].entry as { id?: unknown }; + if (typeof entry.id === 'string' && entry.id.length > 0) return entry.id; + } + return null; +} diff --git a/agentic/run-log-gate/src/gate-decisions.ts b/agentic/run-log-gate/src/gate-decisions.ts new file mode 100644 index 000000000..f4787f839 --- /dev/null +++ b/agentic/run-log-gate/src/gate-decisions.ts @@ -0,0 +1,95 @@ +// Gate decisions into the run log. +// +// An `ask` verdict is already durable — the request and its answer are records +// (see `approvals.ts`). A *policy* verdict is not: a rule that denies `bash` +// blocks the call, tells the model why, and leaves no trace, so "which tool +// calls did the gate block, and on whose authority" is unanswerable after the +// run. The gate already offers every settled decision through `onDecision`; this +// binds that callback to the log, which makes the transcript the audit trail for +// refusals as well as for approvals. +// +// The callback is synchronous and an append is not, so writes are queued onto one +// chain: ordering is preserved, the agent loop is never blocked on a log write, +// and the first failure is re-thrown from `flush()` rather than vanishing into an +// unhandled rejection. + +import type { RunGateDecisionRecord } from '@agentic-kit/harness'; +import type { PiSessionEntry, RunLogStore } from '@agentic-kit/run-log'; +import { gateDecisionMessage } from '@agentic-kit/run-log'; + +import { leafId } from './approvals'; + +export interface RunLogGateDecisionsOptions { + store: RunLogStore; + runId: string; + /** Records per read while looking up the parent entry. */ + pageLimit?: number; +} + +export interface RunLogGateDecisions { + /** Hand this to the gate as its `onDecision`. */ + onDecision: (record: RunGateDecisionRecord) => void; + /** + * Wait for the queued writes and throw the first failure. A host calls this + * before it settles the run: a decision that never reached the log is a hole + * in the audit trail, not a detail. + */ + flush(): Promise; +} + +/** + * The entry id a decision is written under. + * + * Deterministic on the tool call, so a retried execution that re-evaluates the + * same call writes the same entry and the store's idempotency drops the + * duplicate instead of logging the decision twice. + */ +export const gateDecisionEntryId = (toolCallId: string): string => `gate-${toolCallId}`; + +export function runLogGateDecisions(options: RunLogGateDecisionsOptions): RunLogGateDecisions { + const { store, runId } = options; + let queue: Promise = Promise.resolve(); + let failure: unknown; + + const append = async (record: RunGateDecisionRecord): Promise => { + const parentId = await leafId(store, runId, options.pageLimit); + const entry = { + type: 'message', + id: gateDecisionEntryId(record.toolCallId), + parentId, + timestamp: record.decidedAt, + message: gateDecisionMessage({ + toolCallId: record.toolCallId, + toolName: record.toolName, + verdict: record.verdict.decision, + decision: record.decision, + ...(record.reason === undefined ? {} : { reason: record.reason }), + ...(record.actorId === undefined ? {} : { actorId: record.actorId }), + decidedAt: record.decidedAt + }) + } as PiSessionEntry; + await store.append(runId, [entry]); + }; + + return { + onDecision: (record) => { + queue = queue.then(async () => { + try { + await append(record); + } catch (error) { + // Keep the first failure: it is the one with the cause, and a later + // write failing for the same reason would only bury it. + failure ??= error; + } + }); + }, + flush: async () => { + await queue; + if (failure !== undefined) { + const error = failure; + failure = undefined; + throw error; + } + } + }; +} diff --git a/agentic/run-log-gate/src/gate-host.ts b/agentic/run-log-gate/src/gate-host.ts new file mode 100644 index 000000000..055c523fb --- /dev/null +++ b/agentic/run-log-gate/src/gate-host.ts @@ -0,0 +1,131 @@ +// The harness's `GateHost`, with the run log as its UI. +// +// `createConfirmGate` asks a host to confirm a call and tells it when a repeat +// of a declined call was skipped. Here both are records: the request goes into +// the run's log as a `constructive.approval.request` entry, the answer is read +// back as the `constructive.approval.resolution` a human wrote — from any +// surface that projects the log — and the outcome is filed as a +// `constructive.gate.decision`, the same audit entry a policy verdict leaves. +// So a desktop run, a cloud Job and the agentic lane ask and are answered in the +// one place, and nothing executes on the strength of a click: the decision is +// what was persisted and read back. +// +// `notifyToolSkipped` returns void, so its write is queued and `drain()` — +// awaited when the turn ends — rethrows the first failure rather than losing it. + +import type { ApprovalRequest, GateHost } from '@agentic-kit/harness'; + +import type { RunLogApprovalsOptions } from './approvals'; +import { runLogApprovals } from './approvals'; +import { runLogGateDecisions } from './gate-decisions'; + +export interface RunLogGateRequest { + toolCallId: string; + title: string; + message: string; +} + +export interface RunLogGateDecision { + toolCallId: string; + allowed: boolean; + reason?: string; + actorId?: string; +} + +export interface RunLogGateHostOptions extends Omit { + /** Told when a request is pending, so a host with a window can surface it. */ + onRequested?: (request: RunLogGateRequest) => void; + /** Told what the log settled on, so a host can stop asking. */ + onDecided?: (decision: RunLogGateDecision) => void; +} + +export interface RunLogGateHost extends GateHost { + /** + * Wait for the queued writes and throw the first failure. Awaited once the + * turn ends: a decision that never reached the log is a hole in the audit + * trail, not a detail. + */ + drain(): Promise; +} + +/** The text a human sees: the harness's title, then its message. */ +export const gatePrompt = (title: string, message: string): string => `${title}\n\n${message}`; + +/** Reason filed for a repeat of a declined call the gate skipped without asking. */ +export const SKIPPED_REASON = 'skipped: a repeat of a tool call the user already declined'; + +export function createRunLogGateHost(options: RunLogGateHostOptions): RunLogGateHost { + const { onRequested, onDecided, ...channelOptions } = options; + const { store, runId } = options; + const now = options.now ?? (() => new Date()); + + // Prompts are composed per call and the channel writes one per request, so + // the title and message travel by tool call id rather than through the + // channel's request shape. + const prompts = new Map(); + const channel = runLogApprovals({ + ...channelOptions, + prompt: (request: ApprovalRequest) => prompts.get(request.toolCallId) ?? request.toolName + }); + const decisions = runLogGateDecisions({ + store, + runId, + ...(options.pageLimit === undefined ? {} : { pageLimit: options.pageLimit }) + }); + + return { + hasUI: true, + + async confirmTool(toolCallId, title, message) { + prompts.set(toolCallId, gatePrompt(title, message)); + onRequested?.({ toolCallId, title, message }); + try { + const outcome = await channel.request({ + runId, + toolCallId, + toolName: title, + input: {}, + reason: message, + requestedAt: now().toISOString() + }); + decisions.onDecision({ + runId, + toolCallId, + toolName: title, + input: {}, + verdict: { decision: 'ask' }, + decision: outcome.decision, + ...(outcome.reason === undefined ? {} : { reason: outcome.reason }), + ...(outcome.actorId === undefined ? {} : { actorId: outcome.actorId }), + decidedAt: now().toISOString() + }); + onDecided?.({ + toolCallId, + allowed: outcome.decision === 'allow', + ...(outcome.reason === undefined ? {} : { reason: outcome.reason }), + ...(outcome.actorId === undefined ? {} : { actorId: outcome.actorId }) + }); + return outcome.decision === 'allow'; + } finally { + prompts.delete(toolCallId); + } + }, + + notifyToolSkipped(toolCallId) { + // The harness names only the call, and the tool it repeats is in the + // transcript beside it; the projector attributes the entry by call id. + decisions.onDecision({ + runId, + toolCallId, + toolName: '', + input: {}, + verdict: { decision: 'deny', reason: SKIPPED_REASON }, + decision: 'deny', + reason: SKIPPED_REASON, + decidedAt: now().toISOString() + }); + }, + + drain: () => decisions.flush() + }; +} diff --git a/agentic/run-log-gate/src/index.ts b/agentic/run-log-gate/src/index.ts new file mode 100644 index 000000000..bab25baee --- /dev/null +++ b/agentic/run-log-gate/src/index.ts @@ -0,0 +1,11 @@ +export type { RunLogApprovalsOptions } from './approvals'; +export { approvalEntryId, resolveApproval, runLogApprovals } from './approvals'; +export type { RunLogGateDecisions, RunLogGateDecisionsOptions } from './gate-decisions'; +export { gateDecisionEntryId, runLogGateDecisions } from './gate-decisions'; +export type { + RunLogGateDecision, + RunLogGateHost, + RunLogGateHostOptions, + RunLogGateRequest +} from './gate-host'; +export { createRunLogGateHost, gatePrompt, SKIPPED_REASON } from './gate-host'; diff --git a/agentic/run-log-gate/tsconfig.esm.json b/agentic/run-log-gate/tsconfig.esm.json new file mode 100644 index 000000000..624ab17cf --- /dev/null +++ b/agentic/run-log-gate/tsconfig.esm.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "es2022", + "outDir": "dist/esm" + } +} diff --git a/agentic/run-log-gate/tsconfig.json b/agentic/run-log-gate/tsconfig.json new file mode 100644 index 000000000..df063b5ee --- /dev/null +++ b/agentic/run-log-gate/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 89a09b412..4ee770bea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -361,6 +361,16 @@ importers: version: link:../run-log/dist publishDirectory: dist + agentic/run-log-gate: + dependencies: + '@agentic-kit/harness': + specifier: workspace:^ + version: link:../harness/dist + '@agentic-kit/run-log': + specifier: workspace:^ + version: link:../run-log/dist + publishDirectory: dist + examples/codegen-integration: dependencies: '@0no-co/graphql.web':