diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..c7bbe58 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,20 @@ +name: test + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '18.17' + - run: npm install + - run: npm test diff --git a/test/conformance.test.mjs b/test/conformance.test.mjs new file mode 100644 index 0000000..5859574 --- /dev/null +++ b/test/conformance.test.mjs @@ -0,0 +1,254 @@ +// L1 shared conformance corpus adapter for muse-code-acp-plugin. +// davidcrowe/gatewaystack-connect#1344 — one corpus, one adapter per plugin, +// run against the plugin's real entry point (decide(), imported directly — +// this is exactly how test/hook.test.mjs already exercises it) against a +// fake gateway, asserting on what a person would see or what the plugin sent. +// +// See ../../gsc-conformance/conformance/plugin-corpus.json (vendored below at +// test/fixtures/plugin-corpus.json) for the corpus schema, case definitions +// and rationale. + +import { test, before, after } from 'node:test' +import assert from 'node:assert/strict' +import { createServer } from 'node:http' +import { createHash } from 'node:crypto' +import { readFileSync, mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' +import { decide } from '../hook.mjs' + +const HERE = dirname(fileURLToPath(import.meta.url)) +const CORPUS_PATH = join(HERE, 'fixtures', 'plugin-corpus.json') +const PINNED_FINGERPRINT = 'aa186d3fb3e7d18c' // sha256 of the corpus file's raw bytes, first 16 hex chars + +// Any capability muse-code fails despite being marked "supported" in the +// corpus goes here, e.g. { case: 'notice-shown', issue: 'NEW', detail: '...' }. +// Kept empty unless a real failure was observed against this worktree's +// hook.mjs — see the final test in this file, which asserts this list is +// exactly what happened, so neither a new failure nor a quiet fix can hide. +const EXPECTED_DIVERGENCES = [] + +// --- Fingerprint gate: fail loudly before trusting a single byte of the +// vendored corpus. --- +const corpusRaw = readFileSync(CORPUS_PATH) // raw bytes — never re-parse-then-hash +const corpusFingerprint = createHash('sha256').update(corpusRaw).digest('hex').slice(0, 16) + +test('vendored corpus matches the pinned fingerprint', () => { + assert.equal( + corpusFingerprint, + PINNED_FINGERPRINT, + `test/fixtures/plugin-corpus.json fingerprint ${corpusFingerprint} != pinned ${PINNED_FINGERPRINT} — ` + + 're-vendor a byte-identical copy from the canonical corpus (davidcrowe/gatewaystack-connect:conformance/plugin-corpus.json)', + ) +}) + +// Only parse (and only trust case/harness data) once the byte fingerprint above +// has been computed against the raw file — the assertion below still runs even +// if the parse fails. +const corpus = JSON.parse(corpusRaw.toString('utf8')) +const MARKER = corpus.marker // 'ACPCONF7F3A' +const PLUGIN_NAME = 'muse-code-acp-plugin' + +function corpusCase(id) { + const c = corpus.cases.find(c => c.id === id) + assert.ok(c, `corpus is missing case "${id}"`) + return c +} + +function harnessStatus(capability) { + const row = corpus.harnesses.find(h => h.plugin === PLUGIN_NAME && h.capability === capability) + assert.ok(row, `corpus has no harnesses row for ${PLUGIN_NAME}/${capability}`) + return row.status +} + +test('corpus lists both muse-code-acp-plugin capabilities as supported', () => { + assert.equal(harnessStatus('notice'), 'supported') + assert.equal(harnessStatus('post-tool'), 'supported') +}) + +// --- Fake gateway ----------------------------------------------------- +// Every path other than /govern/tool-output answers a bare allow (the notice +// and post-tool cases only ever exercise PostToolUse, but this keeps the +// fake gateway honest against the adapterMust contract for other capabilities). +let server, gatewayBase +let nextToolOutputReply = { decision: 'allow' } +const requests = [] + +before(async () => { + server = createServer((req, res) => { + let body = '' + req.on('data', c => { body += c }) + req.on('end', () => { + let parsed = null + try { parsed = body ? JSON.parse(body) : null } catch { /* leave null */ } + requests.push({ path: req.url, body: parsed }) + res.writeHead(200, { 'Content-Type': 'application/json' }) + if (req.url === '/govern/tool-output') { + res.end(JSON.stringify(nextToolOutputReply)) + } else { + res.end(JSON.stringify({ decision: 'allow' })) + } + }) + }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + // Node 18 runs a file's root after() only once the event loop drains, which + // a listening server prevents: unref it so the file can finish. + server.unref() + gatewayBase = `http://127.0.0.1:${server.address().port}` +}) + +after(() => server.close()) + +// Fresh HOME per call: readToken()/readConfig() resolve ~/.acp under env.HOME, +// so this keeps the developer machine's real ~/.acp/credentials, config.json +// (which could set shadow/agent_tier) and muse-sessions/ state completely out +// of the picture — no cross-test or cross-machine leakage, and no chance a +// stale per-session marker on disk suppresses a notice the corpus expects. +function freshEnv(overrides = {}) { + const home = mkdtempSync(join(tmpdir(), 'muse-acp-conformance-')) + return { + HOME: home, + ACP_BEARER_TOKEN: 'gsk_test_dummy0000000000000000', // dummy credential, never a real key + ACP_GOVERN_BASE: gatewayBase, + ACP_CHECK_TIMEOUT_MS: '1500', + // Explicitly absent unless a case's own env sets it — never inherit the + // real process's ACP_SHADOW. + ...overrides, + } +} + +// ======================================================================= +// capability: notice +// ======================================================================= +// +// hook.mjs's PostToolUse branch (around decide()'s PostToolUse block) puts a +// gateway notice on the person-visible channel like this: +// +// return { out: { systemMessage: data.notice }, warn: data.notice } +// +// runHook() (the real stdin/stdout entry point) then does: +// if (result.warn) process.stderr.write(result.warn + '\n') // channel 1: stderr +// process.stdout.write(JSON.stringify(result.out) + '\n') // channel 2: stdout JSON `systemMessage` +// +// decide() is exactly what runHook() calls internally (and exactly what +// test/hook.test.mjs already imports and drives directly), so `warn` and +// `out.systemMessage` here ARE those two person-visible channels, not a +// stand-in for them. We assert both. + +test('notice-shown: shadow-mode notice reaches stderr and the stdout systemMessage JSON', async () => { + const kase = corpusCase('notice-shown') + nextToolOutputReply = kase.gatewayReply + requests.length = 0 + + const { out, warn } = await decide( + { + hook_event_name: 'PostToolUse', + tool_name: 'shell', + tool_input: { command: 'echo hi' }, + tool_output: 'hi\n', + session_id: 'notice-shown-session', + }, + freshEnv(kase.env), + ) + + assert.equal(requests[0]?.path, '/govern/tool-output') + + const personSees = typeof out.systemMessage === 'string' && out.systemMessage.includes(MARKER) + const stderrSees = typeof warn === 'string' && warn.includes(MARKER) + + if (kase.expect.personSees) { + assert.ok(personSees, `expected out.systemMessage (stdout JSON channel) to contain ${MARKER}, got: ${JSON.stringify(out)}`) + assert.ok(stderrSees, `expected warn (stderr channel) to contain ${MARKER}, got: ${JSON.stringify(warn)}`) + } else { + assert.ok(!personSees && !stderrSees, 'expected no person-visible channel to carry the notice') + } +}) + +test('notice-shadow-off: ACP_SHADOW=off silences the notice on every channel', async () => { + const kase = corpusCase('notice-shadow-off') + assert.equal(kase.env.ACP_SHADOW, 'off') + nextToolOutputReply = kase.gatewayReply + requests.length = 0 + + const { out, warn } = await decide( + { + hook_event_name: 'PostToolUse', + tool_name: 'shell', + tool_input: { command: 'echo hi' }, + tool_output: 'hi\n', + session_id: 'notice-shadow-off-session', + }, + freshEnv(kase.env), + ) + + assert.equal(requests[0]?.path, '/govern/tool-output') + + const stdoutSerialized = JSON.stringify(out) + const warnSerialized = warn === undefined ? '' : String(warn) + + assert.equal(kase.expect.personSees, false) + assert.ok(!stdoutSerialized.includes(MARKER), `expected stdout JSON to be free of ${MARKER}, got: ${stdoutSerialized}`) + assert.ok(!warnSerialized.includes(MARKER), `expected stderr to be free of ${MARKER}, got: ${warnSerialized}`) +}) + +// ======================================================================= +// capability: post-tool +// ======================================================================= +// +// Canonical tool-name mapping declared here (adapterMust: "assert tool_name +// equals the native tool name the adapter fed in, or the plugin's documented +// canonical mapping of it"): muse-code's own hook payload schema is not yet +// publicly published (see hook.mjs's "CONTRACT STATUS" block) — the shipped +// code reads snake_case Claude-Code-shaped keys (tool_name, tool_input, +// tool_output, session_id, hook_event_name) as its primary/native spelling, +// with camelCase and a few aliases accepted defensively. hook.mjs's +// normalize() -> decide() forwards call.toolName straight through as the +// outgoing gateway body's tool_name with NO renaming or taxonomy mapping. +// So the mapping this adapter declares is the identity mapping: whatever +// native tool identifier we feed in as `tool_name` must come out unchanged +// as the gateway payload's `tool_name`. +const NATIVE_TOOL_NAME_MAP = Object.freeze({ shell: 'shell' }) + +test('post-tool-fields: outgoing /govern/tool-output body carries the required fields', async () => { + const kase = corpusCase('post-tool-fields') + nextToolOutputReply = kase.gatewayReply + requests.length = 0 + + // muse-code's native PostToolUse hook payload, built from the corpus call. + const nativePayload = { + hook_event_name: 'PostToolUse', + tool_name: kase.call.tool, // 'shell' — fed straight through, see NATIVE_TOOL_NAME_MAP above + tool_input: { command: kase.call.command }, + tool_output: kase.call.output, + session_id: kase.call.sessionId, + cwd: '/work', + } + + await decide(nativePayload, freshEnv(kase.env)) + + const sent = requests.find(r => r.path === '/govern/tool-output') + assert.ok(sent, 'expected a POST to /govern/tool-output') + const body = sent.body + + assert.equal(body.hook_event_name, kase.expect.hook_event_name) + assert.equal(body.tool_name, NATIVE_TOOL_NAME_MAP[nativePayload.tool_name]) + assert.equal(body.tool_name, nativePayload.tool_name) // equals the native tool name fed in + + assert.equal(typeof body.tool_input, 'object') + assert.ok(body.tool_input && !Array.isArray(body.tool_input)) + assert.ok(JSON.stringify(body.tool_input).includes(MARKER), `tool_input serialisation missing ${MARKER}: ${JSON.stringify(body.tool_input)}`) + + assert.ok(JSON.stringify(body.tool_output).includes(MARKER), `tool_output serialisation missing ${MARKER}: ${JSON.stringify(body.tool_output)}`) + + assert.equal(typeof body.session_id, 'string') + assert.ok(body.session_id.length > 0) +}) + +// ======================================================================= +// EXPECTED_DIVERGENCES must exactly match reality: neither a new failure +// nor a quiet fix can pass CI silently. +// ======================================================================= +test('EXPECTED_DIVERGENCES is exactly empty for muse-code-acp-plugin', () => { + assert.deepEqual(EXPECTED_DIVERGENCES, []) +}) diff --git a/test/fixtures/plugin-corpus.json b/test/fixtures/plugin-corpus.json new file mode 100644 index 0000000..8f0d2de --- /dev/null +++ b/test/fixtures/plugin-corpus.json @@ -0,0 +1,90 @@ +{ + "corpus": "acp-plugin-conformance", + "version": 1, + "canonicalHome": "davidcrowe/gatewaystack-connect:conformance/plugin-corpus.json", + "tracking": "davidcrowe/gatewaystack-connect#1344 (L1 shared conformance corpus, build step 1)", + "purpose": "One table of plugin capabilities, run against EVERY ACP harness plugin through a thin per-plugin adapter. The adapter drives the plugin's real entry point against a fake gateway on 127.0.0.1 (or a stubbed fetch/urlopen where the repo already does that) and asserts on what the person would see or on what the plugin sent. Unit tests stayed green while four plugins dropped every gateway notice (#1334); this corpus is the seam test that would have caught it.", + "vendoring": "Each plugin vendors a byte-identical copy at test/fixtures/plugin-corpus.json (tests/fixtures/ for pytest repos). Its adapter pins the fingerprint below and fails when the copy differs. To change the corpus: edit this file, recompute the fingerprint, and update every plugin's copy and pin in the same change.", + "fingerprint": "sha256 of the raw file bytes, first 16 hex characters. Byte hashing (not a hash of a re-serialised object) so Node and Python compute the same value without agreeing on JSON serialisation.", + "divergences": "A plugin that fails a supported capability records it in its adapter's EXPECTED_DIVERGENCES list with the issue number. The adapter asserts the list exactly: a NEW failure fails CI, and a FIX also fails CI until the entry is removed. Neither can be quietly forgotten. A divergence is not a not-possible row: not-possible means the harness gives the plugin no way to do it.", + "marker": "ACPCONF7F3A", + "capabilities": { + "notice": { + "contract": "When the gateway's reply to POST /govern/tool-output contains notice: \"\", that text reaches the person-visible channel of the harness (stdout systemMessage, stderr, a toast, a UI notify call, a logger the harness shows, or an editor protocol message). ACP_SHADOW=off silences it on the client side.", + "adapterMust": "Answer every other gateway path with an allow verdict. Isolate HOME (or the plugin's state dir) to a temp directory so first-per-session markers from the developer's machine cannot suppress the notice." + }, + "post-tool": { + "contract": "Given the harness's NATIVE post-tool payload, the plugin's outgoing POST /govern/tool-output request body carries tool_name, tool_input, tool_output, session_id and hook_event_name \"PostToolUse\".", + "adapterMust": "Build the payload in the harness's own native shape (its own field names and tool name) from the canonical call below. Assert tool_name equals the native tool name the adapter fed in, or the plugin's documented canonical mapping of it (the adapter names that mapping explicitly)." + } + }, + "cases": [ + { + "id": "notice-shown", + "capability": "notice", + "env": {}, + "gatewayReply": { "decision": "allow", "notice": "ACPCONF7F3A shadow mode: this call would have been held for review" }, + "expect": { "personSees": true, "contains": "ACPCONF7F3A" }, + "issue": "#1334", + "why": "Codex, OpenCode, Hermes and fx dropped every notice while their unit tests stayed green." + }, + { + "id": "notice-shadow-off", + "capability": "notice", + "env": { "ACP_SHADOW": "off" }, + "gatewayReply": { "decision": "allow", "notice": "ACPCONF7F3A shadow mode: this call would have been held for review" }, + "expect": { "personSees": false, "contains": "ACPCONF7F3A" }, + "issue": "#1334", + "why": "ACP_SHADOW=off is the client-side belt to the server's own shadow switch." + }, + { + "id": "post-tool-fields", + "capability": "post-tool", + "env": {}, + "call": { + "tool": "shell", + "command": "echo ACPCONF7F3A", + "output": "ACPCONF7F3A\n", + "sessionId": "acpconf-session-0001" + }, + "gatewayReply": { "decision": "allow" }, + "expect": { + "method": "POST", + "path": "/govern/tool-output", + "hook_event_name": "PostToolUse", + "tool_name": "equals the native tool name fed in, or the adapter's declared canonical mapping", + "tool_input": "a JSON object whose serialisation contains the marker", + "tool_output": "a value whose serialisation contains the marker", + "session_id": "a non-empty string" + }, + "issue": "#1344", + "why": "fx's post-tool call sends no tool_name or tool_input, so the gateway cannot scan or attribute the output." + } + ], + "harnesses": [ + { "plugin": "claude-code-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "claude-code-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "codex-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "codex-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "opencode-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "opencode-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "hermes-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "hermes-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "pi-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "pi-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "grok-build-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "grok-build-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "antigravity-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "antigravity-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "openclaw-acp-plugin", "capability": "notice", "status": "not-possible", "reason": "OpenClaw exposes no after-tool hook to plugins, so the plugin never calls /govern/tool-output and has no reply to surface. Revisit when OpenClaw ships one." }, + { "plugin": "openclaw-acp-plugin", "capability": "post-tool", "status": "not-possible", "reason": "OpenClaw exposes no after-tool hook to plugins, so there is no native post-tool payload to forward. Revisit when OpenClaw ships one." }, + { "plugin": "dsh-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "dsh-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "fx-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "fx-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "muse-code-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "muse-code-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "prime-agent-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "prime-agent-acp-plugin", "capability": "post-tool", "status": "supported" } + ] +} diff --git a/test/hook.test.mjs b/test/hook.test.mjs index 4376997..1cd8633 100644 --- a/test/hook.test.mjs +++ b/test/hook.test.mjs @@ -26,6 +26,9 @@ before(async () => { }) }) await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + // Node 18 runs a file's root after() only once the event loop drains, which + // a listening server prevents: unref it so the file can finish. + server.unref() base = `http://127.0.0.1:${server.address().port}` })