stash eql verify: assert the installed EQL surface is complete

.changeset/eql-verify-surface.md

@@ -0,0 +1,5 @@
+---
+'stash': minor
+---
+
+New `stash eql verify`: assert the installed EQL surface is complete and coherent, independent of any application schema. A partial install — domains present, some of their comparison functions or operators absent — used to report success at install time and fail at query time on a specific predicate (e.g. `weight >= x`); nothing detected it. `eql verify` compares the database against everything the pinned bundle installs (every domain, function overload, operator, cast, and the ORE operator class) via read-only catalog queries, reports damage grouped per domain, and distinguishes expected absence from damage: the ORE operator class being skipped on managed Postgres, with its loud-failure fallback in place, reads as the supported configuration it is rather than a failed install. Exit 0 means exactly one thing — the surface was checked and found complete; damage, EQL absent, and a version mismatch with the pinned bundle (nothing verifiable) all exit 1. `--json` emits the structured report for agents. `stash eql install` now runs the same check automatically before declaring success, on the fresh-install path and the already-installed early exit alike — there, only damage fails the install: a version mismatch warns and continues, so a no-op re-run over an older EQL stays exit 0 for idempotent provisioning scripts. A valueless `--database-url` (booleanised by the parser when the next token is another flag) is now rejected up front on every command instead of silently falling back to `DATABASE_URL` — previously `eql install --database-url --force` could drop and reinstall the EQL schemas on a database the command never named.

.github/workflows/tests.yml

@@ -259,8 +259,27 @@ jobs:
           echo "DATABASE_URL=postgres://cipherstash:password@localhost:5432/cipherstash" >> ./packages/stack/.env
 
       # Run TurboRepo tests
+      #
+      # STASH_TEST_DATABASE_URL points the CLI's live-Postgres suites
+      # (`packages/cli/src/**/__tests__/**.live.test.ts`) at this job's
+      # service container — without it they self-skip, and the only guard on
+      # the `stash eql verify` parser↔catalog spelling seam
+      # (`installer/__tests__/verify.live.test.ts`) would run in no CI
+      # workflow at all: a routine `@cipherstash/eql` bump could then make
+      # every `stash eql install` fail with phantom damage, on green CI.
+      # These suites need Postgres only, no CipherStash credentials; the
+      # verify suite installs EQL v3 into its own schemas, which coexists
+      # with the image's pre-installed EQL v2 that the stack tests use.
+      # They share that one database, so the CLI vitest config runs them
+      # serially (the `live` project sets `fileParallelism: false` —
+      # verify.live's bundle install opens with DROP SCHEMA … CASCADE, which
+      # races destructively under the other suites in parallel forks).
+      # (`supabase-push.live.test.ts` gates on different env vars and still
+      # skips here — it needs the Supabase CLI binary.)
       - name: Run tests
         run: pnpm run test
+        env:
+          STASH_TEST_DATABASE_URL: postgres://cipherstash:password@localhost:5432/cipherstash
 
       # CLI E2E tests drive the built `dist/bin/stash.js` through a real
       # pseudo-terminal via node-pty. Run via turbo so the `^build` + `build`

packages/cli/src/bin/main.ts

@@ -111,6 +111,7 @@ Commands:
 
   eql preflight        Report whether this database role can install EQL, before trying
   eql install          Scaffold stash.config.ts (if missing) and install EQL extensions
+  eql verify           Check the installed EQL surface is complete (catches partial installs)
   eql migration        Generate an EQL v3 install migration (Drizzle, or supabase/migrations/)
   eql repair           Repair migrations with an un-runnable ALTER COLUMN to an encrypted type
   eql upgrade          Upgrade EQL extensions to the latest version
@@ -257,6 +258,32 @@ function rejectRetiredEqlFlags(
   }
 }
 
+/**
+ * `parseArgs` booleanises a `--database-url` whose value is missing (next
+ * token starts with `-`, or the flag is last), so a typo'd `--database-url
+ * --force` would silently fall back to env/config resolution — and the
+ * command would act on a different database than the user targeted, without
+ * ever naming it. Harmless for the read-only diagnostics, catastrophic for
+ * `eql install --force` (DROP SCHEMA … CASCADE, no confirmation). A valueless
+ * `--database-url` is always a typo, so `dispatch()` rejects it for EVERY
+ * command rather than per-case, keeping stdout parseable in `--json` mode
+ * (same pattern as `stash env`'s `nameMissingValue`).
+ */
+async function rejectMissingDatabaseUrlValue(
+  flags: Record<string, boolean>,
+): Promise<void> {
+  if (flags['database-url'] !== true) return
+  const message =
+    '`--database-url` needs a value (e.g. --database-url postgres://...). Without one the command would silently resolve a different database from DATABASE_URL or stash.config.ts.'
+  if (flags.json) {
+    const { emitJsonError } = await import('../commands/auth/events.js')
+    emitJsonError('missing_flag_value', message)
+  } else {
+    p.log.error(message)
+  }
+  throw new CliExit(1)
+}
+
 async function runEqlCommand(
   sub: string | undefined,
   flags: Record<string, boolean>,
@@ -272,6 +299,14 @@ async function runEqlCommand(
     case 'install':
       await runInstall(flags, values)
       break
+    case 'verify': {
+      const { verifyCommand } = await import('../commands/eql/verify.js')
+      await verifyCommand({
+        databaseUrl: values['database-url'],
+        json: flags.json,
+      })
+      break
+    }
     case 'migration': {
       const { eqlMigrationCommand } = await import(
         '../commands/eql/migration.js'
@@ -544,6 +579,7 @@ async function dispatch(
   flags: Record<string, boolean>,
   values: Record<string, string>,
 ) {
+  await rejectMissingDatabaseUrlValue(flags)
   switch (command) {
     case 'init':
       await initCommand(flags, values)

packages/cli/src/cli/registry.ts

@@ -344,6 +344,53 @@ export const registry: CommandGroup[] = [
           DATABASE_URL_FLAG,
         ],
       },
+      {
+        name: 'eql verify',
+        summary:
+          'Check the installed EQL surface is complete, not just present',
+        long: [
+          'Compare what the database actually has against everything the pinned',
+          'EQL v3 bundle installs — every domain, function overload, operator,',
+          'cast, and the ORE operator class — via read-only catalog queries. A',
+          'partial install (domains present, some comparison functions or',
+          'operators absent) reports success at install time and fails at query',
+          'time on a specific predicate; this is the check that catches it.',
+          '',
+          'Expected absences read as such: on managed Postgres the bundle',
+          'legitimately skips the ORE operator class (creating it requires',
+          'superuser) and poisons the `_ord_ore` domains to fail loudly — that',
+          'is a supported configuration, reported as info.',
+          '',
+          'Exit 0 means exactly one thing: the surface was checked and found',
+          'complete. Damage, EQL not installed, and a version mismatch with',
+          'the pinned bundle all exit 1 — on a mismatch the object-level diff',
+          'is skipped (the pinned bundle is the wrong manifest to compare',
+          'against) and the command suggests `eql upgrade` (or a one-shot',
+          '`eql install --force --database-url ...` where no stash.config.ts',
+          'exists — `eql upgrade` requires one).',
+          '',
+          'Runs automatically at the end of `stash eql install`, on the',
+          'fresh-install path and the already-installed early exit alike.',
+          'There, only damage fails the install — a version mismatch warns',
+          'and continues, keeping a no-op re-run over an older EQL exit 0',
+          'for idempotent provisioning scripts.',
+        ].join('\n'),
+        examples: ['eql verify', 'eql verify --json'],
+        flags: [
+          {
+            name: '--json',
+            description:
+              'Emit the machine-readable verification report instead of the table.',
+          },
+          {
+            name: '--database-url',
+            value: '<url>',
+            description:
+              "One-shot, like `eql install`'s: bypasses config loading entirely, so the database you name is the database that gets judged. Also settable via DATABASE_URL.",
+            env: 'DATABASE_URL',
+          },
+        ],
+      },
       {
         name: 'eql migration',
         summary:

packages/cli/src/commands/db/__tests__/install-verify-gate.test.ts

@@ -0,0 +1,155 @@
+/**
+ * `verifySurfaceOrExit` — the #890 gate `installCommand` runs on both the
+ * fresh-install tail and the already-installed early exit. The differ's own
+ * behaviour is covered in `installer/__tests__/verify.test.ts`; what lives
+ * here is the install-side POLICY layered on top of the report:
+ *
+ * - damage exits 1 (a committed-but-incomplete install must not read as
+ *   success);
+ * - a version mismatch does NOT — `ok: false` there means "nothing was
+ *   checked", and a no-op `eql install` re-run over an older EQL was exit 0
+ *   before verification existed. Idempotent provisioning scripts (and
+ *   `stash init`, whose direct-install route calls `installCommand` inside a
+ *   try/catch that `process.exit` escapes) depend on that. `stash eql verify`
+ *   keeps the strict gate; the installer must not inherit it.
+ * - a verification error (connection dropped) warns and continues — the
+ *   install itself is committed.
+ */
+
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type { VerifyReport } from '@/installer/verify.js'
+import { verifySurfaceOrExit } from '../install.js'
+
+const clack = vi.hoisted(() => ({
+  spinnerInstance: { start: vi.fn(), stop: vi.fn() },
+  log: {
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+    success: vi.fn(),
+    step: vi.fn(),
+  },
+  intro: vi.fn(),
+  note: vi.fn(),
+  outro: vi.fn(),
+}))
+vi.mock('@clack/prompts', () => ({
+  spinner: vi.fn(() => clack.spinnerInstance),
+  log: clack.log,
+  intro: clack.intro,
+  note: clack.note,
+  outro: clack.outro,
+}))
+
+const verifier = vi.hoisted(() => ({ verifyEqlSurface: vi.fn() }))
+vi.mock('@/installer/verify.js', () => ({
+  verifyEqlSurface: verifier.verifyEqlSurface,
+}))
+
+// Imported dynamically by the damage path for its findings renderer.
+const findingsReporter = vi.hoisted(() => ({ reportVerifyFindings: vi.fn() }))
+vi.mock('../../eql/verify.js', () => ({
+  reportVerifyFindings: findingsReporter.reportVerifyFindings,
+}))
+
+function report(overrides: Partial<VerifyReport>): VerifyReport {
+  return {
+    status: 'complete',
+    bundleVersion: '3.0.4',
+    installedVersion: '3.0.4',
+    counts: null,
+    ore: null,
+    findings: [],
+    ok: true,
+    ...overrides,
+  }
+}
+
+function spinner() {
+  return clack.spinnerInstance as unknown as ReturnType<
+    typeof import('@clack/prompts').spinner
+  >
+}
+
+describe('verifySurfaceOrExit', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('returns without exiting on a complete surface', async () => {
+    verifier.verifyEqlSurface.mockResolvedValueOnce(report({}))
+    await expect(
+      verifySurfaceOrExit('postgres://db', spinner(), { remedy: 'r' }),
+    ).resolves.toBeUndefined()
+  })
+
+  it('exits 1 on damage, after reporting the findings and the remedy', async () => {
+    verifier.verifyEqlSurface.mockResolvedValueOnce(
+      report({
+        status: 'incomplete',
+        ok: false,
+        findings: [
+          { severity: 'damage', kind: 'operator', message: 'op missing' },
+        ],
+      }),
+    )
+    const exit = vi
+      .spyOn(process, 'exit')
+      .mockImplementation((code?: string | number | null | undefined) => {
+        throw new Error(`exit ${code}`)
+      })
+    try {
+      await expect(
+        verifySurfaceOrExit('postgres://db', spinner(), {
+          remedy: 'use --force',
+        }),
+      ).rejects.toThrow('exit 1')
+      expect(findingsReporter.reportVerifyFindings).toHaveBeenCalled()
+      expect(clack.log.error).toHaveBeenCalledWith('use --force')
+    } finally {
+      exit.mockRestore()
+    }
+  })
+
+  it('does NOT exit on a version mismatch — warns with the skew instead', async () => {
+    const mismatch = report({
+      status: 'version-mismatch',
+      installedVersion: '3.0.2',
+      ok: false,
+      findings: [
+        {
+          severity: 'warning',
+          kind: 'version',
+          message:
+            'EQL 3.0.2 installed, CLI pins 3.0.4 — run stash eql upgrade',
+        },
+      ],
+    })
+    verifier.verifyEqlSurface.mockResolvedValueOnce(mismatch)
+    const exit = vi.spyOn(process, 'exit').mockImplementation(() => {
+      throw new Error('exit called')
+    })
+    try {
+      await expect(
+        verifySurfaceOrExit('postgres://db', spinner(), { remedy: 'r' }),
+      ).resolves.toBeUndefined()
+      expect(clack.log.warn).toHaveBeenCalledWith(mismatch.findings[0].message)
+      expect(clack.log.error).not.toHaveBeenCalled()
+      expect(exit).not.toHaveBeenCalled()
+    } finally {
+      exit.mockRestore()
+    }
+  })
+
+  it('warns and continues when verification itself errors', async () => {
+    verifier.verifyEqlSurface.mockRejectedValueOnce(
+      new Error('connection terminated'),
+    )
+    await expect(
+      verifySurfaceOrExit('postgres://db', spinner(), { remedy: 'r' }),
+    ).resolves.toBeUndefined()
+    expect(clack.log.warn).toHaveBeenCalledWith(
+      expect.stringContaining('connection terminated'),
+    )
+  })
+})