test(driver-sql): name the dialects a local run did not exercise, under the counts

packages/drivers/driver-sql/src/live-dialect-coverage.reporter.ts

@@ -0,0 +1,267 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * #18200 — say, at the end of the run, which dialects this run did NOT exercise.
+ *
+ * ## The failure this exists for
+ *
+ * A local `pnpm --filter @objectstack/driver-sql test` with no servers present
+ * ends on a line a round reads as coverage:
+ *
+ * ```
+ *   Test Files  178 passed | 11 skipped (189)
+ *        Tests  2627 passed | 168 skipped (2795)
+ * ```
+ *
+ * Every live-Postgres and live-MySQL cell in this package is inside those
+ * skipped counts, and `178 passed` is SQLite only. Measured cost, three
+ * recorded instances: #17469 round 1 shipped four red CI jobs after a clean
+ * local sweep, one of them a live-PG-only fixture nothing local could have run;
+ * #17231 declared its live cells skipped locally and then went red on the MySQL
+ * cell nobody had run. The run "was healthy and delivered nothing".
+ *
+ * Worse than the 11 is what the 11 hides: on the same run, **56 further files
+ * report as PASSED while carrying a skipped live cell inside them**. A file
+ * with a green tick next to it is the least visible skip in the output.
+ *
+ * ## What this is, and deliberately is not
+ *
+ * DECLARATION ONLY. It reads the run that already happened and writes a block
+ * to stdout. It never fails, never changes an exit code, never skips or
+ * un-skips anything, and adds no gate — a suite that refuses to go green on a
+ * skipped live cell is a new required gate and is not this file's to introduce
+ * (#18200 carries that question).
+ *
+ * Every hook body is wrapped, so a defect in this reporter cannot redden a run
+ * either: a signal that can break the thing it reports on would be removed, and
+ * then there would be no signal.
+ *
+ * ## Why the two states differ, and where the numbers come from
+ *
+ *   - WHICH dialects ran is read from {@link DIALECT_CELLS} — this package's own
+ *     single source of truth for the driver axis — never inferred from test
+ *     names. A cell's `env` is the variable that provisions it, so the block can
+ *     name the exact knob rather than warning in the abstract.
+ *   - HOW MANY tests were skipped is read from the run's own results. Those are
+ *     reported as the raw skip counts they are; the block does not claim each
+ *     one is a backend skip. Measured on this package with no servers: 167 of
+ *     the 168 are, and the one that is not is skipped on the SQLite cell
+ *     (`schema-drift.base-type-mismatch.test.ts`, `skipIf(!corrupts)`). The
+ *     causal sentence is attached to the per-dialect lines, which are exact.
+ *
+ * Attribution is deliberately NOT attempted per test. The reporter runs in the
+ * main process, where a skipped task carries no `meta` and no skip note (only
+ * `ctx.skip(note)` sets one, and a skipped test never reaches its body), so the
+ * only per-test channel left is the test's own NAME. Nine files in this package
+ * guard their live cells with a hand-rolled `skipIf` whose names the testkit
+ * never wrote, so a name matcher would silently report a smaller number than the
+ * truth — which is this card's own disease.
+ *
+ * With a backend provisioned the block says so and the counts fall, so the
+ * signal is not the same text in both states.
+ *
+ * ## The recipe is printed because a recipe nobody looks for is not a signal
+ *
+ * #18200: 「a recipe nobody knows to look for is not a signal」. The steps below
+ * were run in the CI-shaped configuration before being written here —
+ * PostgreSQL 16.13 from `/usr/lib/postgresql/16/bin`, `initdb` + `pg_ctl` on a
+ * non-default port, server `timezone=Asia/Shanghai` against process
+ * `TZ=America/New_York`. Two details that are not cosmetic:
+ *
+ *   - On Debian/Ubuntu `initdb` and `pg_ctl` are not on `PATH`, and `initdb`
+ *     refuses to run as root — run both from `/usr/lib/postgresql/<major>/bin`
+ *     as a non-root user (`su postgres -c '…'` in a container).
+ *   - The three clocks must disagree: server zone, process `TZ`, and UTC. The
+ *     matrix asserts that skew on purpose (`assertThreeWayZoneSkew`), because
+ *     identical answers from a UTC server are answers no timezone could have
+ *     perturbed — a pass that means nothing.
+ *
+ * MySQL is deliberately not in the printed recipe: provisioning it means
+ * installing a server package into a shared container, which is not something
+ * to put in front of a reader as a casual next step. The env var is named, so a
+ * reader who has one knows what to set.
+ *
+ * ## Wiring notes
+ *
+ *   - Registered in `vitest.config.ts` AFTER `'default'`, so this block lands
+ *     under the summary counts it qualifies — the place a round actually looks.
+ *   - An explicit `--reporter=` on the command line replaces the configured
+ *     list, this reporter included. That is vitest's own semantics; a round that
+ *     asks for `--reporter=json` is not reading a terminal summary anyway.
+ */
+
+import type { Reporter, TestModule } from 'vitest/node';
+import { DIALECT_CELLS, EXPECT_LIVE_DIALECTS } from './live-dialect-matrix.testkit.js';
+
+/** Everything this reporter needs to know about the run, in one pass. */
+interface SkipCensus {
+  /** Modules vitest collected in this run. */
+  files: number;
+  /** Tests collected across them. */
+  tests: number;
+  /** Tests whose result state is `skipped`. */
+  skippedTests: number;
+  /** Modules where EVERY collected test was skipped — the "N skipped" files. */
+  filesFullySkipped: number;
+  /** Modules that ran, with at least one skipped test inside — the hidden half. */
+  filesPartlySkipped: number;
+}
+
+function census(testModules: ReadonlyArray<TestModule>): SkipCensus {
+  const c: SkipCensus = {
+    files: testModules.length,
+    tests: 0,
+    skippedTests: 0,
+    filesFullySkipped: 0,
+    filesPartlySkipped: 0,
+  };
+  for (const mod of testModules) {
+    let total = 0;
+    let skipped = 0;
+    for (const test of mod.children.allTests()) {
+      total += 1;
+      if (test.result().state === 'skipped') skipped += 1;
+    }
+    c.tests += total;
+    c.skippedTests += skipped;
+    if (skipped === 0) continue;
+    if (skipped === total) c.filesFullySkipped += 1;
+    else c.filesPartlySkipped += 1;
+  }
+  return c;
+}
+
+const PG_RECIPE = [
+  '    PGBIN=/usr/lib/postgresql/16/bin   # Debian/Ubuntu: off PATH, and initdb refuses root',
+  '    $PGBIN/initdb -D /tmp/os-pg -U postgres --auth=trust',
+  '    $PGBIN/pg_ctl -D /tmp/os-pg -l /tmp/os-pg/server.log -w start \\',
+  "        -o '-p 54988 -c timezone=Asia/Shanghai'",
+  '    OS_TEST_POSTGRES_URL=postgres://postgres@127.0.0.1:54988/postgres TZ=America/New_York \\',
+  '        pnpm --filter @objectstack/driver-sql test',
+  '    $PGBIN/pg_ctl -D /tmp/os-pg -m fast stop && rm -rf /tmp/os-pg    # teardown',
+];
+
+/** Terminal-friendly wrapping, so a long sentence does not arrive as one ragged line. */
+function wrap(text: string, indent = '  ', width = 92): string[] {
+  const out: string[] = [];
+  let line = indent;
+  for (const word of text.split(/\s+/).filter(Boolean)) {
+    if (line.length > indent.length && line.length + 1 + word.length > width) {
+      out.push(line);
+      line = indent;
+    }
+    line += line.length > indent.length ? ` ${word}` : word;
+  }
+  if (line.length > indent.length) out.push(line);
+  return out;
+}
+
+function report(testModules: ReadonlyArray<TestModule>): string {
+  const live = DIALECT_CELLS.filter((cell) => cell.live);
+  const missing = live.filter((cell) => !cell.available);
+  const ran = DIALECT_CELLS.filter((cell) => cell.available);
+  const c = census(testModules);
+  const lines: string[] = [''];
+
+  const filesWithSkips = c.filesFullySkipped + c.filesPartlySkipped;
+
+  if (missing.length === 0) {
+    lines.push(
+      ...wrap(
+        `driver-sql live-dialect coverage: all ${DIALECT_CELLS.length} dialects were ` +
+          `exercised (${ran.map((cell) => cell.label).join(', ')}).` +
+          (c.skippedTests > 0
+            ? ` ${c.skippedTests} test(s) were still skipped, in ${filesWithSkips} of ` +
+              `${c.files} files, for reasons other than a missing backend.`
+            : ''),
+      ),
+      '',
+    );
+    return lines.join('\n');
+  }
+
+  if (EXPECT_LIVE_DIALECTS) {
+    // The runner declared it provisioned the servers, so the testkit has already
+    // turned each missing cell into a named FAILURE. Repeating the warning here
+    // would compete with a red the run is already carrying; name the cause once.
+    lines.push(
+      ...wrap(
+        `driver-sql live-dialect coverage: OS_EXPECT_LIVE_DIALECT_MATRIX=1, but ` +
+          `${missing.map((cell) => `${cell.label} (${cell.env})`).join(' and ')} ` +
+          `${missing.length === 1 ? 'was' : 'were'} not provisioned — this run reported that ` +
+          `as a named failure, not as a skip.`,
+      ),
+      '',
+    );
+    return lines.join('\n');
+  }
+
+  const width = Math.max(...DIALECT_CELLS.map((cell) => cell.label.length));
+  lines.push(
+    `  !! driver-sql live-dialect coverage: this run exercised ${ran.length} of ` +
+      `${DIALECT_CELLS.length} dialects.`,
+    '',
+  );
+  for (const cell of DIALECT_CELLS) {
+    const label = cell.label.padEnd(width);
+    lines.push(
+      cell.available ? `     ${label}  RAN` : `     ${label}  NOT RUN -- set ${cell.env} to run it`,
+    );
+  }
+  lines.push(
+    '',
+    ...wrap(
+      `The counts above this block are NOT coverage of the dialect(s) marked NOT RUN. This ` +
+        `run skipped ${c.skippedTests} test(s) across ${filesWithSkips} of its ${c.files} ` +
+        `files: ${c.filesFullySkipped} vitest reported as skipped, and ${c.filesPartlySkipped} ` +
+        `more it reported as PASSED with skipped tests inside them — the least visible skip ` +
+        `in the output. Every ${missing.map((cell) => cell.label).join(' and ')} cell in this ` +
+        `package is in that population, and a green above says nothing about any of them.`,
+    ),
+    '',
+  );
+  const ciNote =
+    'CI runs those cells in `Temporal Conformance (live PG + MySQL)`, which sets ' +
+    'OS_EXPECT_LIVE_DIALECT_MATRIX=1 — there an unprovisioned cell is a named failure ' +
+    'rather than a skip.';
+  if (missing.some((cell) => cell.id === 'pg')) {
+    lines.push(
+      ...wrap(`${ciNote} To run the postgres half here (measured: ~1 min to provision):`),
+      '',
+      ...PG_RECIPE,
+      '',
+      ...wrap(
+        'The server zone, TZ and UTC must all differ: the matrix asserts that skew, because ' +
+          'identical answers from a UTC server are answers no timezone could have perturbed.',
+      ),
+    );
+  } else {
+    // Only MySQL is left, and provisioning one means installing a server package
+    // into whatever container this is running in. That is not a step to put in
+    // front of a reader as a casual next line; naming the variable is enough.
+    lines.push(
+      ...wrap(
+        `${ciNote} Provisioning MySQL locally means installing a server into this ` +
+          `environment, so no recipe is offered here — set OS_TEST_MYSQL_URL if you already ` +
+          `have one, otherwise CI is where that cell runs.`,
+      ),
+    );
+  }
+  lines.push('');
+  return lines.join('\n');
+}
+
+/**
+ * Prints the block described above after the summary. Reads the run; writes
+ * stdout; returns nothing.
+ */
+export default class LiveDialectCoverageReporter implements Reporter {
+  onTestRunEnd(testModules: ReadonlyArray<TestModule>): void {
+    try {
+      process.stdout.write(`${report(testModules)}\n`);
+    } catch {
+      // Declaration-only means declaration-only: a reporter that can fail a run
+      // is a reporter someone deletes, and then the blind spot is back.
+    }
+  }
+}

packages/drivers/driver-sql/vitest.config.ts

@@ -22,6 +22,21 @@ export default defineConfig({
     // `beforeEach`, and the testkit module is cached per WORKER, not per file.
     // This runs once, before any worker, and is a no-op without a live URL.
     globalSetup: ['./src/live-dialect-matrix.globalsetup.ts'],
+    // #18200: name the dialects this run did NOT exercise, underneath the
+    // summary counts a round would otherwise read as coverage. ORDER IS THE
+    // POINT — the default reporter prints those counts from its own
+    // `onTestRunEnd` and reporters are invoked in list order, so this one comes
+    // second to land below them. What it prints, and how the two states differ:
+    // `src/live-dialect-coverage.reporter.ts`.
+    //
+    // `github-actions` is re-added by hand because vitest appends it only
+    // `if (!resolved.reporters.length)` — naming ANY reporter here would drop
+    // the annotations CI gets today. Same condition vitest itself uses, so CI's
+    // output is unchanged (vitest 4.1.11, `dist/chunks/coverage.*.js`).
+    reporters:
+      process.env.GITHUB_ACTIONS === 'true'
+        ? ['default', 'github-actions', './src/live-dialect-coverage.reporter.ts']
+        : ['default', './src/live-dialect-coverage.reporter.ts'],
   },
   resolve: {
     alias: [