fix(objectql): a read answers a selected declared column the driver did not return as null (#29408)

.changeset/29408-selected-declared-column-null.md

@@ -0,0 +1,13 @@
+---
+'@objectstack/objectql': patch
+---
+
+fix(objectql): a read answers a selected declared column the driver did not return as `null`, so one record reads the same on every driver (#29408)
+
+Clause-②: no — nothing a request may send is refused or newly admitted; measured on base and head, every read returns the same rows, and every value a driver returned is answered as before. A record body gains a `null` key for a stored declared column the read selected and the driver returned no key for, and on driver-mongodb and driver-memory a `formula` over such a column now evaluates against that `null`, as it already did on driver-sql.
+Metadata-protocol: none
+
+- **Before.** driver-sql answered a declared field that was never written as `null` (its table has the column), while driver-mongodb and driver-memory answered no key at all (they store only the columns a write touched). The same record read `{ title: 'x', note: null }` on SQL and `{ title: 'x' }` on mongo or memory.
+- **Now.** The engine answers every stored declared column the read selected as `null` when the driver returned no key for it, in `find` and `findOne`, so every door on them (`GET /data/:object/:id`, the list route and `POST /data/:object/query`, export, search, `expand` sub-reads, a hook's or a flow's in-process read) answers alike on every driver. Selected means the whole declared record when the read names no `fields`, else the declared columns its `fields` name.
+- **Still absent.** A column the read's `fields` do not name; a column field-level security refuses the caller (the security mask still deletes it); a `formula` field itself (virtual: never `null`-filled, it is computed after the fill); a name outside the declared set; an injected platform column a federated (`external`) object has no storage for; and, on a federated object, a `created_at` / `updated_at` it neither declares nor has injected (`systemFields: false` or `systemFields.audit: false`), which no table stores.
+- **Who sees it.** A client of an object served by driver-mongodb or driver-memory whose records have unwritten declared fields. Such a field now arrives as a key holding `null` instead of a missing key: a client-side `has(record.x)` is now `true` for it, as it already was on SQL, so test emptiness with `record.x != null`. A `formula` over such a field changes value on those two drivers: it used to fault on the missing key and answer `null`, and now it evaluates against `null` and answers what driver-sql answered (`record.qty != null ? record.qty * 2 : -1` was `null`, now `-1`). A write's stored row is unchanged; the record a write answers (insert or update echo) is unchanged.

packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts

@@ -0,0 +1,123 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * [#29408] driver-mongodb through the engine: a declared column the read
+ * selected and the write never touched is answered `null`, as driver-sql's
+ * column answers — not a missing key.
+ *
+ * This driver stores no key for a field given no value (`withoutUndefinedOwnKeys`
+ * at the insert doors, `mongodb-driver.ts`), and a MongoDB projection that
+ * names a key the document lacks returns no such key. So the row the driver
+ * hands the engine is sparse; the engine answers it on the selected stored
+ * columns (`selectedStoredColumns`, `@objectstack/objectql`'s
+ * `declared-read-columns.ts`). Measured on `origin/main` before the fix with
+ * this same harness: every read below answered no `note` key.
+ *
+ * Runs without a server, on the fake `Db` pattern `mongodb-findone-options.test.ts`
+ * established (`getCollection` is `this.db.collection(name)`). The fake answers
+ * a projection as a real mongod does — the named keys the document HAS, never
+ * an own key holding `undefined` for one it lacks — which is the one property
+ * of the server this pin depends on. The real-server suites are gated on
+ * `mongodb-memory-server` (#5517).
+ */
+
+import { describe, it, expect, beforeEach } from 'vitest';
+import { ObjectQL } from '@objectstack/objectql';
+import { MongoDBDriver } from './mongodb-driver.js';
+
+type Doc = Record<string, unknown>;
+
+/** A fake `Db`: one collection, stored documents kept as inserted. */
+function fakeDb() {
+  const docs: Doc[] = [];
+  const project = (doc: Doc, projection: Record<string, number> | undefined): Doc => {
+    const keep = Object.keys(projection ?? {}).filter((k) => projection![k] === 1);
+    const out: Doc = {};
+    for (const [k, v] of Object.entries(doc)) {
+      if (k === '_id' && projection?._id === 0) continue;
+      if (keep.length > 0 && !keep.includes(k)) continue;
+      out[k] = v;
+    }
+    return out;
+  };
+  // Equality on plain keys only — the reads below filter by `id` alone. A
+  // combinator or any other operator is REFUSED, never read as a field name.
+  const matches = (doc: Doc, filter: Doc | undefined): boolean =>
+    Object.entries(filter ?? {}).every(([k, v]) => {
+      if (k.startsWith('$')) throw new Error(`fake Db: filter operator ${k} is not implemented`);
+      if (v !== null && typeof v === 'object') {
+        const ops = Object.keys(v as Doc);
+        if (ops.length !== 1 || ops[0] !== '$eq') throw new Error(`fake Db: operator ${ops.join(',')} is not implemented`);
+        return doc[k] === (v as Doc).$eq;
+      }
+      return doc[k] === v;
+    });
+  const collection = {
+    insertOne: async (doc: Doc) => { docs.push({ _id: `oid_${docs.length}`, ...doc }); return { insertedId: `oid_${docs.length}` }; },
+    findOne: async (filter: Doc, options: { projection?: Record<string, number> }) => {
+      const d = docs.find((x) => matches(x, filter));
+      return d ? project(d, options?.projection) : null;
+    },
+    find: (filter: Doc, options: { projection?: Record<string, number> }) => ({
+      toArray: async () => docs.filter((x) => matches(x, filter)).map((d) => project(d, options?.projection)),
+    }),
+  };
+  return { db: { collection: () => collection }, docs };
+}
+
+const OBJECT = 'nk_mongo_item';
+
+const own = (row: unknown, key: string): boolean =>
+  row != null && typeof row === 'object' && Object.prototype.hasOwnProperty.call(row, key);
+
+describe('[#29408] driver-mongodb: a selected, unwritten declared column reads null through the engine', () => {
+  let engine: ObjectQL;
+  let docs: Doc[];
+  let id: string;
+
+  beforeEach(async () => {
+    const driver = new MongoDBDriver({ url: 'mongodb://127.0.0.1:1/unused' });
+    const fake = fakeDb();
+    docs = fake.docs;
+    (driver as any).db = fake.db;
+    (driver as any).connect = async () => {};
+    (driver as any).disconnect = async () => {};
+    engine = new ObjectQL();
+    engine.registerDriver(driver as never, true);
+    await engine.init();
+    engine.registry.registerObject({
+      name: OBJECT,
+      label: 'Item',
+      fields: { name: { type: 'text' }, note: { type: 'text' } },
+    } as never);
+    const created = await engine.insert(OBJECT, { name: 'Ada' }) as Doc;
+    id = String(created.id);
+  });
+
+  it('the fixture is real: the stored document carries no key for the unwritten column', () => {
+    expect(docs).toHaveLength(1);
+    expect(own(docs[0], 'note')).toBe(false);
+  });
+
+  it('whole-record read and list read answer it null', async () => {
+    const one = await engine.findOne(OBJECT, { where: { id } });
+    expect(own(one, 'note')).toBe(true);
+    expect(one!.note).toBeNull();
+    const rows = await engine.find(OBJECT, {});
+    expect(own(rows[0], 'note')).toBe(true);
+    expect(rows[0]!.note).toBeNull();
+  });
+
+  it('a projection naming it answers it null', async () => {
+    const one = await engine.findOne(OBJECT, { where: { id }, fields: ['name', 'note'] });
+    expect(own(one, 'note')).toBe(true);
+    expect(one!.note).toBeNull();
+  });
+
+  it('NOT SELECTED stays absent: a projection that does not name it never answers it', async () => {
+    const one = await engine.findOne(OBJECT, { where: { id }, fields: ['name'] });
+    expect(own(one, 'note')).toBe(false);
+    const rows = await engine.find(OBJECT, { fields: ['name'] });
+    expect(own(rows[0], 'note')).toBe(false);
+  });
+});

packages/objectql/src/declared-read-columns.ts

@@ -1,5 +1,8 @@
 // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
 
+import { unprovisionedInjectedColumns } from '@objectstack/spec/data';
+import { isFederatedObject } from './federated-object.js';
+
 /**
  * [#21571] The columns a read may serve: the object's DECLARED fields plus the
  * platform-provisioned columns — one set, answered from the registry's field
@@ -129,3 +132,115 @@ export function rowsWithDeclaredColumnsOnly<T>(rows: T[], declared: ReadonlySet<
   });
   return changed ? shaped : rows;
 }
+
+/**
+ * [#29408] The STORED declared columns a read selected — the set a row is
+ * answered on, key for key, whatever the driver happened to return for it.
+ *
+ * ## The defect this closes
+ *
+ * A driver that stores only the columns a write touched (driver-mongodb drops
+ * a field given no value at its insert doors; driver-memory keeps stored
+ * `null` and a never-written key apart) answered such a field with NO key,
+ * while driver-sql, whose table has the column, answered `null`. One record,
+ * two shapes, by driver: a client predicate such as an action's `visible`
+ * reading `record.x` faulted on the missing key on one and evaluated on the
+ * other (follow-up D.1 of the #29094 ruling, record 6579342813).
+ *
+ * The read's persisted state is IN HAND here — the driver just answered for
+ * the row — so `null` is the honest value of a selected declared column the
+ * driver did not return, exactly the rule `materializeDeclaredFields`
+ * (`declared-fields.ts`) applies to every CEL binding.
+ *
+ * ## What is selected
+ *
+ * - an explicit projection selects the declared columns it names — the
+ *   projection the DRIVER was handed, so a `beforeFind` hook's rewrite and the
+ *   engine's own widening (formula inputs, a file field's owner key, each cut
+ *   back off the answer before the verb returns) are what count;
+ * - no projection (a whole-record GET, a list read) selects every declared
+ *   column.
+ *
+ * ## What stays absent, and why
+ *
+ * - a column the read did not select — a projection is the caller's own
+ *   choice of keys, and filling the rest would answer what was not asked;
+ * - a `formula` field — virtual, no column on any driver (`fieldHasColumn` in
+ *   driver-sql); `applyFormulaPlan` answers it after this, unchanged;
+ * - a name outside the declared set — the intersection below, so nothing this
+ *   adds can widen what {@link withDeclaredColumnsOnly} serves;
+ * - an injected column a federated (ADR-0015 `external`) object does not
+ *   provision (`unprovisionedInjectedColumns`) — there is no storage behind
+ *   it, so there is no persisted state to answer for;
+ * - on a federated object, a {@link PLATFORM_PROVISIONED_COLUMNS} timestamp
+ *   its field map does not declare (`systemFields: false` or
+ *   `systemFields.audit: false`, so nothing injected it either). The three are
+ *   "provisioned" because driver-sql creates them on every table IT builds,
+ *   and the platform builds no table for a federated object
+ *   (`isFederatedObject`): the remote holds such a timestamp only where the
+ *   author mapped one, which is a declared field. `id` stays — the remote's
+ *   own primary key backs it through the binding;
+ * - a column field-level security refuses — this runs inside the verb's
+ *   middleware chain, on the rows as the driver returned them, and the
+ *   security middleware's field mask deletes a refused key from the result
+ *   AFTER the verb's body; a `null` added here is removed with it.
+ *
+ * Same "no opinion" cases as {@link declaredColumnSet}: no checkable field map
+ * ⇒ `undefined`, and the rows are left exactly as the driver returned them.
+ */
+export function selectedStoredColumns(
+  schema: { fields?: unknown } | null | undefined,
+  projection: readonly unknown[] | null | undefined,
+): ReadonlySet<string> | undefined {
+  const declared = declaredColumnSet(schema);
+  if (!declared) return undefined;
+  const fields = schema!.fields as Record<string, { type?: unknown } | null | undefined>;
+  const unprovisioned = new Set(unprovisionedInjectedColumns(schema));
+  const federated = isFederatedObject(schema);
+  const stored = new Set<string>();
+  for (const name of declared) {
+    if (fields[name]?.type === 'formula' || unprovisioned.has(name)) continue;
+    // A platform-provisioned column the object does not declare has storage
+    // only on a table the platform built.
+    if (federated && name !== 'id' && !Object.prototype.hasOwnProperty.call(fields, name)) continue;
+    stored.add(name);
+  }
+  if (!Array.isArray(projection) || projection.length === 0) return stored;
+  const selected = new Set<string>();
+  for (const entry of projection) {
+    if (typeof entry === 'string' && stored.has(entry)) selected.add(entry);
+  }
+  return selected;
+}
+
+/**
+ * The row with every {@link selectedStoredColumns} column it does not carry
+ * answered as `null` — a NEW object when there was anything to answer, the
+ * same reference otherwise. `undefined` counts as not carried (an own key
+ * holding `undefined` is what CEL and JSON both read as absent). Never
+ * mutates the row it is given, for the reason {@link withDeclaredColumnsOnly}
+ * gives.
+ */
+export function withSelectedColumnsAnswered<T>(row: T, selected: ReadonlySet<string> | undefined): T {
+  if (!selected || selected.size === 0 || !row || typeof row !== 'object' || Array.isArray(row)) return row;
+  const source = row as unknown as Record<string, unknown>;
+  let shaped: Record<string, unknown> | undefined;
+  for (const name of selected) {
+    if (source[name] !== undefined) continue;
+    shaped ??= { ...source };
+    shaped[name] = null;
+  }
+  return (shaped ?? row) as unknown as T;
+}
+
+/** {@link withSelectedColumnsAnswered} over a driver's result page. */
+export function rowsWithSelectedColumnsAnswered<T>(rows: T[], selected: ReadonlySet<string> | undefined): T[] {
+  if (!selected || selected.size === 0) return rows;
+  let changed = false;
+  const shaped = rows.map((row) => {
+    const next = withSelectedColumnsAnswered(row, selected);
+    if (next !== row) changed = true;
+    return next;
+  });
+  return changed ? shaped : rows;
+}

packages/objectql/src/engine-formula-projection-trim.test.ts

@@ -187,7 +187,9 @@ describe.each<Verb>(['find', 'findOne'])('#22300 — %s: a formula in the projec
   it('CONTROL: a read with no projection is unchanged — every declared column, the formula computed', async () => {
     const { engine, projections } = await boot();
     const row = await readOne(engine, verb);
-    expect(row).toStrictEqual({ ...STORED, total_price: 21 });
+    // [#29408] `queue` (registry-injected) is declared but absent from the
+    // stored row: a whole-record read answers it null.
+    expect(row).toStrictEqual({ ...STORED, queue: null, total_price: 21 });
     expect(projections.at(-1)).toBeUndefined();
   });
 

packages/objectql/src/engine.test.ts

@@ -1723,8 +1723,10 @@ describe('ObjectQL Engine', () => {
             const result = await engine.find('task', { expand: { assignee: { object: 'assignee' } } });
 
             expect(result).toHaveLength(2);
-            expect(result[0].assignee).toEqual({ id: 'u1', name: 'Alice' });
-            expect(result[1].assignee).toEqual({ id: 'u2', name: 'Bob' });
+            // [#29408] The related record is answered on every declared column the
+            // read selected; the double's rows carry no timestamps, so they read null.
+            expect(result[0].assignee).toEqual({ id: 'u1', name: 'Alice', created_at: null, updated_at: null });
+            expect(result[1].assignee).toEqual({ id: 'u2', name: 'Bob', created_at: null, updated_at: null });
 
             // Verify the expand query used $in. The expand sub-read now routes
             // through the secured `find` path ([#2850]), so the referenced
@@ -1768,7 +1770,8 @@ describe('ObjectQL Engine', () => {
 
             const result = await engine.find('ticket', { expand: { owner: { object: 'owner' } } });
 
-            expect(result[0].owner).toEqual({ id: 'u1', name: 'Alice' });
+            // [#29408] Unwritten declared columns of the related record read null.
+            expect(result[0].owner).toEqual({ id: 'u1', name: 'Alice', created_at: null, updated_at: null });
             expect(mockDriver.find).toHaveBeenLastCalledWith(
                 'sys_user',
                 expect.objectContaining({ where: { id: { $in: ['u1'] } } }),
@@ -1825,7 +1828,8 @@ describe('ObjectQL Engine', () => {
             );
 
             // Records the driver filtered out keep their raw FK id (unresolved).
-            expect(result[0].assignee).toEqual({ id: 'u1', name: 'Alice', active: true });
+            // [#29408] Unwritten declared columns of the related record read null.
+            expect(result[0].assignee).toEqual({ id: 'u1', name: 'Alice', active: true, created_at: null, updated_at: null });
             expect(result[1].assignee).toBe('u2');
         });
 
@@ -2071,7 +2075,8 @@ describe('ObjectQL Engine', () => {
                 ]);
 
             const result = await engine.find('order_item', { expand: { order: { object: 'order' } } });
-            expect(result[0].order).toEqual({ id: 'o1', total: 100 });
+            // [#29408] Unwritten declared columns of the related record read null.
+            expect(result[0].order).toEqual({ id: 'o1', total: 100, created_at: null, updated_at: null });
         });
 
         it('should skip expand for fields without reference definition', async () => {
@@ -2558,7 +2563,8 @@ describe('ObjectQL Engine', () => {
             const result = await engine.find('task', { expand: { project: { object: 'project' } } });
 
             // Project expanded, but org inside project remains as raw ID
-            expect(result[0].project).toEqual({ id: 'p1', org: 'o1' });
+            // [#29408] Unwritten declared columns of the related record read null.
+            expect(result[0].project).toEqual({ id: 'p1', org: 'o1', created_at: null, updated_at: null });
             expect(mockDriver.find).toHaveBeenCalledTimes(2); // Only primary + 1 expand query
         });
 

packages/objectql/src/engine.ts

@@ -386,7 +386,10 @@ import {
   PLATFORM_PROVISIONED_COLUMNS,
   declaredColumnSet,
   rowsWithDeclaredColumnsOnly,
+  rowsWithSelectedColumnsAnswered,
+  selectedStoredColumns,
   withDeclaredColumnsOnly,
+  withSelectedColumnsAnswered,
 } from './declared-read-columns.js';
 // [#21777] "Is this schema the remote's?" One predicate, shared with the boot sync.
 // [#21910, #21918] And its injected-column refinement, which both cascade walks ask.
@@ -14619,6 +14622,13 @@ export class ObjectQL implements IObjectQLEngine {
           // declared record. See `declared-read-columns.ts`.
           if (Array.isArray(result)) {
             result = rowsWithDeclaredColumnsOnly(result, declaredColumnSet(_findSchema));
+            // [#29408] …and answered on every stored declared column the read
+            // SELECTED: one the driver returned no key for (a driver that
+            // stores only written columns) is `null`, as driver-sql's column
+            // answers — the row's persisted state is in hand. Unselected,
+            // formula and FLS-refused columns stay absent; see
+            // `selectedStoredColumns`.
+            result = rowsWithSelectedColumnsAnswered(result, selectedStoredColumns(_findSchema, driverAst?.fields));
           }
 
           // Post-process: evaluate formula virtual fields against the raw rows.
@@ -14919,6 +14929,8 @@ export class ObjectQL implements IObjectQLEngine {
       // [#21571] Same default projection as `find`, same position: the
       // declared field set, applied to the row as the driver returned it.
       result = withDeclaredColumnsOnly(result, declaredColumnSet(_findOneSchema));
+      // [#29408] Same answer on the selected stored columns as `find`.
+      result = withSelectedColumnsAnswered(result, selectedStoredColumns(_findOneSchema, driverAst?.fields));
 
       // Post-process: evaluate formula virtual fields against the raw row
       // ([#20082] with the caller's permission map when a formula calls `can`).