From ac1a171d8ba03e7760ff07f248bc9d9d3c6e26c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 15:56:18 +0000 Subject: [PATCH 1/9] fix(objectql): a read answers a selected declared column the driver did not return as null Driver-mongodb and driver-memory return no key for a declared field that was never written, while driver-sql answers its column as null. The engine now answers every stored declared column the read selected (the whole record, or the columns an explicit projection names) as null when the driver returned no key for it. Unselected columns, formula fields, names outside the declared set and a federated object's unprovisioned injected columns stay absent; a column field-level security refuses is still deleted by the security middleware's mask, which runs after the verb's body. Claude-Session: https://claude.ai/code/session_01ADHkmRKR6AV3yXf6HimS9d Co-authored-by: Claude --- .../objectql/src/declared-read-columns.ts | 101 ++++++++ packages/objectql/src/engine.ts | 12 + .../federated-injected-column-readers.test.ts | 4 + ...selected-declared-columns-answered.test.ts | 242 ++++++++++++++++++ 4 files changed, 359 insertions(+) create mode 100644 packages/objectql/src/selected-declared-columns-answered.test.ts diff --git a/packages/objectql/src/declared-read-columns.ts b/packages/objectql/src/declared-read-columns.ts index 6e1d7f92c49..7a8e501cbbb 100644 --- a/packages/objectql/src/declared-read-columns.ts +++ b/packages/objectql/src/declared-read-columns.ts @@ -1,5 +1,7 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +import { unprovisionedInjectedColumns } from '@objectstack/spec/data'; + /** * [#21571] The columns a read may serve: the object's DECLARED fields plus the * platform-provisioned columns — one set, answered from the registry's field @@ -129,3 +131,102 @@ export function rowsWithDeclaredColumnsOnly(rows: T[], declared: ReadonlySet< }); return changed ? shaped : rows; } + +/** + * [#29408] The STORED declared columns a read selected — the set a row is + * answered on, key for key, whatever the driver happened to return for it. + * + * ## The defect this closes + * + * A driver that stores only the columns a write touched (driver-mongodb drops + * a field given no value at its insert doors; driver-memory keeps stored + * `null` and a never-written key apart) answered such a field with NO key, + * while driver-sql, whose table has the column, answered `null`. One record, + * two shapes, by driver: a client predicate such as an action's `visible` + * reading `record.x` faulted on the missing key on one and evaluated on the + * other (follow-up D.1 of the #29094 ruling, record 6579342813). + * + * The read's persisted state is IN HAND here — the driver just answered for + * the row — so `null` is the honest value of a selected declared column the + * driver did not return, exactly the rule `materializeDeclaredFields` + * (`declared-fields.ts`) applies to every CEL binding. + * + * ## What is selected + * + * - an explicit projection selects the declared columns it names — the + * projection the DRIVER was handed, so a `beforeFind` hook's rewrite and the + * engine's own widening (formula inputs, a file field's owner key, each cut + * back off the answer before the verb returns) are what count; + * - no projection (a whole-record GET, a list read) selects every declared + * column. + * + * ## What stays absent, and why + * + * - a column the read did not select — a projection is the caller's own + * choice of keys, and filling the rest would answer what was not asked; + * - a `formula` field — virtual, no column on any driver (`fieldHasColumn` in + * driver-sql); `applyFormulaPlan` answers it after this, unchanged; + * - a name outside the declared set — the intersection below, so nothing this + * adds can widen what {@link withDeclaredColumnsOnly} serves; + * - an injected column a federated (ADR-0015 `external`) object does not + * provision (`unprovisionedInjectedColumns`) — there is no storage behind + * it, so there is no persisted state to answer for; + * - a column field-level security refuses — this runs inside the verb's + * middleware chain, on the rows as the driver returned them, and the + * security middleware's field mask deletes a refused key from the result + * AFTER the verb's body; a `null` added here is removed with it. + * + * Same "no opinion" cases as {@link declaredColumnSet}: no checkable field map + * ⇒ `undefined`, and the rows are left exactly as the driver returned them. + */ +export function selectedStoredColumns( + schema: { fields?: unknown } | null | undefined, + projection: readonly unknown[] | null | undefined, +): ReadonlySet | undefined { + const declared = declaredColumnSet(schema); + if (!declared) return undefined; + const fields = schema!.fields as Record; + const unprovisioned = new Set(unprovisionedInjectedColumns(schema)); + const stored = new Set(); + for (const name of declared) { + if (fields[name]?.type !== 'formula' && !unprovisioned.has(name)) stored.add(name); + } + if (!Array.isArray(projection) || projection.length === 0) return stored; + const selected = new Set(); + for (const entry of projection) { + if (typeof entry === 'string' && stored.has(entry)) selected.add(entry); + } + return selected; +} + +/** + * The row with every {@link selectedStoredColumns} column it does not carry + * answered as `null` — a NEW object when there was anything to answer, the + * same reference otherwise. `undefined` counts as not carried (an own key + * holding `undefined` is what CEL and JSON both read as absent). Never + * mutates the row it is given, for the reason {@link withDeclaredColumnsOnly} + * gives. + */ +export function withSelectedColumnsAnswered(row: T, selected: ReadonlySet | undefined): T { + if (!selected || selected.size === 0 || !row || typeof row !== 'object' || Array.isArray(row)) return row; + const source = row as unknown as Record; + let shaped: Record | undefined; + for (const name of selected) { + if (source[name] !== undefined) continue; + shaped ??= { ...source }; + shaped[name] = null; + } + return (shaped ?? row) as unknown as T; +} + +/** {@link withSelectedColumnsAnswered} over a driver's result page. */ +export function rowsWithSelectedColumnsAnswered(rows: T[], selected: ReadonlySet | undefined): T[] { + if (!selected || selected.size === 0) return rows; + let changed = false; + const shaped = rows.map((row) => { + const next = withSelectedColumnsAnswered(row, selected); + if (next !== row) changed = true; + return next; + }); + return changed ? shaped : rows; +} diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 93b1fa9ba76..e8fbc0af9ef 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -386,7 +386,10 @@ import { PLATFORM_PROVISIONED_COLUMNS, declaredColumnSet, rowsWithDeclaredColumnsOnly, + rowsWithSelectedColumnsAnswered, + selectedStoredColumns, withDeclaredColumnsOnly, + withSelectedColumnsAnswered, } from './declared-read-columns.js'; // [#21777] "Is this schema the remote's?" One predicate, shared with the boot sync. // [#21910, #21918] And its injected-column refinement, which both cascade walks ask. @@ -14619,6 +14622,13 @@ export class ObjectQL implements IObjectQLEngine { // declared record. See `declared-read-columns.ts`. if (Array.isArray(result)) { result = rowsWithDeclaredColumnsOnly(result, declaredColumnSet(_findSchema)); + // [#29408] …and answered on every stored declared column the read + // SELECTED: one the driver returned no key for (a driver that + // stores only written columns) is `null`, as driver-sql's column + // answers — the row's persisted state is in hand. Unselected, + // formula and FLS-refused columns stay absent; see + // `selectedStoredColumns`. + result = rowsWithSelectedColumnsAnswered(result, selectedStoredColumns(_findSchema, driverAst?.fields)); } // Post-process: evaluate formula virtual fields against the raw rows. @@ -14919,6 +14929,8 @@ export class ObjectQL implements IObjectQLEngine { // [#21571] Same default projection as `find`, same position: the // declared field set, applied to the row as the driver returned it. result = withDeclaredColumnsOnly(result, declaredColumnSet(_findOneSchema)); + // [#29408] Same answer on the selected stored columns as `find`. + result = withSelectedColumnsAnswered(result, selectedStoredColumns(_findOneSchema, driverAst?.fields)); // Post-process: evaluate formula virtual fields against the raw row // ([#20082] with the caller's permission map when a formula calls `can`). diff --git a/packages/objectql/src/federated-injected-column-readers.test.ts b/packages/objectql/src/federated-injected-column-readers.test.ts index 30e9972393b..4419f7e55d4 100644 --- a/packages/objectql/src/federated-injected-column-readers.test.ts +++ b/packages/objectql/src/federated-injected-column-readers.test.ts @@ -249,6 +249,10 @@ const READERS: Record = { disposition: 'excludes', why: 'the provenance the exclusion reads', }, + 'declared-read-columns.ts#selectedStoredColumns :: unprovisionedInjectedColumns()': { + disposition: 'excludes', + why: 'a read never answers an unprovisioned injected column as null: no storage stands behind it', + }, // ── Seams that are not a storage read of a federated object's column ───── 'engine-referential-integrity.ts#assertReferencesResolve :: referenceTargetOf()': { diff --git a/packages/objectql/src/selected-declared-columns-answered.test.ts b/packages/objectql/src/selected-declared-columns-answered.test.ts new file mode 100644 index 00000000000..6490de8c159 --- /dev/null +++ b/packages/objectql/src/selected-declared-columns-answered.test.ts @@ -0,0 +1,242 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#29408] A read answers every stored declared column it SELECTED: a column + * the driver returned no key for is `null`, as driver-sql's column answers. + * + * The driver here stores only the keys a write carried and answers a + * projection the way MongoDB does — a named key the document lacks is simply + * absent. That is the driver-mongodb / driver-memory shape (measured on both + * through the engine, `packages/runtime/src/declared-column-null-on-read.test.ts` + * and `packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts`), + * and the one this engine seam exists for. + * + * What must stay absent is pinned beside what must appear: a column the read + * did not select, a column outside the declared set, a federated object's + * unprovisioned injected column, and a `formula` field (computed, never + * `null`-filled ahead of its formula). Field-level security's refusal is + * pinned on real drivers with the security plugin composed, in the runtime + * file named above. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; + +import { ObjectQL } from './engine.js'; +import { selectedStoredColumns, withSelectedColumnsAnswered } from './declared-read-columns.js'; + +type Row = Record; + +interface DriverAst { + where?: Record; + fields?: string[]; + limit?: number; +} + +function makeSparseDriver(name: string) { + const tables = new Map>(); + const tableFor = (o: string): Map => { + let t = tables.get(o); + if (!t) { t = new Map(); tables.set(o, t); } + return t; + }; + const matches = (row: Row, where: Record | undefined): boolean => { + if (!where) return true; + for (const [k, v] of Object.entries(where)) { + if (k.startsWith('$')) continue; + if ((row[k] ?? null) !== (v ?? null)) return false; + } + return true; + }; + const run = (object: string, ast: DriverAst | undefined): Row[] => { + let out = Array.from(tableFor(object).values()).filter((r) => matches(r, ast?.where)); + if (typeof ast?.limit === 'number') out = out.slice(0, ast.limit); + const fields = Array.isArray(ast?.fields) && ast.fields.length > 0 ? ast.fields : undefined; + // A projection keeps the named keys the stored document HAS — no key for + // one it lacks — plus `id`, as driver-mongodb projects. + return out.map((r) => (fields + ? Object.fromEntries(Object.entries(r).filter(([k]) => k === 'id' || fields.includes(k))) + : { ...r })); + }; + let seq = 0; + const driver = { + name, version: '0.0.0', supports: {}, + async connect(): Promise {}, + async disconnect(): Promise {}, + async checkHealth(): Promise { return true; }, + async execute(): Promise { return null; }, + async find(object: string, ast?: DriverAst): Promise { return run(object, ast); }, + async findOne(object: string, ast?: DriverAst): Promise { return run(object, ast)[0] ?? null; }, + async create(object: string, data: Row): Promise { + seq += 1; + const row: Row = { ...data, id: (data.id as string | undefined) ?? `new_${seq}` }; + tableFor(object).set(String(row.id), row); + return { ...row }; + }, + async update(object: string, id: string, data: Row): Promise { + const next: Row = { ...tableFor(object).get(id), ...data, id }; + tableFor(object).set(id, next); + return { ...next }; + }, + async delete(object: string, id: string): Promise { return tableFor(object).delete(id); }, + async count(object: string, ast?: DriverAst): Promise { return run(object, ast).length; }, + registerExternalObject() {}, + async syncSchema() {}, + }; + return { + driver, + seed: (object: string, row: Row) => { tableFor(object).set(String(row.id), { ...row }); }, + stored: (object: string, id: string) => tableFor(object).get(id), + }; +} + +const ITEM = 'nk_item'; +const REMOTE = 'nk_remote_customer'; +const PACKAGE_ID = 'com.example.nk'; + +/** The fields of {@link ITEM}: `note` and `qty` are never written on `itm_1`. */ +const ITEM_FIELDS = { + name: { type: 'text' }, + note: { type: 'text' }, + qty: { type: 'number' }, + doubled: { type: 'formula', expression: { dialect: 'cel', source: 'record.qty != null ? record.qty * 2 : -1' } }, +}; + +async function boot() { + const engine = new ObjectQL(); + const local = makeSparseDriver('memory'); + const remote = makeSparseDriver('nk_external'); + engine.registerDriver(local.driver as never, true); + engine.registerDriver(remote.driver as never); + await engine.init(); + engine.registry.registerObject({ name: ITEM, label: 'Item', fields: ITEM_FIELDS } as never, PACKAGE_ID); + engine.registry.registerObject({ + name: REMOTE, + datasource: 'nk_external', + external: { remoteName: 'customers' }, + fields: { name: { type: 'text' }, email: { type: 'text' } }, + } as never, PACKAGE_ID); + // Only `name` written — every other declared column is a key the driver + // never stored. `id` and the two timestamps are stored the way a driver + // stamps them. + local.seed(ITEM, { id: 'itm_1', name: 'Ada', created_at: '2026-01-01T00:00:00.000Z', updated_at: '2026-01-01T00:00:00.000Z' }); + remote.seed(REMOTE, { id: 'cus_1', name: 'Remote Co' }); + return { engine, local, protocol: new ObjectStackProtocolImplementation(engine as never) }; +} + +const own = (row: unknown, key: string): boolean => + row != null && typeof row === 'object' && Object.prototype.hasOwnProperty.call(row, key); + +describe('[#29408] a selected declared column the driver did not return is answered null', () => { + let h: Awaited>; + beforeEach(async () => { h = await boot(); }); + + it('the fixture is real: the stored row carries no key for the unwritten columns', () => { + const stored = h.local.stored(ITEM, 'itm_1')!; + expect(own(stored, 'note')).toBe(false); + expect(own(stored, 'qty')).toBe(false); + }); + + it('a whole-record read (findOne, no projection) answers every unwritten stored column null', async () => { + const row = await h.engine.findOne(ITEM, { where: { id: 'itm_1' } }); + expect(row).toMatchObject({ id: 'itm_1', name: 'Ada', note: null, qty: null }); + expect(own(row, 'note')).toBe(true); + expect(own(row, 'qty')).toBe(true); + }); + + it('a list read (find, no projection) answers them null too', async () => { + const rows = await h.engine.find(ITEM, {}); + expect(rows).toHaveLength(1); + expect(own(rows[0], 'note')).toBe(true); + expect(rows[0]).toMatchObject({ note: null, qty: null }); + }); + + it('an explicit projection naming the column answers it null, on find and findOne', async () => { + const one = await h.engine.findOne(ITEM, { where: { id: 'itm_1' }, fields: ['name', 'note'] }); + expect(own(one, 'note')).toBe(true); + expect(one!.note).toBeNull(); + const rows = await h.engine.find(ITEM, { fields: ['name', 'note'] }); + expect(own(rows[0], 'note')).toBe(true); + expect(rows[0]!.note).toBeNull(); + }); + + it('NOT SELECTED stays absent: a projection that does not name the column never answers it', async () => { + const one = await h.engine.findOne(ITEM, { where: { id: 'itm_1' }, fields: ['name'] }); + expect(own(one, 'note')).toBe(false); + expect(own(one, 'qty')).toBe(false); + const rows = await h.engine.find(ITEM, { fields: ['name'] }); + expect(own(rows[0], 'note')).toBe(false); + expect(own(rows[0], 'qty')).toBe(false); + }); + + it('a formula field keeps its computed value — its inputs read as null, never the formula itself', async () => { + const row = await h.engine.findOne(ITEM, { where: { id: 'itm_1' } }); + expect(row!.doubled).toBe(-1); + const narrowed = await h.engine.findOne(ITEM, { where: { id: 'itm_1' }, fields: ['name', 'doubled'] }); + // `qty` was read for the formula (the projection widening) and is cut off + // the answer again: widening is not selecting. + expect(narrowed!.doubled).toBe(-1); + expect(own(narrowed, 'qty')).toBe(false); + }); + + it('a stored value and a stored null are answered as stored', async () => { + await h.engine.update(ITEM, { id: 'itm_1', note: 'kept', qty: null } as never); + const row = await h.engine.findOne(ITEM, { where: { id: 'itm_1' } }); + expect(row).toMatchObject({ note: 'kept', qty: null }); + }); + + it('FEDERATED stays absent: an unprovisioned injected column of an external object is not answered', async () => { + const row = await h.engine.findOne(REMOTE, { where: { id: 'cus_1' } }); + // The declared remote column the driver did not return is answered… + expect(own(row, 'email')).toBe(true); + expect(row!.email).toBeNull(); + // …the platform's injected anchors, which have no storage behind them + // on a federated object, are not. + for (const injected of ['organization_id', 'owner_id', 'created_by', 'created_at']) { + expect(own(row, injected)).toBe(false); + } + }); + + it('the doors: GET /data/:object/:id (getData) and the list route (findData) carry the null', async () => { + const got: any = await h.protocol.getData({ object: ITEM, id: 'itm_1' }); + expect(own(got.record, 'note')).toBe(true); + expect(got.record.note).toBeNull(); + const listed: any = await h.protocol.findData({ object: ITEM, query: {} }); + expect(own(listed.records[0], 'note')).toBe(true); + expect(listed.records[0].note).toBeNull(); + }); + + it('the driver\'s stored row is never mutated by the read', async () => { + await h.engine.find(ITEM, {}); + await h.engine.findOne(ITEM, { where: { id: 'itm_1' } }); + expect(own(h.local.stored(ITEM, 'itm_1'), 'note')).toBe(false); + }); +}); + +describe('[#29408] selectedStoredColumns — the set a row is answered on', () => { + const schema = { fields: ITEM_FIELDS }; + + it('no projection selects every stored declared column and the provisioned three, never a formula', () => { + expect([...selectedStoredColumns(schema, undefined)!].sort()).toEqual( + ['created_at', 'id', 'name', 'note', 'qty', 'updated_at'], + ); + }); + + it('OUTSIDE THE DECLARED SET stays absent: a projected name no field declares is never selected', () => { + expect([...selectedStoredColumns(schema, ['name', 'ghost', 'doubled'])!]).toEqual(['name']); + expect(withSelectedColumnsAnswered({ id: 'x' }, selectedStoredColumns(schema, ['ghost']))).toStrictEqual({ id: 'x' }); + }); + + it('no checkable field map is no opinion: the row is returned as given', () => { + const row = { id: 'x' }; + expect(selectedStoredColumns({ fields: {} }, undefined)).toBeUndefined(); + expect(withSelectedColumnsAnswered(row, selectedStoredColumns(undefined, undefined))).toBe(row); + }); + + it('an own key holding undefined is answered null; nothing to answer keeps the reference', () => { + const selected = selectedStoredColumns(schema, ['name', 'note'])!; + expect(withSelectedColumnsAnswered({ name: 'a', note: undefined }, selected)).toStrictEqual({ name: 'a', note: null }); + const full = { name: 'a', note: 'b' }; + expect(withSelectedColumnsAnswered(full, selected)).toBe(full); + }); +}); From 90223ad08efea2622d9d7db54c115a56f01c6f88 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 15:59:18 +0000 Subject: [PATCH 2/9] test: pin a selected unwritten declared column as null on SQLite, memory and mongo, and FLS refusal as absent Claude-Session: https://claude.ai/code/session_01ADHkmRKR6AV3yXf6HimS9d Co-authored-by: Claude --- .../29408-selected-declared-column-null.md | 13 ++ ...ngodb-declared-column-null-on-read.test.ts | 114 +++++++++++ .../src/declared-column-null-on-read.test.ts | 179 ++++++++++++++++++ 3 files changed, 306 insertions(+) create mode 100644 .changeset/29408-selected-declared-column-null.md create mode 100644 packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts create mode 100644 packages/runtime/src/declared-column-null-on-read.test.ts diff --git a/.changeset/29408-selected-declared-column-null.md b/.changeset/29408-selected-declared-column-null.md new file mode 100644 index 00000000000..fd8021b28a4 --- /dev/null +++ b/.changeset/29408-selected-declared-column-null.md @@ -0,0 +1,13 @@ +--- +'@objectstack/objectql': patch +--- + +fix(objectql): a read answers a selected declared column the driver did not return as `null`, so one record reads the same on every driver (#29408) + +Clause-②: no — nothing a request may send is refused or newly admitted; measured on base and head, every read returns the same rows and the same values, and only a record body gains a `null` key for a stored declared column the read selected and the driver returned no key for. +Metadata-protocol: none + +- **Before.** driver-sql answered a declared field that was never written as `null` (its table has the column), while driver-mongodb and driver-memory answered no key at all (they store only the columns a write touched). The same record read `{ title: 'x', note: null }` on SQL and `{ title: 'x' }` on mongo or memory. +- **Now.** The engine answers every stored declared column the read selected as `null` when the driver returned no key for it, in `find` and `findOne`, so every door on them (`GET /data/:object/:id`, the list route and `POST /data/:object/query`, export, search, `expand` sub-reads, a hook's or a flow's in-process read) answers alike on every driver. Selected means the whole declared record when the read names no `fields`, else the declared columns its `fields` name. +- **Still absent.** A column the read's `fields` do not name; a column field-level security refuses the caller (the security mask still deletes it); a `formula` field ahead of its formula (it keeps its computed value); a name outside the declared set; and an injected platform column a federated (`external`) object has no storage for. +- **Who sees it.** A client of an object served by driver-mongodb or driver-memory whose records have unwritten declared fields. Such a field now arrives as a key holding `null` instead of a missing key: a client-side `has(record.x)` is now `true` for it, as it already was on SQL, so test emptiness with `record.x != null`. A write's stored row is unchanged; the record a write answers (insert or update echo) is unchanged. diff --git a/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts b/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts new file mode 100644 index 00000000000..285b6a02df2 --- /dev/null +++ b/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts @@ -0,0 +1,114 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#29408] driver-mongodb through the engine: a declared column the read + * selected and the write never touched is answered `null`, as driver-sql's + * column answers — not a missing key. + * + * This driver stores no key for a field given no value (`withoutUndefinedOwnKeys` + * at the insert doors, `mongodb-driver.ts`), and a MongoDB projection that + * names a key the document lacks returns no such key. So the row the driver + * hands the engine is sparse; the engine answers it on the selected stored + * columns (`selectedStoredColumns`, `@objectstack/objectql`'s + * `declared-read-columns.ts`). Measured on `origin/main` before the fix with + * this same harness: every read below answered no `note` key. + * + * Runs without a server, on the fake `Db` pattern `mongodb-findone-options.test.ts` + * established (`getCollection` is `this.db.collection(name)`). The fake answers + * a projection as a real mongod does — the named keys the document HAS, never + * an own key holding `undefined` for one it lacks — which is the one property + * of the server this pin depends on. The real-server suites are gated on + * `mongodb-memory-server` (#5517). + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { MongoDBDriver } from './mongodb-driver.js'; + +type Doc = Record; + +/** A fake `Db`: one collection, stored documents kept as inserted. */ +function fakeDb() { + const docs: Doc[] = []; + const project = (doc: Doc, projection: Record | undefined): Doc => { + const keep = Object.keys(projection ?? {}).filter((k) => projection![k] === 1); + const out: Doc = {}; + for (const [k, v] of Object.entries(doc)) { + if (k === '_id' && projection?._id === 0) continue; + if (keep.length > 0 && !keep.includes(k)) continue; + out[k] = v; + } + return out; + }; + const matches = (doc: Doc, filter: Doc | undefined): boolean => + Object.entries(filter ?? {}).every(([k, v]) => + v !== null && typeof v === 'object' && '$eq' in (v as Doc) ? doc[k] === (v as Doc).$eq : doc[k] === v); + const collection = { + insertOne: async (doc: Doc) => { docs.push({ _id: `oid_${docs.length}`, ...doc }); return { insertedId: `oid_${docs.length}` }; }, + findOne: async (filter: Doc, options: { projection?: Record }) => { + const d = docs.find((x) => matches(x, filter)); + return d ? project(d, options?.projection) : null; + }, + find: (filter: Doc, options: { projection?: Record }) => ({ + toArray: async () => docs.filter((x) => matches(x, filter)).map((d) => project(d, options?.projection)), + }), + }; + return { db: { collection: () => collection }, docs }; +} + +const OBJECT = 'nk_mongo_item'; + +const own = (row: unknown, key: string): boolean => + row != null && typeof row === 'object' && Object.prototype.hasOwnProperty.call(row, key); + +describe('[#29408] driver-mongodb: a selected, unwritten declared column reads null through the engine', () => { + let engine: ObjectQL; + let docs: Doc[]; + let id: string; + + beforeEach(async () => { + const driver = new MongoDBDriver({ url: 'mongodb://127.0.0.1:1/unused' }); + const fake = fakeDb(); + docs = fake.docs; + (driver as any).db = fake.db; + (driver as any).connect = async () => {}; + (driver as any).disconnect = async () => {}; + engine = new ObjectQL(); + engine.registerDriver(driver as never, true); + await engine.init(); + engine.registry.registerObject({ + name: OBJECT, + label: 'Item', + fields: { name: { type: 'text' }, note: { type: 'text' } }, + } as never); + const created = await engine.insert(OBJECT, { name: 'Ada' }) as Doc; + id = String(created.id); + }); + + it('the fixture is real: the stored document carries no key for the unwritten column', () => { + expect(docs).toHaveLength(1); + expect(own(docs[0], 'note')).toBe(false); + }); + + it('whole-record read and list read answer it null', async () => { + const one = await engine.findOne(OBJECT, { where: { id } }); + expect(own(one, 'note')).toBe(true); + expect(one!.note).toBeNull(); + const rows = await engine.find(OBJECT, {}); + expect(own(rows[0], 'note')).toBe(true); + expect(rows[0]!.note).toBeNull(); + }); + + it('a projection naming it answers it null', async () => { + const one = await engine.findOne(OBJECT, { where: { id }, fields: ['name', 'note'] }); + expect(own(one, 'note')).toBe(true); + expect(one!.note).toBeNull(); + }); + + it('NOT SELECTED stays absent: a projection that does not name it never answers it', async () => { + const one = await engine.findOne(OBJECT, { where: { id }, fields: ['name'] }); + expect(own(one, 'note')).toBe(false); + const rows = await engine.find(OBJECT, { fields: ['name'] }); + expect(own(rows[0], 'note')).toBe(false); + }); +}); diff --git a/packages/runtime/src/declared-column-null-on-read.test.ts b/packages/runtime/src/declared-column-null-on-read.test.ts new file mode 100644 index 00000000000..bfe44555196 --- /dev/null +++ b/packages/runtime/src/declared-column-null-on-read.test.ts @@ -0,0 +1,179 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#29408] One record reads the same on every driver: a declared column the + * read selected and the write never touched is `null`, not a missing key. + * + * Measured on `origin/main` before the fix, through the engine, a row inserted + * as `{ title }` on an object that also declares `note`: + * + * | read | driver-sql (SQLite) | driver-memory | + * |:-----------------------------|:--------------------|:--------------| + * | findOne, no projection | `note: null` | no `note` key | + * | findOne `fields` naming note | `note: null` | no `note` key | + * | findOne `fields` without it | no `note` key | no `note` key | + * | find, no projection | `note: null` | no `note` key | + * + * driver-mongodb answers as driver-memory does (its insert doors store no key + * for a field given no value); it is pinned in its own package, + * `mongodb-declared-column-null-on-read.test.ts`. + * + * The engine answers the selected column `null` in one place + * (`selectedStoredColumns` in `@objectstack/objectql`'s + * `declared-read-columns.ts`), so both drivers here answer alike. The security + * plugin is composed with a member who may not read `secret`: field-level + * security's refusal must keep the key ABSENT — a refused column must never + * reappear as `null`, which would tell the member the field exists and is + * empty. A system read of the same row is the control: there `secret` IS a + * selected, readable, unwritten column, and is answered `null`. + * + * Reads `@objectstack/objectql` from its built `dist/`, as every runtime suite + * does: rebuild objectql before trusting a run of this file. + */ + +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; +import { PermissionSetSchema } from '@objectstack/spec/security'; +import type { EngineQueryOptions } from '@objectstack/spec/data'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { InMemoryDriver } from '@objectstack/driver-memory'; +import { SecurityPlugin } from '@objectstack/plugin-security'; +import { installDefaultOrganization } from './default-organization.testkit.js'; + +const LEDGER = 'nk_null_ledger'; +const ROW_ID = 'nk1'; + +const SYS_CTX = { isSystem: true, userId: 'usr_system' }; + +const MEMBER_SET = PermissionSetSchema.parse({ + name: 'member_default', + label: 'Member', + objects: { '*': { allowRead: true, allowCreate: true, allowEdit: true, allowDelete: true } }, + fields: { [`${LEDGER}.secret`]: { readable: false, editable: false } }, +}); + +const MEMBER_CTX = { userId: 'usr_member', positions: [] as string[], permissions: [MEMBER_SET.name], posture: 'MEMBER' as const }; + +type Ctx = EngineQueryOptions['context']; + +const own = (row: unknown, key: string): boolean => + row != null && typeof row === 'object' && Object.prototype.hasOwnProperty.call(row, key); + +/** + * Each driver with the composition it serves: driver-sql under a real + * `single` install's Default Organization (its tenant column is NOT NULL); + * driver-memory with none, since it refuses any tenant-scoped call (it has no + * row-level tenant isolation) and is served unscoped. + */ +const DRIVERS: Array<[string, () => unknown, boolean]> = [ + ['driver-sql (SQLite)', () => new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true } as never), true], + ['driver-memory', () => new InMemoryDriver(), false], +]; + +describe.each(DRIVERS)('[#29408] a selected, unwritten declared column reads null — %s', (_label, makeDriver, withOrganization) => { + let engine: ObjectQL; + + beforeAll(async () => { + engine = new ObjectQL(); + engine.registerDriver(makeDriver() as never, true); + await engine.init(); + engine.registerApp({ + id: 'com.objectstack.qa.declared-column-null-on-read', + name: 'Declared column null on read', + version: '1.0.0', + type: 'plugin', + scope: 'system', + objects: [ + { + name: LEDGER, + label: 'Ledger', + sharingModel: 'public_read_write', + fields: { + title: { name: 'title', type: 'text' }, + note: { name: 'note', type: 'text' }, + secret: { name: 'secret', type: 'text' }, + }, + }, + ], + } as never); + await engine.syncSchemas(); + if (withOrganization) await installDefaultOrganization(engine); + + const services: Record = { + manifest: { register: vi.fn() }, + objectql: engine, + metadata: { + get: async (_type: string, name: string) => engine.getSchema(name) ?? null, + list: async () => [MEMBER_SET], + }, + }; + const ctx = { + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, + registerService: vi.fn(), + getService: (name: string) => { + if (!(name in services)) throw new Error(`service not registered: ${name}`); + return services[name]; + }, + }; + const plugin = new SecurityPlugin({ fallbackPermissionSet: 'member_default' }); + await plugin.init(ctx as never); + await plugin.start(ctx as never); + vi.spyOn((engine as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => undefined); + + // Only `title` is written: `note` and `secret` are never given a value. + await engine.insert(LEDGER, { id: ROW_ID, title: 'written' }, { context: SYS_CTX } as never); + }, 60_000); + + afterAll(async () => { + try { await engine?.destroy(); } catch { /* noop */ } + }); + + const byId = (context: Ctx, fields?: string[]): EngineQueryOptions => + ({ where: { id: ROW_ID }, ...(fields ? { fields } : {}), context }); + + it('whole-record read (findOne, no projection): the unwritten column is null', async () => { + const row = await engine.findOne(LEDGER, byId(MEMBER_CTX)); + expect(row).toMatchObject({ id: ROW_ID, title: 'written' }); + expect(own(row, 'note')).toBe(true); + expect(row!.note).toBeNull(); + }); + + it('list read (find, no projection): the unwritten column is null', async () => { + const rows = await engine.find(LEDGER, { context: MEMBER_CTX }); + expect(rows).toHaveLength(1); + expect(own(rows[0], 'note')).toBe(true); + expect(rows[0]!.note).toBeNull(); + }); + + it('a projection naming the column answers it null, on findOne and find', async () => { + const one = await engine.findOne(LEDGER, byId(MEMBER_CTX, ['title', 'note'])); + expect(own(one, 'note')).toBe(true); + expect(one!.note).toBeNull(); + const rows = await engine.find(LEDGER, { fields: ['title', 'note'], context: MEMBER_CTX }); + expect(own(rows[0], 'note')).toBe(true); + expect(rows[0]!.note).toBeNull(); + }); + + it('NOT SELECTED stays absent: a projection that does not name the column never answers it', async () => { + const one = await engine.findOne(LEDGER, byId(MEMBER_CTX, ['title'])); + expect(own(one, 'note')).toBe(false); + const rows = await engine.find(LEDGER, { fields: ['title'], context: MEMBER_CTX }); + expect(own(rows[0], 'note')).toBe(false); + }); + + it('FLS-REFUSED stays absent: a column the member may not read never reappears as null', async () => { + const whole = await engine.findOne(LEDGER, byId(MEMBER_CTX)); + expect(own(whole, 'secret')).toBe(false); + const listed = await engine.find(LEDGER, { context: MEMBER_CTX }); + expect(own(listed[0], 'secret')).toBe(false); + const named = await engine.findOne(LEDGER, byId(MEMBER_CTX, ['title', 'note', 'secret'])); + expect(own(named, 'secret')).toBe(false); + expect(own(named, 'note')).toBe(true); + }); + + it('control: the same column, read by the system, is a selected unwritten column and answers null', async () => { + const row = await engine.findOne(LEDGER, byId(SYS_CTX)); + expect(own(row, 'secret')).toBe(true); + expect(row!.secret).toBeNull(); + }); +}); From dd72a56fd700a648d66c5254b25ba158302edb85 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 16:14:15 +0000 Subject: [PATCH 3/9] test(objectql): expectations move with a selected unwritten declared column now read null Five expand cases in engine.test.ts read a related record whose test-double rows carry no timestamps, and the formula-projection CONTROL reads a stored row without the registry-injected queue column: each now answers those declared columns null. Claude-Session: https://claude.ai/code/session_01ADHkmRKR6AV3yXf6HimS9d Co-authored-by: Claude --- .../src/engine-formula-projection-trim.test.ts | 4 +++- packages/objectql/src/engine.test.ts | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/packages/objectql/src/engine-formula-projection-trim.test.ts b/packages/objectql/src/engine-formula-projection-trim.test.ts index b38f7af428f..abd6f9e6bd6 100644 --- a/packages/objectql/src/engine-formula-projection-trim.test.ts +++ b/packages/objectql/src/engine-formula-projection-trim.test.ts @@ -187,7 +187,9 @@ describe.each(['find', 'findOne'])('#22300 — %s: a formula in the projec it('CONTROL: a read with no projection is unchanged — every declared column, the formula computed', async () => { const { engine, projections } = await boot(); const row = await readOne(engine, verb); - expect(row).toStrictEqual({ ...STORED, total_price: 21 }); + // [#29408] `queue` (registry-injected) is declared but absent from the + // stored row: a whole-record read answers it null. + expect(row).toStrictEqual({ ...STORED, queue: null, total_price: 21 }); expect(projections.at(-1)).toBeUndefined(); }); diff --git a/packages/objectql/src/engine.test.ts b/packages/objectql/src/engine.test.ts index c4cc39c07ef..04666dd12ed 100644 --- a/packages/objectql/src/engine.test.ts +++ b/packages/objectql/src/engine.test.ts @@ -1723,8 +1723,10 @@ describe('ObjectQL Engine', () => { const result = await engine.find('task', { expand: { assignee: { object: 'assignee' } } }); expect(result).toHaveLength(2); - expect(result[0].assignee).toEqual({ id: 'u1', name: 'Alice' }); - expect(result[1].assignee).toEqual({ id: 'u2', name: 'Bob' }); + // [#29408] The related record is answered on every declared column the + // read selected; the double's rows carry no timestamps, so they read null. + expect(result[0].assignee).toEqual({ id: 'u1', name: 'Alice', created_at: null, updated_at: null }); + expect(result[1].assignee).toEqual({ id: 'u2', name: 'Bob', created_at: null, updated_at: null }); // Verify the expand query used $in. The expand sub-read now routes // through the secured `find` path ([#2850]), so the referenced @@ -1768,7 +1770,8 @@ describe('ObjectQL Engine', () => { const result = await engine.find('ticket', { expand: { owner: { object: 'owner' } } }); - expect(result[0].owner).toEqual({ id: 'u1', name: 'Alice' }); + // [#29408] Unwritten declared columns of the related record read null. + expect(result[0].owner).toEqual({ id: 'u1', name: 'Alice', created_at: null, updated_at: null }); expect(mockDriver.find).toHaveBeenLastCalledWith( 'sys_user', expect.objectContaining({ where: { id: { $in: ['u1'] } } }), @@ -1825,7 +1828,8 @@ describe('ObjectQL Engine', () => { ); // Records the driver filtered out keep their raw FK id (unresolved). - expect(result[0].assignee).toEqual({ id: 'u1', name: 'Alice', active: true }); + // [#29408] Unwritten declared columns of the related record read null. + expect(result[0].assignee).toEqual({ id: 'u1', name: 'Alice', active: true, created_at: null, updated_at: null }); expect(result[1].assignee).toBe('u2'); }); @@ -2071,7 +2075,8 @@ describe('ObjectQL Engine', () => { ]); const result = await engine.find('order_item', { expand: { order: { object: 'order' } } }); - expect(result[0].order).toEqual({ id: 'o1', total: 100 }); + // [#29408] Unwritten declared columns of the related record read null. + expect(result[0].order).toEqual({ id: 'o1', total: 100, created_at: null, updated_at: null }); }); it('should skip expand for fields without reference definition', async () => { @@ -2558,7 +2563,8 @@ describe('ObjectQL Engine', () => { const result = await engine.find('task', { expand: { project: { object: 'project' } } }); // Project expanded, but org inside project remains as raw ID - expect(result[0].project).toEqual({ id: 'p1', org: 'o1' }); + // [#29408] Unwritten declared columns of the related record read null. + expect(result[0].project).toEqual({ id: 'p1', org: 'o1', created_at: null, updated_at: null }); expect(mockDriver.find).toHaveBeenCalledTimes(2); // Only primary + 1 expand query }); From 7daf965284caf8713e85f7acc5c462c93ec4ebba Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 16:17:30 +0000 Subject: [PATCH 4/9] test: bind no new driver-memory consumer; its shape stays pinned by the objectql sparse driver Claude-Session: https://claude.ai/code/session_01ADHkmRKR6AV3yXf6HimS9d Co-authored-by: Claude --- .../selected-declared-columns-answered.test.ts | 13 ++++++++----- .../src/declared-column-null-on-read.test.ts | 17 +++++++++-------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/objectql/src/selected-declared-columns-answered.test.ts b/packages/objectql/src/selected-declared-columns-answered.test.ts index 6490de8c159..bbdf44fb7ed 100644 --- a/packages/objectql/src/selected-declared-columns-answered.test.ts +++ b/packages/objectql/src/selected-declared-columns-answered.test.ts @@ -6,16 +6,19 @@ * * The driver here stores only the keys a write carried and answers a * projection the way MongoDB does — a named key the document lacks is simply - * absent. That is the driver-mongodb / driver-memory shape (measured on both - * through the engine, `packages/runtime/src/declared-column-null-on-read.test.ts` - * and `packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts`), - * and the one this engine seam exists for. + * absent. That is the driver-mongodb / driver-memory shape (both measured + * through the engine on `origin/main`: no key for a never-written declared + * field, where driver-sql answers `null`), and the one this engine seam exists + * for. driver-mongodb is pinned in its own package + * (`mongodb-declared-column-null-on-read.test.ts`) and driver-sql in + * `@objectstack/runtime` (`declared-column-null-on-read.test.ts`); this file is + * the driver-memory shape's pin. * * What must stay absent is pinned beside what must appear: a column the read * did not select, a column outside the declared set, a federated object's * unprovisioned injected column, and a `formula` field (computed, never * `null`-filled ahead of its formula). Field-level security's refusal is - * pinned on real drivers with the security plugin composed, in the runtime + * pinned on a real driver with the security plugin composed, in the runtime * file named above. */ diff --git a/packages/runtime/src/declared-column-null-on-read.test.ts b/packages/runtime/src/declared-column-null-on-read.test.ts index bfe44555196..3dff936bc21 100644 --- a/packages/runtime/src/declared-column-null-on-read.test.ts +++ b/packages/runtime/src/declared-column-null-on-read.test.ts @@ -16,11 +16,16 @@ * * driver-mongodb answers as driver-memory does (its insert doors store no key * for a field given no value); it is pinned in its own package, - * `mongodb-declared-column-null-on-read.test.ts`. + * `mongodb-declared-column-null-on-read.test.ts`. driver-memory is not bound + * here: a new test consumer of it is a maintainer ruling + * (`check:driver-memory-census`), so its shape is pinned through the engine by + * the sparse driver in `@objectstack/objectql`'s + * `selected-declared-columns-answered.test.ts`, and the table's memory column + * was measured once, with a probe that is not kept. * * The engine answers the selected column `null` in one place * (`selectedStoredColumns` in `@objectstack/objectql`'s - * `declared-read-columns.ts`), so both drivers here answer alike. The security + * `declared-read-columns.ts`), so every driver answers as SQL does. The security * plugin is composed with a member who may not read `secret`: field-level * security's refusal must keep the key ABSENT — a refused column must never * reappear as `null`, which would tell the member the field exists and is @@ -36,7 +41,6 @@ import { PermissionSetSchema } from '@objectstack/spec/security'; import type { EngineQueryOptions } from '@objectstack/spec/data'; import { ObjectQL } from '@objectstack/objectql'; import { SqlDriver } from '@objectstack/driver-sql'; -import { InMemoryDriver } from '@objectstack/driver-memory'; import { SecurityPlugin } from '@objectstack/plugin-security'; import { installDefaultOrganization } from './default-organization.testkit.js'; @@ -60,14 +64,11 @@ const own = (row: unknown, key: string): boolean => row != null && typeof row === 'object' && Object.prototype.hasOwnProperty.call(row, key); /** - * Each driver with the composition it serves: driver-sql under a real - * `single` install's Default Organization (its tenant column is NOT NULL); - * driver-memory with none, since it refuses any tenant-scoped call (it has no - * row-level tenant isolation) and is served unscoped. + * The driver with the composition it serves: driver-sql under a real `single` + * install's Default Organization (its tenant column is NOT NULL). */ const DRIVERS: Array<[string, () => unknown, boolean]> = [ ['driver-sql (SQLite)', () => new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true } as never), true], - ['driver-memory', () => new InMemoryDriver(), false], ]; describe.each(DRIVERS)('[#29408] a selected, unwritten declared column reads null — %s', (_label, makeDriver, withOrganization) => { From b38c236021b01ebd317aa863b9e089b1d4776b43 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 17:22:40 +0000 Subject: [PATCH 5/9] test(driver-mongodb): the fake Db refuses a filter operator it does not implement Claude-Session: https://claude.ai/code/session_01ADHkmRKR6AV3yXf6HimS9d Co-authored-by: Claude --- .../mongodb-declared-column-null-on-read.test.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts b/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts index 285b6a02df2..d75ba4a3a12 100644 --- a/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts +++ b/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts @@ -40,9 +40,18 @@ function fakeDb() { } return out; }; + // Equality on plain keys only — the reads below filter by `id` alone. A + // combinator or any other operator is REFUSED, never read as a field name. const matches = (doc: Doc, filter: Doc | undefined): boolean => - Object.entries(filter ?? {}).every(([k, v]) => - v !== null && typeof v === 'object' && '$eq' in (v as Doc) ? doc[k] === (v as Doc).$eq : doc[k] === v); + Object.entries(filter ?? {}).every(([k, v]) => { + if (k.startsWith('$')) throw new Error(`fake Db: filter operator ${k} is not implemented`); + if (v !== null && typeof v === 'object') { + const ops = Object.keys(v as Doc); + if (ops.length !== 1 || ops[0] !== '$eq') throw new Error(`fake Db: operator ${ops.join(',')} is not implemented`); + return doc[k] === (v as Doc).$eq; + } + return doc[k] === v; + }); const collection = { insertOne: async (doc: Doc) => { docs.push({ _id: `oid_${docs.length}`, ...doc }); return { insertedId: `oid_${docs.length}` }; }, findOne: async (filter: Doc, options: { projection?: Record }) => { From 59e1395cdf112c02d54fdfa06645919503c8ed3e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 18:15:55 +0000 Subject: [PATCH 6/9] test(plugin-audit): read the stamped keys off the stored row, not a read-back A read now answers every selected declared column, null where nothing was stored (#29408), so an engine read-back cannot tell a key the read-audit writer stamped from a column it never wrote. The #9539 guard takes its key set from the stub driver's stored row; both assertions stay strict. Claude-Session: https://claude.ai/code/session_01ADHkmRKR6AV3yXf6HimS9d Co-authored-by: Claude --- ...sys-audit-log-record-views-columns.test.ts | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/plugins/plugin-audit/src/objects/sys-audit-log-record-views-columns.test.ts b/packages/plugins/plugin-audit/src/objects/sys-audit-log-record-views-columns.test.ts index 9cea192c108..922bee46a0d 100644 --- a/packages/plugins/plugin-audit/src/objects/sys-audit-log-record-views-columns.test.ts +++ b/packages/plugins/plugin-audit/src/objects/sys-audit-log-record-views-columns.test.ts @@ -21,8 +21,11 @@ * introspected directly — so this test runs the writer for real, against a * real engine, on a read shaped to make every conditionally-stamped key * present (a human principal that ALSO carries a service `actor` label, on a - * record that carries an `organization_id`), and reads the keys back off the - * row the writer actually persisted. If `buildRow` ever stops stamping a key + * record that carries an `organization_id`), and reads the keys off the row + * the writer actually persisted — the stub driver's STORED row, not an engine + * read of it: a read answers every declared column it selected, `null` where + * nothing was stored (#29408), so a read-back cannot tell a stamped key from + * an unwritten column. If `buildRow` ever stops stamping a key * this view lists, the observed key set shrinks and the assertion goes red — * no hand-kept list to fall out of sync with the writer it is supposed to * police. @@ -110,6 +113,10 @@ function makeStubDriver() { return []; }, async bulkDelete() {}, + /** The rows as stored — what the writer persisted, before any read shapes them. */ + storedRows(object: string): Record[] { + return Array.from(storeFor(object).values()).map((r) => ({ ...r })); + }, async updateMany() { return 0; }, @@ -139,7 +146,8 @@ let stampedKeys: Set; beforeAll(async () => { const engine = new ObjectQL(); - engine.registerDriver(makeStubDriver(), true); + const driver = makeStubDriver(); + engine.registerDriver(driver, true); await engine.init(); engine.registry.registerObject(contactObject as any, HARNESS_PACKAGE); // The REAL sys_audit_log object under test — not a hand-copied stand-in — @@ -165,9 +173,14 @@ beforeAll(async () => { }); await writer.flush(); + // The read still sees exactly the one row the writer persisted… const rows = (await engine.find('sys_audit_log', {})) as Array>; expect(rows).toHaveLength(1); - stampedKeys = new Set(Object.keys(rows[0])); + // …and the stamped keys are the STORED row's own: a read answers an + // unwritten declared column as `null` (#29408), which is not a stamp. + const stored = driver.storedRows('sys_audit_log'); + expect(stored).toHaveLength(1); + stampedKeys = new Set(Object.keys(stored[0])); }); /** The columns the shipped `record_views` list view declares. */ From fbc9148c1cad63df88d3f1c7101b5d31ceebb400 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 18:18:15 +0000 Subject: [PATCH 7/9] fix(objectql): a federated object's undeclared platform timestamp stays absent on a read created_at / updated_at count as provisioned because driver-sql builds them on every table it creates. The platform builds no table for a federated (external) object, so on one with systemFields false or audit false nothing stores them, and a read must not answer them null. A timestamp the author declares (maps from the remote) is a stored column and is still answered. Pinned in selected-declared-columns-answered.test.ts; census row added. Claude-Session: https://claude.ai/code/session_01ADHkmRKR6AV3yXf6HimS9d Co-authored-by: Claude --- .../objectql/src/declared-read-columns.ts | 16 ++++- .../federated-injected-column-readers.test.ts | 4 ++ ...selected-declared-columns-answered.test.ts | 70 ++++++++++++++++++- 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/packages/objectql/src/declared-read-columns.ts b/packages/objectql/src/declared-read-columns.ts index 7a8e501cbbb..7cb95364703 100644 --- a/packages/objectql/src/declared-read-columns.ts +++ b/packages/objectql/src/declared-read-columns.ts @@ -1,6 +1,7 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. import { unprovisionedInjectedColumns } from '@objectstack/spec/data'; +import { isFederatedObject } from './federated-object.js'; /** * [#21571] The columns a read may serve: the object's DECLARED fields plus the @@ -171,6 +172,14 @@ export function rowsWithDeclaredColumnsOnly(rows: T[], declared: ReadonlySet< * - an injected column a federated (ADR-0015 `external`) object does not * provision (`unprovisionedInjectedColumns`) — there is no storage behind * it, so there is no persisted state to answer for; + * - on a federated object, a {@link PLATFORM_PROVISIONED_COLUMNS} timestamp + * its field map does not declare (`systemFields: false` or + * `systemFields.audit: false`, so nothing injected it either). The three are + * "provisioned" because driver-sql creates them on every table IT builds, + * and the platform builds no table for a federated object + * (`isFederatedObject`): the remote holds such a timestamp only where the + * author mapped one, which is a declared field. `id` stays — the remote's + * own primary key backs it through the binding; * - a column field-level security refuses — this runs inside the verb's * middleware chain, on the rows as the driver returned them, and the * security middleware's field mask deletes a refused key from the result @@ -187,9 +196,14 @@ export function selectedStoredColumns( if (!declared) return undefined; const fields = schema!.fields as Record; const unprovisioned = new Set(unprovisionedInjectedColumns(schema)); + const federated = isFederatedObject(schema); const stored = new Set(); for (const name of declared) { - if (fields[name]?.type !== 'formula' && !unprovisioned.has(name)) stored.add(name); + if (fields[name]?.type === 'formula' || unprovisioned.has(name)) continue; + // A platform-provisioned column the object does not declare has storage + // only on a table the platform built. + if (federated && name !== 'id' && !Object.prototype.hasOwnProperty.call(fields, name)) continue; + stored.add(name); } if (!Array.isArray(projection) || projection.length === 0) return stored; const selected = new Set(); diff --git a/packages/objectql/src/federated-injected-column-readers.test.ts b/packages/objectql/src/federated-injected-column-readers.test.ts index 4419f7e55d4..2c788eeeeea 100644 --- a/packages/objectql/src/federated-injected-column-readers.test.ts +++ b/packages/objectql/src/federated-injected-column-readers.test.ts @@ -253,6 +253,10 @@ const READERS: Record = { disposition: 'excludes', why: 'a read never answers an unprovisioned injected column as null: no storage stands behind it', }, + 'declared-read-columns.ts#selectedStoredColumns :: isFederatedObject()': { + disposition: 'excludes', + why: 'a read never answers a platform timestamp a federated object does not declare as null: the platform builds no table for it', + }, // ── Seams that are not a storage read of a federated object's column ───── 'engine-referential-integrity.ts#assertReferencesResolve :: referenceTargetOf()': { diff --git a/packages/objectql/src/selected-declared-columns-answered.test.ts b/packages/objectql/src/selected-declared-columns-answered.test.ts index bbdf44fb7ed..da24529075f 100644 --- a/packages/objectql/src/selected-declared-columns-answered.test.ts +++ b/packages/objectql/src/selected-declared-columns-answered.test.ts @@ -16,8 +16,11 @@ * * What must stay absent is pinned beside what must appear: a column the read * did not select, a column outside the declared set, a federated object's - * unprovisioned injected column, and a `formula` field (computed, never - * `null`-filled ahead of its formula). Field-level security's refusal is + * unprovisioned injected column, a platform timestamp a federated object + * neither declares nor has injected (`systemFields: false` / + * `systemFields.audit: false`), and a `formula` field (virtual, never + * `null`-filled itself; it is computed after the fill, so over an unwritten + * input it now evaluates against `null` instead of faulting). Field-level security's refusal is * pinned on a real driver with the security plugin composed, in the runtime * file named above. */ @@ -95,6 +98,12 @@ function makeSparseDriver(name: string) { const ITEM = 'nk_item'; const REMOTE = 'nk_remote_customer'; +/** Federated, and opted out of the injected columns: nothing provisions a timestamp. */ +const REMOTE_BARE = 'nk_remote_bare'; +/** Federated, audit family off: no injected timestamp either. */ +const REMOTE_NO_AUDIT = 'nk_remote_no_audit'; +/** Federated, the author maps a remote `updated_at`: a declared column. */ +const REMOTE_MAPPED = 'nk_remote_mapped'; const PACKAGE_ID = 'com.example.nk'; /** The fields of {@link ITEM}: `note` and `qty` are never written on `itm_1`. */ @@ -123,7 +132,31 @@ async function boot() { // never stored. `id` and the two timestamps are stored the way a driver // stamps them. local.seed(ITEM, { id: 'itm_1', name: 'Ada', created_at: '2026-01-01T00:00:00.000Z', updated_at: '2026-01-01T00:00:00.000Z' }); + engine.registry.registerObject({ + name: REMOTE_BARE, + datasource: 'nk_external', + external: { remoteName: 'bare' }, + systemFields: false, + fields: { name: { type: 'text' }, email: { type: 'text' } }, + } as never, PACKAGE_ID); + engine.registry.registerObject({ + name: REMOTE_NO_AUDIT, + datasource: 'nk_external', + external: { remoteName: 'no_audit' }, + systemFields: { audit: false }, + fields: { name: { type: 'text' }, email: { type: 'text' } }, + } as never, PACKAGE_ID); + engine.registry.registerObject({ + name: REMOTE_MAPPED, + datasource: 'nk_external', + external: { remoteName: 'mapped' }, + systemFields: false, + fields: { name: { type: 'text' }, updated_at: { type: 'datetime' } }, + } as never, PACKAGE_ID); remote.seed(REMOTE, { id: 'cus_1', name: 'Remote Co' }); + remote.seed(REMOTE_BARE, { id: 'bar_1', name: 'Bare Co' }); + remote.seed(REMOTE_NO_AUDIT, { id: 'noa_1', name: 'No Audit Co' }); + remote.seed(REMOTE_MAPPED, { id: 'map_1', name: 'Mapped Co' }); return { engine, local, protocol: new ObjectStackProtocolImplementation(engine as never) }; } @@ -172,7 +205,7 @@ describe('[#29408] a selected declared column the driver did not return is answe expect(own(rows[0], 'qty')).toBe(false); }); - it('a formula field keeps its computed value — its inputs read as null, never the formula itself', async () => { + it('a formula field is never null-filled itself: over its unwritten input it evaluates against null', async () => { const row = await h.engine.findOne(ITEM, { where: { id: 'itm_1' } }); expect(row!.doubled).toBe(-1); const narrowed = await h.engine.findOne(ITEM, { where: { id: 'itm_1' }, fields: ['name', 'doubled'] }); @@ -200,6 +233,31 @@ describe('[#29408] a selected declared column the driver did not return is answe } }); + it.each([ + ['systemFields: false', REMOTE_BARE, 'bar_1'], + ['systemFields.audit: false', REMOTE_NO_AUDIT, 'noa_1'], + ])('UNPROVISIONED TIMESTAMP stays absent: a federated object with %s is not answered created_at / updated_at', async (_label, object, id) => { + const one = await h.engine.findOne(object, { where: { id } }); + const rows = await h.engine.find(object, {}); + const named = await h.engine.findOne(object, { where: { id }, fields: ['name', 'created_at', 'updated_at'] }); + for (const row of [one, rows[0], named]) { + expect(own(row, 'created_at')).toBe(false); + expect(own(row, 'updated_at')).toBe(false); + // The remote's own primary key backs `id`. + expect(row!.id).toBe(id); + } + // The declared remote column the driver did not return is still answered. + expect(own(one, 'email')).toBe(true); + expect(one!.email).toBeNull(); + }); + + it('a timestamp the federated object DECLARES is a remote column, and is answered null', async () => { + const row = await h.engine.findOne(REMOTE_MAPPED, { where: { id: 'map_1' } }); + expect(own(row, 'updated_at')).toBe(true); + expect(row!.updated_at).toBeNull(); + expect(own(row, 'created_at')).toBe(false); + }); + it('the doors: GET /data/:object/:id (getData) and the list route (findData) carry the null', async () => { const got: any = await h.protocol.getData({ object: ITEM, id: 'itm_1' }); expect(own(got.record, 'note')).toBe(true); @@ -225,6 +283,12 @@ describe('[#29408] selectedStoredColumns — the set a row is answered on', () = ); }); + it('a local object keeps the provisioned three whatever systemFields says: driver-sql builds them on every table', () => { + expect([...selectedStoredColumns({ systemFields: false, fields: { name: { type: 'text' } } }, undefined)!].sort()).toEqual( + ['created_at', 'id', 'name', 'updated_at'], + ); + }); + it('OUTSIDE THE DECLARED SET stays absent: a projected name no field declares is never selected', () => { expect([...selectedStoredColumns(schema, ['name', 'ghost', 'doubled'])!]).toEqual(['name']); expect(withSelectedColumnsAnswered({ id: 'x' }, selectedStoredColumns(schema, ['ghost']))).toStrictEqual({ id: 'x' }); From 30a77059854cd2601a04120ff36ed7740ebd3c30 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 18:18:49 +0000 Subject: [PATCH 8/9] docs(changeset): a formula over an unwritten field now evaluates against null on mongo/memory Claude-Session: https://claude.ai/code/session_01ADHkmRKR6AV3yXf6HimS9d Co-authored-by: Claude --- .changeset/29408-selected-declared-column-null.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.changeset/29408-selected-declared-column-null.md b/.changeset/29408-selected-declared-column-null.md index fd8021b28a4..aa2687f135b 100644 --- a/.changeset/29408-selected-declared-column-null.md +++ b/.changeset/29408-selected-declared-column-null.md @@ -4,10 +4,10 @@ fix(objectql): a read answers a selected declared column the driver did not return as `null`, so one record reads the same on every driver (#29408) -Clause-②: no — nothing a request may send is refused or newly admitted; measured on base and head, every read returns the same rows and the same values, and only a record body gains a `null` key for a stored declared column the read selected and the driver returned no key for. +Clause-②: no — nothing a request may send is refused or newly admitted; measured on base and head, every read returns the same rows, and every value a driver returned is answered as before. A record body gains a `null` key for a stored declared column the read selected and the driver returned no key for, and on driver-mongodb and driver-memory a `formula` over such a column now evaluates against that `null`, as it already did on driver-sql. Metadata-protocol: none - **Before.** driver-sql answered a declared field that was never written as `null` (its table has the column), while driver-mongodb and driver-memory answered no key at all (they store only the columns a write touched). The same record read `{ title: 'x', note: null }` on SQL and `{ title: 'x' }` on mongo or memory. - **Now.** The engine answers every stored declared column the read selected as `null` when the driver returned no key for it, in `find` and `findOne`, so every door on them (`GET /data/:object/:id`, the list route and `POST /data/:object/query`, export, search, `expand` sub-reads, a hook's or a flow's in-process read) answers alike on every driver. Selected means the whole declared record when the read names no `fields`, else the declared columns its `fields` name. -- **Still absent.** A column the read's `fields` do not name; a column field-level security refuses the caller (the security mask still deletes it); a `formula` field ahead of its formula (it keeps its computed value); a name outside the declared set; and an injected platform column a federated (`external`) object has no storage for. -- **Who sees it.** A client of an object served by driver-mongodb or driver-memory whose records have unwritten declared fields. Such a field now arrives as a key holding `null` instead of a missing key: a client-side `has(record.x)` is now `true` for it, as it already was on SQL, so test emptiness with `record.x != null`. A write's stored row is unchanged; the record a write answers (insert or update echo) is unchanged. +- **Still absent.** A column the read's `fields` do not name; a column field-level security refuses the caller (the security mask still deletes it); a `formula` field itself (virtual: never `null`-filled, it is computed after the fill); a name outside the declared set; an injected platform column a federated (`external`) object has no storage for; and, on a federated object, a `created_at` / `updated_at` it neither declares nor has injected (`systemFields: false` or `systemFields.audit: false`), which no table stores. +- **Who sees it.** A client of an object served by driver-mongodb or driver-memory whose records have unwritten declared fields. Such a field now arrives as a key holding `null` instead of a missing key: a client-side `has(record.x)` is now `true` for it, as it already was on SQL, so test emptiness with `record.x != null`. A `formula` over such a field changes value on those two drivers: it used to fault on the missing key and answer `null`, and now it evaluates against `null` and answers what driver-sql answered (`record.qty != null ? record.qty * 2 : -1` was `null`, now `-1`). A write's stored row is unchanged; the record a write answers (insert or update echo) is unchanged. From b55451a4568c13e077880b6a1dec7281f0b2a684 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 21:02:28 +0000 Subject: [PATCH 9/9] test(objectql): type the systemFields: false fixture through a variable The test tsconfig rejects the excess property on the inline literal (TS2353); a named fixture carries it. Claude-Session: https://claude.ai/code/session_01ADHkmRKR6AV3yXf6HimS9d Co-authored-by: Claude --- .../objectql/src/selected-declared-columns-answered.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/objectql/src/selected-declared-columns-answered.test.ts b/packages/objectql/src/selected-declared-columns-answered.test.ts index da24529075f..e07409c7b44 100644 --- a/packages/objectql/src/selected-declared-columns-answered.test.ts +++ b/packages/objectql/src/selected-declared-columns-answered.test.ts @@ -284,7 +284,8 @@ describe('[#29408] selectedStoredColumns — the set a row is answered on', () = }); it('a local object keeps the provisioned three whatever systemFields says: driver-sql builds them on every table', () => { - expect([...selectedStoredColumns({ systemFields: false, fields: { name: { type: 'text' } } }, undefined)!].sort()).toEqual( + const optedOut = { systemFields: false, fields: { name: { type: 'text' } } }; + expect([...selectedStoredColumns(optedOut, undefined)!].sort()).toEqual( ['created_at', 'id', 'name', 'updated_at'], ); });