diff --git a/.changeset/29408-selected-declared-column-null.md b/.changeset/29408-selected-declared-column-null.md new file mode 100644 index 00000000000..aa2687f135b --- /dev/null +++ b/.changeset/29408-selected-declared-column-null.md @@ -0,0 +1,13 @@ +--- +'@objectstack/objectql': patch +--- + +fix(objectql): a read answers a selected declared column the driver did not return as `null`, so one record reads the same on every driver (#29408) + +Clause-②: no — nothing a request may send is refused or newly admitted; measured on base and head, every read returns the same rows, and every value a driver returned is answered as before. A record body gains a `null` key for a stored declared column the read selected and the driver returned no key for, and on driver-mongodb and driver-memory a `formula` over such a column now evaluates against that `null`, as it already did on driver-sql. +Metadata-protocol: none + +- **Before.** driver-sql answered a declared field that was never written as `null` (its table has the column), while driver-mongodb and driver-memory answered no key at all (they store only the columns a write touched). The same record read `{ title: 'x', note: null }` on SQL and `{ title: 'x' }` on mongo or memory. +- **Now.** The engine answers every stored declared column the read selected as `null` when the driver returned no key for it, in `find` and `findOne`, so every door on them (`GET /data/:object/:id`, the list route and `POST /data/:object/query`, export, search, `expand` sub-reads, a hook's or a flow's in-process read) answers alike on every driver. Selected means the whole declared record when the read names no `fields`, else the declared columns its `fields` name. +- **Still absent.** A column the read's `fields` do not name; a column field-level security refuses the caller (the security mask still deletes it); a `formula` field itself (virtual: never `null`-filled, it is computed after the fill); a name outside the declared set; an injected platform column a federated (`external`) object has no storage for; and, on a federated object, a `created_at` / `updated_at` it neither declares nor has injected (`systemFields: false` or `systemFields.audit: false`), which no table stores. +- **Who sees it.** A client of an object served by driver-mongodb or driver-memory whose records have unwritten declared fields. Such a field now arrives as a key holding `null` instead of a missing key: a client-side `has(record.x)` is now `true` for it, as it already was on SQL, so test emptiness with `record.x != null`. A `formula` over such a field changes value on those two drivers: it used to fault on the missing key and answer `null`, and now it evaluates against `null` and answers what driver-sql answered (`record.qty != null ? record.qty * 2 : -1` was `null`, now `-1`). A write's stored row is unchanged; the record a write answers (insert or update echo) is unchanged. diff --git a/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts b/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts new file mode 100644 index 00000000000..d75ba4a3a12 --- /dev/null +++ b/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-read.test.ts @@ -0,0 +1,123 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#29408] driver-mongodb through the engine: a declared column the read + * selected and the write never touched is answered `null`, as driver-sql's + * column answers — not a missing key. + * + * This driver stores no key for a field given no value (`withoutUndefinedOwnKeys` + * at the insert doors, `mongodb-driver.ts`), and a MongoDB projection that + * names a key the document lacks returns no such key. So the row the driver + * hands the engine is sparse; the engine answers it on the selected stored + * columns (`selectedStoredColumns`, `@objectstack/objectql`'s + * `declared-read-columns.ts`). Measured on `origin/main` before the fix with + * this same harness: every read below answered no `note` key. + * + * Runs without a server, on the fake `Db` pattern `mongodb-findone-options.test.ts` + * established (`getCollection` is `this.db.collection(name)`). The fake answers + * a projection as a real mongod does — the named keys the document HAS, never + * an own key holding `undefined` for one it lacks — which is the one property + * of the server this pin depends on. The real-server suites are gated on + * `mongodb-memory-server` (#5517). + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { MongoDBDriver } from './mongodb-driver.js'; + +type Doc = Record; + +/** A fake `Db`: one collection, stored documents kept as inserted. */ +function fakeDb() { + const docs: Doc[] = []; + const project = (doc: Doc, projection: Record | undefined): Doc => { + const keep = Object.keys(projection ?? {}).filter((k) => projection![k] === 1); + const out: Doc = {}; + for (const [k, v] of Object.entries(doc)) { + if (k === '_id' && projection?._id === 0) continue; + if (keep.length > 0 && !keep.includes(k)) continue; + out[k] = v; + } + return out; + }; + // Equality on plain keys only — the reads below filter by `id` alone. A + // combinator or any other operator is REFUSED, never read as a field name. + const matches = (doc: Doc, filter: Doc | undefined): boolean => + Object.entries(filter ?? {}).every(([k, v]) => { + if (k.startsWith('$')) throw new Error(`fake Db: filter operator ${k} is not implemented`); + if (v !== null && typeof v === 'object') { + const ops = Object.keys(v as Doc); + if (ops.length !== 1 || ops[0] !== '$eq') throw new Error(`fake Db: operator ${ops.join(',')} is not implemented`); + return doc[k] === (v as Doc).$eq; + } + return doc[k] === v; + }); + const collection = { + insertOne: async (doc: Doc) => { docs.push({ _id: `oid_${docs.length}`, ...doc }); return { insertedId: `oid_${docs.length}` }; }, + findOne: async (filter: Doc, options: { projection?: Record }) => { + const d = docs.find((x) => matches(x, filter)); + return d ? project(d, options?.projection) : null; + }, + find: (filter: Doc, options: { projection?: Record }) => ({ + toArray: async () => docs.filter((x) => matches(x, filter)).map((d) => project(d, options?.projection)), + }), + }; + return { db: { collection: () => collection }, docs }; +} + +const OBJECT = 'nk_mongo_item'; + +const own = (row: unknown, key: string): boolean => + row != null && typeof row === 'object' && Object.prototype.hasOwnProperty.call(row, key); + +describe('[#29408] driver-mongodb: a selected, unwritten declared column reads null through the engine', () => { + let engine: ObjectQL; + let docs: Doc[]; + let id: string; + + beforeEach(async () => { + const driver = new MongoDBDriver({ url: 'mongodb://127.0.0.1:1/unused' }); + const fake = fakeDb(); + docs = fake.docs; + (driver as any).db = fake.db; + (driver as any).connect = async () => {}; + (driver as any).disconnect = async () => {}; + engine = new ObjectQL(); + engine.registerDriver(driver as never, true); + await engine.init(); + engine.registry.registerObject({ + name: OBJECT, + label: 'Item', + fields: { name: { type: 'text' }, note: { type: 'text' } }, + } as never); + const created = await engine.insert(OBJECT, { name: 'Ada' }) as Doc; + id = String(created.id); + }); + + it('the fixture is real: the stored document carries no key for the unwritten column', () => { + expect(docs).toHaveLength(1); + expect(own(docs[0], 'note')).toBe(false); + }); + + it('whole-record read and list read answer it null', async () => { + const one = await engine.findOne(OBJECT, { where: { id } }); + expect(own(one, 'note')).toBe(true); + expect(one!.note).toBeNull(); + const rows = await engine.find(OBJECT, {}); + expect(own(rows[0], 'note')).toBe(true); + expect(rows[0]!.note).toBeNull(); + }); + + it('a projection naming it answers it null', async () => { + const one = await engine.findOne(OBJECT, { where: { id }, fields: ['name', 'note'] }); + expect(own(one, 'note')).toBe(true); + expect(one!.note).toBeNull(); + }); + + it('NOT SELECTED stays absent: a projection that does not name it never answers it', async () => { + const one = await engine.findOne(OBJECT, { where: { id }, fields: ['name'] }); + expect(own(one, 'note')).toBe(false); + const rows = await engine.find(OBJECT, { fields: ['name'] }); + expect(own(rows[0], 'note')).toBe(false); + }); +}); diff --git a/packages/objectql/src/declared-read-columns.ts b/packages/objectql/src/declared-read-columns.ts index 6e1d7f92c49..7cb95364703 100644 --- a/packages/objectql/src/declared-read-columns.ts +++ b/packages/objectql/src/declared-read-columns.ts @@ -1,5 +1,8 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +import { unprovisionedInjectedColumns } from '@objectstack/spec/data'; +import { isFederatedObject } from './federated-object.js'; + /** * [#21571] The columns a read may serve: the object's DECLARED fields plus the * platform-provisioned columns — one set, answered from the registry's field @@ -129,3 +132,115 @@ export function rowsWithDeclaredColumnsOnly(rows: T[], declared: ReadonlySet< }); return changed ? shaped : rows; } + +/** + * [#29408] The STORED declared columns a read selected — the set a row is + * answered on, key for key, whatever the driver happened to return for it. + * + * ## The defect this closes + * + * A driver that stores only the columns a write touched (driver-mongodb drops + * a field given no value at its insert doors; driver-memory keeps stored + * `null` and a never-written key apart) answered such a field with NO key, + * while driver-sql, whose table has the column, answered `null`. One record, + * two shapes, by driver: a client predicate such as an action's `visible` + * reading `record.x` faulted on the missing key on one and evaluated on the + * other (follow-up D.1 of the #29094 ruling, record 6579342813). + * + * The read's persisted state is IN HAND here — the driver just answered for + * the row — so `null` is the honest value of a selected declared column the + * driver did not return, exactly the rule `materializeDeclaredFields` + * (`declared-fields.ts`) applies to every CEL binding. + * + * ## What is selected + * + * - an explicit projection selects the declared columns it names — the + * projection the DRIVER was handed, so a `beforeFind` hook's rewrite and the + * engine's own widening (formula inputs, a file field's owner key, each cut + * back off the answer before the verb returns) are what count; + * - no projection (a whole-record GET, a list read) selects every declared + * column. + * + * ## What stays absent, and why + * + * - a column the read did not select — a projection is the caller's own + * choice of keys, and filling the rest would answer what was not asked; + * - a `formula` field — virtual, no column on any driver (`fieldHasColumn` in + * driver-sql); `applyFormulaPlan` answers it after this, unchanged; + * - a name outside the declared set — the intersection below, so nothing this + * adds can widen what {@link withDeclaredColumnsOnly} serves; + * - an injected column a federated (ADR-0015 `external`) object does not + * provision (`unprovisionedInjectedColumns`) — there is no storage behind + * it, so there is no persisted state to answer for; + * - on a federated object, a {@link PLATFORM_PROVISIONED_COLUMNS} timestamp + * its field map does not declare (`systemFields: false` or + * `systemFields.audit: false`, so nothing injected it either). The three are + * "provisioned" because driver-sql creates them on every table IT builds, + * and the platform builds no table for a federated object + * (`isFederatedObject`): the remote holds such a timestamp only where the + * author mapped one, which is a declared field. `id` stays — the remote's + * own primary key backs it through the binding; + * - a column field-level security refuses — this runs inside the verb's + * middleware chain, on the rows as the driver returned them, and the + * security middleware's field mask deletes a refused key from the result + * AFTER the verb's body; a `null` added here is removed with it. + * + * Same "no opinion" cases as {@link declaredColumnSet}: no checkable field map + * ⇒ `undefined`, and the rows are left exactly as the driver returned them. + */ +export function selectedStoredColumns( + schema: { fields?: unknown } | null | undefined, + projection: readonly unknown[] | null | undefined, +): ReadonlySet | undefined { + const declared = declaredColumnSet(schema); + if (!declared) return undefined; + const fields = schema!.fields as Record; + const unprovisioned = new Set(unprovisionedInjectedColumns(schema)); + const federated = isFederatedObject(schema); + const stored = new Set(); + for (const name of declared) { + if (fields[name]?.type === 'formula' || unprovisioned.has(name)) continue; + // A platform-provisioned column the object does not declare has storage + // only on a table the platform built. + if (federated && name !== 'id' && !Object.prototype.hasOwnProperty.call(fields, name)) continue; + stored.add(name); + } + if (!Array.isArray(projection) || projection.length === 0) return stored; + const selected = new Set(); + for (const entry of projection) { + if (typeof entry === 'string' && stored.has(entry)) selected.add(entry); + } + return selected; +} + +/** + * The row with every {@link selectedStoredColumns} column it does not carry + * answered as `null` — a NEW object when there was anything to answer, the + * same reference otherwise. `undefined` counts as not carried (an own key + * holding `undefined` is what CEL and JSON both read as absent). Never + * mutates the row it is given, for the reason {@link withDeclaredColumnsOnly} + * gives. + */ +export function withSelectedColumnsAnswered(row: T, selected: ReadonlySet | undefined): T { + if (!selected || selected.size === 0 || !row || typeof row !== 'object' || Array.isArray(row)) return row; + const source = row as unknown as Record; + let shaped: Record | undefined; + for (const name of selected) { + if (source[name] !== undefined) continue; + shaped ??= { ...source }; + shaped[name] = null; + } + return (shaped ?? row) as unknown as T; +} + +/** {@link withSelectedColumnsAnswered} over a driver's result page. */ +export function rowsWithSelectedColumnsAnswered(rows: T[], selected: ReadonlySet | undefined): T[] { + if (!selected || selected.size === 0) return rows; + let changed = false; + const shaped = rows.map((row) => { + const next = withSelectedColumnsAnswered(row, selected); + if (next !== row) changed = true; + return next; + }); + return changed ? shaped : rows; +} diff --git a/packages/objectql/src/engine-formula-projection-trim.test.ts b/packages/objectql/src/engine-formula-projection-trim.test.ts index b38f7af428f..abd6f9e6bd6 100644 --- a/packages/objectql/src/engine-formula-projection-trim.test.ts +++ b/packages/objectql/src/engine-formula-projection-trim.test.ts @@ -187,7 +187,9 @@ describe.each(['find', 'findOne'])('#22300 — %s: a formula in the projec it('CONTROL: a read with no projection is unchanged — every declared column, the formula computed', async () => { const { engine, projections } = await boot(); const row = await readOne(engine, verb); - expect(row).toStrictEqual({ ...STORED, total_price: 21 }); + // [#29408] `queue` (registry-injected) is declared but absent from the + // stored row: a whole-record read answers it null. + expect(row).toStrictEqual({ ...STORED, queue: null, total_price: 21 }); expect(projections.at(-1)).toBeUndefined(); }); diff --git a/packages/objectql/src/engine.test.ts b/packages/objectql/src/engine.test.ts index c4cc39c07ef..04666dd12ed 100644 --- a/packages/objectql/src/engine.test.ts +++ b/packages/objectql/src/engine.test.ts @@ -1723,8 +1723,10 @@ describe('ObjectQL Engine', () => { const result = await engine.find('task', { expand: { assignee: { object: 'assignee' } } }); expect(result).toHaveLength(2); - expect(result[0].assignee).toEqual({ id: 'u1', name: 'Alice' }); - expect(result[1].assignee).toEqual({ id: 'u2', name: 'Bob' }); + // [#29408] The related record is answered on every declared column the + // read selected; the double's rows carry no timestamps, so they read null. + expect(result[0].assignee).toEqual({ id: 'u1', name: 'Alice', created_at: null, updated_at: null }); + expect(result[1].assignee).toEqual({ id: 'u2', name: 'Bob', created_at: null, updated_at: null }); // Verify the expand query used $in. The expand sub-read now routes // through the secured `find` path ([#2850]), so the referenced @@ -1768,7 +1770,8 @@ describe('ObjectQL Engine', () => { const result = await engine.find('ticket', { expand: { owner: { object: 'owner' } } }); - expect(result[0].owner).toEqual({ id: 'u1', name: 'Alice' }); + // [#29408] Unwritten declared columns of the related record read null. + expect(result[0].owner).toEqual({ id: 'u1', name: 'Alice', created_at: null, updated_at: null }); expect(mockDriver.find).toHaveBeenLastCalledWith( 'sys_user', expect.objectContaining({ where: { id: { $in: ['u1'] } } }), @@ -1825,7 +1828,8 @@ describe('ObjectQL Engine', () => { ); // Records the driver filtered out keep their raw FK id (unresolved). - expect(result[0].assignee).toEqual({ id: 'u1', name: 'Alice', active: true }); + // [#29408] Unwritten declared columns of the related record read null. + expect(result[0].assignee).toEqual({ id: 'u1', name: 'Alice', active: true, created_at: null, updated_at: null }); expect(result[1].assignee).toBe('u2'); }); @@ -2071,7 +2075,8 @@ describe('ObjectQL Engine', () => { ]); const result = await engine.find('order_item', { expand: { order: { object: 'order' } } }); - expect(result[0].order).toEqual({ id: 'o1', total: 100 }); + // [#29408] Unwritten declared columns of the related record read null. + expect(result[0].order).toEqual({ id: 'o1', total: 100, created_at: null, updated_at: null }); }); it('should skip expand for fields without reference definition', async () => { @@ -2558,7 +2563,8 @@ describe('ObjectQL Engine', () => { const result = await engine.find('task', { expand: { project: { object: 'project' } } }); // Project expanded, but org inside project remains as raw ID - expect(result[0].project).toEqual({ id: 'p1', org: 'o1' }); + // [#29408] Unwritten declared columns of the related record read null. + expect(result[0].project).toEqual({ id: 'p1', org: 'o1', created_at: null, updated_at: null }); expect(mockDriver.find).toHaveBeenCalledTimes(2); // Only primary + 1 expand query }); diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 93b1fa9ba76..e8fbc0af9ef 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -386,7 +386,10 @@ import { PLATFORM_PROVISIONED_COLUMNS, declaredColumnSet, rowsWithDeclaredColumnsOnly, + rowsWithSelectedColumnsAnswered, + selectedStoredColumns, withDeclaredColumnsOnly, + withSelectedColumnsAnswered, } from './declared-read-columns.js'; // [#21777] "Is this schema the remote's?" One predicate, shared with the boot sync. // [#21910, #21918] And its injected-column refinement, which both cascade walks ask. @@ -14619,6 +14622,13 @@ export class ObjectQL implements IObjectQLEngine { // declared record. See `declared-read-columns.ts`. if (Array.isArray(result)) { result = rowsWithDeclaredColumnsOnly(result, declaredColumnSet(_findSchema)); + // [#29408] …and answered on every stored declared column the read + // SELECTED: one the driver returned no key for (a driver that + // stores only written columns) is `null`, as driver-sql's column + // answers — the row's persisted state is in hand. Unselected, + // formula and FLS-refused columns stay absent; see + // `selectedStoredColumns`. + result = rowsWithSelectedColumnsAnswered(result, selectedStoredColumns(_findSchema, driverAst?.fields)); } // Post-process: evaluate formula virtual fields against the raw rows. @@ -14919,6 +14929,8 @@ export class ObjectQL implements IObjectQLEngine { // [#21571] Same default projection as `find`, same position: the // declared field set, applied to the row as the driver returned it. result = withDeclaredColumnsOnly(result, declaredColumnSet(_findOneSchema)); + // [#29408] Same answer on the selected stored columns as `find`. + result = withSelectedColumnsAnswered(result, selectedStoredColumns(_findOneSchema, driverAst?.fields)); // Post-process: evaluate formula virtual fields against the raw row // ([#20082] with the caller's permission map when a formula calls `can`). diff --git a/packages/objectql/src/federated-injected-column-readers.test.ts b/packages/objectql/src/federated-injected-column-readers.test.ts index 30e9972393b..2c788eeeeea 100644 --- a/packages/objectql/src/federated-injected-column-readers.test.ts +++ b/packages/objectql/src/federated-injected-column-readers.test.ts @@ -249,6 +249,14 @@ const READERS: Record = { disposition: 'excludes', why: 'the provenance the exclusion reads', }, + 'declared-read-columns.ts#selectedStoredColumns :: unprovisionedInjectedColumns()': { + disposition: 'excludes', + why: 'a read never answers an unprovisioned injected column as null: no storage stands behind it', + }, + 'declared-read-columns.ts#selectedStoredColumns :: isFederatedObject()': { + disposition: 'excludes', + why: 'a read never answers a platform timestamp a federated object does not declare as null: the platform builds no table for it', + }, // ── Seams that are not a storage read of a federated object's column ───── 'engine-referential-integrity.ts#assertReferencesResolve :: referenceTargetOf()': { diff --git a/packages/objectql/src/selected-declared-columns-answered.test.ts b/packages/objectql/src/selected-declared-columns-answered.test.ts new file mode 100644 index 00000000000..e07409c7b44 --- /dev/null +++ b/packages/objectql/src/selected-declared-columns-answered.test.ts @@ -0,0 +1,310 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#29408] A read answers every stored declared column it SELECTED: a column + * the driver returned no key for is `null`, as driver-sql's column answers. + * + * The driver here stores only the keys a write carried and answers a + * projection the way MongoDB does — a named key the document lacks is simply + * absent. That is the driver-mongodb / driver-memory shape (both measured + * through the engine on `origin/main`: no key for a never-written declared + * field, where driver-sql answers `null`), and the one this engine seam exists + * for. driver-mongodb is pinned in its own package + * (`mongodb-declared-column-null-on-read.test.ts`) and driver-sql in + * `@objectstack/runtime` (`declared-column-null-on-read.test.ts`); this file is + * the driver-memory shape's pin. + * + * What must stay absent is pinned beside what must appear: a column the read + * did not select, a column outside the declared set, a federated object's + * unprovisioned injected column, a platform timestamp a federated object + * neither declares nor has injected (`systemFields: false` / + * `systemFields.audit: false`), and a `formula` field (virtual, never + * `null`-filled itself; it is computed after the fill, so over an unwritten + * input it now evaluates against `null` instead of faulting). Field-level security's refusal is + * pinned on a real driver with the security plugin composed, in the runtime + * file named above. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; + +import { ObjectQL } from './engine.js'; +import { selectedStoredColumns, withSelectedColumnsAnswered } from './declared-read-columns.js'; + +type Row = Record; + +interface DriverAst { + where?: Record; + fields?: string[]; + limit?: number; +} + +function makeSparseDriver(name: string) { + const tables = new Map>(); + const tableFor = (o: string): Map => { + let t = tables.get(o); + if (!t) { t = new Map(); tables.set(o, t); } + return t; + }; + const matches = (row: Row, where: Record | undefined): boolean => { + if (!where) return true; + for (const [k, v] of Object.entries(where)) { + if (k.startsWith('$')) continue; + if ((row[k] ?? null) !== (v ?? null)) return false; + } + return true; + }; + const run = (object: string, ast: DriverAst | undefined): Row[] => { + let out = Array.from(tableFor(object).values()).filter((r) => matches(r, ast?.where)); + if (typeof ast?.limit === 'number') out = out.slice(0, ast.limit); + const fields = Array.isArray(ast?.fields) && ast.fields.length > 0 ? ast.fields : undefined; + // A projection keeps the named keys the stored document HAS — no key for + // one it lacks — plus `id`, as driver-mongodb projects. + return out.map((r) => (fields + ? Object.fromEntries(Object.entries(r).filter(([k]) => k === 'id' || fields.includes(k))) + : { ...r })); + }; + let seq = 0; + const driver = { + name, version: '0.0.0', supports: {}, + async connect(): Promise {}, + async disconnect(): Promise {}, + async checkHealth(): Promise { return true; }, + async execute(): Promise { return null; }, + async find(object: string, ast?: DriverAst): Promise { return run(object, ast); }, + async findOne(object: string, ast?: DriverAst): Promise { return run(object, ast)[0] ?? null; }, + async create(object: string, data: Row): Promise { + seq += 1; + const row: Row = { ...data, id: (data.id as string | undefined) ?? `new_${seq}` }; + tableFor(object).set(String(row.id), row); + return { ...row }; + }, + async update(object: string, id: string, data: Row): Promise { + const next: Row = { ...tableFor(object).get(id), ...data, id }; + tableFor(object).set(id, next); + return { ...next }; + }, + async delete(object: string, id: string): Promise { return tableFor(object).delete(id); }, + async count(object: string, ast?: DriverAst): Promise { return run(object, ast).length; }, + registerExternalObject() {}, + async syncSchema() {}, + }; + return { + driver, + seed: (object: string, row: Row) => { tableFor(object).set(String(row.id), { ...row }); }, + stored: (object: string, id: string) => tableFor(object).get(id), + }; +} + +const ITEM = 'nk_item'; +const REMOTE = 'nk_remote_customer'; +/** Federated, and opted out of the injected columns: nothing provisions a timestamp. */ +const REMOTE_BARE = 'nk_remote_bare'; +/** Federated, audit family off: no injected timestamp either. */ +const REMOTE_NO_AUDIT = 'nk_remote_no_audit'; +/** Federated, the author maps a remote `updated_at`: a declared column. */ +const REMOTE_MAPPED = 'nk_remote_mapped'; +const PACKAGE_ID = 'com.example.nk'; + +/** The fields of {@link ITEM}: `note` and `qty` are never written on `itm_1`. */ +const ITEM_FIELDS = { + name: { type: 'text' }, + note: { type: 'text' }, + qty: { type: 'number' }, + doubled: { type: 'formula', expression: { dialect: 'cel', source: 'record.qty != null ? record.qty * 2 : -1' } }, +}; + +async function boot() { + const engine = new ObjectQL(); + const local = makeSparseDriver('memory'); + const remote = makeSparseDriver('nk_external'); + engine.registerDriver(local.driver as never, true); + engine.registerDriver(remote.driver as never); + await engine.init(); + engine.registry.registerObject({ name: ITEM, label: 'Item', fields: ITEM_FIELDS } as never, PACKAGE_ID); + engine.registry.registerObject({ + name: REMOTE, + datasource: 'nk_external', + external: { remoteName: 'customers' }, + fields: { name: { type: 'text' }, email: { type: 'text' } }, + } as never, PACKAGE_ID); + // Only `name` written — every other declared column is a key the driver + // never stored. `id` and the two timestamps are stored the way a driver + // stamps them. + local.seed(ITEM, { id: 'itm_1', name: 'Ada', created_at: '2026-01-01T00:00:00.000Z', updated_at: '2026-01-01T00:00:00.000Z' }); + engine.registry.registerObject({ + name: REMOTE_BARE, + datasource: 'nk_external', + external: { remoteName: 'bare' }, + systemFields: false, + fields: { name: { type: 'text' }, email: { type: 'text' } }, + } as never, PACKAGE_ID); + engine.registry.registerObject({ + name: REMOTE_NO_AUDIT, + datasource: 'nk_external', + external: { remoteName: 'no_audit' }, + systemFields: { audit: false }, + fields: { name: { type: 'text' }, email: { type: 'text' } }, + } as never, PACKAGE_ID); + engine.registry.registerObject({ + name: REMOTE_MAPPED, + datasource: 'nk_external', + external: { remoteName: 'mapped' }, + systemFields: false, + fields: { name: { type: 'text' }, updated_at: { type: 'datetime' } }, + } as never, PACKAGE_ID); + remote.seed(REMOTE, { id: 'cus_1', name: 'Remote Co' }); + remote.seed(REMOTE_BARE, { id: 'bar_1', name: 'Bare Co' }); + remote.seed(REMOTE_NO_AUDIT, { id: 'noa_1', name: 'No Audit Co' }); + remote.seed(REMOTE_MAPPED, { id: 'map_1', name: 'Mapped Co' }); + return { engine, local, protocol: new ObjectStackProtocolImplementation(engine as never) }; +} + +const own = (row: unknown, key: string): boolean => + row != null && typeof row === 'object' && Object.prototype.hasOwnProperty.call(row, key); + +describe('[#29408] a selected declared column the driver did not return is answered null', () => { + let h: Awaited>; + beforeEach(async () => { h = await boot(); }); + + it('the fixture is real: the stored row carries no key for the unwritten columns', () => { + const stored = h.local.stored(ITEM, 'itm_1')!; + expect(own(stored, 'note')).toBe(false); + expect(own(stored, 'qty')).toBe(false); + }); + + it('a whole-record read (findOne, no projection) answers every unwritten stored column null', async () => { + const row = await h.engine.findOne(ITEM, { where: { id: 'itm_1' } }); + expect(row).toMatchObject({ id: 'itm_1', name: 'Ada', note: null, qty: null }); + expect(own(row, 'note')).toBe(true); + expect(own(row, 'qty')).toBe(true); + }); + + it('a list read (find, no projection) answers them null too', async () => { + const rows = await h.engine.find(ITEM, {}); + expect(rows).toHaveLength(1); + expect(own(rows[0], 'note')).toBe(true); + expect(rows[0]).toMatchObject({ note: null, qty: null }); + }); + + it('an explicit projection naming the column answers it null, on find and findOne', async () => { + const one = await h.engine.findOne(ITEM, { where: { id: 'itm_1' }, fields: ['name', 'note'] }); + expect(own(one, 'note')).toBe(true); + expect(one!.note).toBeNull(); + const rows = await h.engine.find(ITEM, { fields: ['name', 'note'] }); + expect(own(rows[0], 'note')).toBe(true); + expect(rows[0]!.note).toBeNull(); + }); + + it('NOT SELECTED stays absent: a projection that does not name the column never answers it', async () => { + const one = await h.engine.findOne(ITEM, { where: { id: 'itm_1' }, fields: ['name'] }); + expect(own(one, 'note')).toBe(false); + expect(own(one, 'qty')).toBe(false); + const rows = await h.engine.find(ITEM, { fields: ['name'] }); + expect(own(rows[0], 'note')).toBe(false); + expect(own(rows[0], 'qty')).toBe(false); + }); + + it('a formula field is never null-filled itself: over its unwritten input it evaluates against null', async () => { + const row = await h.engine.findOne(ITEM, { where: { id: 'itm_1' } }); + expect(row!.doubled).toBe(-1); + const narrowed = await h.engine.findOne(ITEM, { where: { id: 'itm_1' }, fields: ['name', 'doubled'] }); + // `qty` was read for the formula (the projection widening) and is cut off + // the answer again: widening is not selecting. + expect(narrowed!.doubled).toBe(-1); + expect(own(narrowed, 'qty')).toBe(false); + }); + + it('a stored value and a stored null are answered as stored', async () => { + await h.engine.update(ITEM, { id: 'itm_1', note: 'kept', qty: null } as never); + const row = await h.engine.findOne(ITEM, { where: { id: 'itm_1' } }); + expect(row).toMatchObject({ note: 'kept', qty: null }); + }); + + it('FEDERATED stays absent: an unprovisioned injected column of an external object is not answered', async () => { + const row = await h.engine.findOne(REMOTE, { where: { id: 'cus_1' } }); + // The declared remote column the driver did not return is answered… + expect(own(row, 'email')).toBe(true); + expect(row!.email).toBeNull(); + // …the platform's injected anchors, which have no storage behind them + // on a federated object, are not. + for (const injected of ['organization_id', 'owner_id', 'created_by', 'created_at']) { + expect(own(row, injected)).toBe(false); + } + }); + + it.each([ + ['systemFields: false', REMOTE_BARE, 'bar_1'], + ['systemFields.audit: false', REMOTE_NO_AUDIT, 'noa_1'], + ])('UNPROVISIONED TIMESTAMP stays absent: a federated object with %s is not answered created_at / updated_at', async (_label, object, id) => { + const one = await h.engine.findOne(object, { where: { id } }); + const rows = await h.engine.find(object, {}); + const named = await h.engine.findOne(object, { where: { id }, fields: ['name', 'created_at', 'updated_at'] }); + for (const row of [one, rows[0], named]) { + expect(own(row, 'created_at')).toBe(false); + expect(own(row, 'updated_at')).toBe(false); + // The remote's own primary key backs `id`. + expect(row!.id).toBe(id); + } + // The declared remote column the driver did not return is still answered. + expect(own(one, 'email')).toBe(true); + expect(one!.email).toBeNull(); + }); + + it('a timestamp the federated object DECLARES is a remote column, and is answered null', async () => { + const row = await h.engine.findOne(REMOTE_MAPPED, { where: { id: 'map_1' } }); + expect(own(row, 'updated_at')).toBe(true); + expect(row!.updated_at).toBeNull(); + expect(own(row, 'created_at')).toBe(false); + }); + + it('the doors: GET /data/:object/:id (getData) and the list route (findData) carry the null', async () => { + const got: any = await h.protocol.getData({ object: ITEM, id: 'itm_1' }); + expect(own(got.record, 'note')).toBe(true); + expect(got.record.note).toBeNull(); + const listed: any = await h.protocol.findData({ object: ITEM, query: {} }); + expect(own(listed.records[0], 'note')).toBe(true); + expect(listed.records[0].note).toBeNull(); + }); + + it('the driver\'s stored row is never mutated by the read', async () => { + await h.engine.find(ITEM, {}); + await h.engine.findOne(ITEM, { where: { id: 'itm_1' } }); + expect(own(h.local.stored(ITEM, 'itm_1'), 'note')).toBe(false); + }); +}); + +describe('[#29408] selectedStoredColumns — the set a row is answered on', () => { + const schema = { fields: ITEM_FIELDS }; + + it('no projection selects every stored declared column and the provisioned three, never a formula', () => { + expect([...selectedStoredColumns(schema, undefined)!].sort()).toEqual( + ['created_at', 'id', 'name', 'note', 'qty', 'updated_at'], + ); + }); + + it('a local object keeps the provisioned three whatever systemFields says: driver-sql builds them on every table', () => { + const optedOut = { systemFields: false, fields: { name: { type: 'text' } } }; + expect([...selectedStoredColumns(optedOut, undefined)!].sort()).toEqual( + ['created_at', 'id', 'name', 'updated_at'], + ); + }); + + it('OUTSIDE THE DECLARED SET stays absent: a projected name no field declares is never selected', () => { + expect([...selectedStoredColumns(schema, ['name', 'ghost', 'doubled'])!]).toEqual(['name']); + expect(withSelectedColumnsAnswered({ id: 'x' }, selectedStoredColumns(schema, ['ghost']))).toStrictEqual({ id: 'x' }); + }); + + it('no checkable field map is no opinion: the row is returned as given', () => { + const row = { id: 'x' }; + expect(selectedStoredColumns({ fields: {} }, undefined)).toBeUndefined(); + expect(withSelectedColumnsAnswered(row, selectedStoredColumns(undefined, undefined))).toBe(row); + }); + + it('an own key holding undefined is answered null; nothing to answer keeps the reference', () => { + const selected = selectedStoredColumns(schema, ['name', 'note'])!; + expect(withSelectedColumnsAnswered({ name: 'a', note: undefined }, selected)).toStrictEqual({ name: 'a', note: null }); + const full = { name: 'a', note: 'b' }; + expect(withSelectedColumnsAnswered(full, selected)).toBe(full); + }); +}); diff --git a/packages/plugins/plugin-audit/src/objects/sys-audit-log-record-views-columns.test.ts b/packages/plugins/plugin-audit/src/objects/sys-audit-log-record-views-columns.test.ts index 9cea192c108..922bee46a0d 100644 --- a/packages/plugins/plugin-audit/src/objects/sys-audit-log-record-views-columns.test.ts +++ b/packages/plugins/plugin-audit/src/objects/sys-audit-log-record-views-columns.test.ts @@ -21,8 +21,11 @@ * introspected directly — so this test runs the writer for real, against a * real engine, on a read shaped to make every conditionally-stamped key * present (a human principal that ALSO carries a service `actor` label, on a - * record that carries an `organization_id`), and reads the keys back off the - * row the writer actually persisted. If `buildRow` ever stops stamping a key + * record that carries an `organization_id`), and reads the keys off the row + * the writer actually persisted — the stub driver's STORED row, not an engine + * read of it: a read answers every declared column it selected, `null` where + * nothing was stored (#29408), so a read-back cannot tell a stamped key from + * an unwritten column. If `buildRow` ever stops stamping a key * this view lists, the observed key set shrinks and the assertion goes red — * no hand-kept list to fall out of sync with the writer it is supposed to * police. @@ -110,6 +113,10 @@ function makeStubDriver() { return []; }, async bulkDelete() {}, + /** The rows as stored — what the writer persisted, before any read shapes them. */ + storedRows(object: string): Record[] { + return Array.from(storeFor(object).values()).map((r) => ({ ...r })); + }, async updateMany() { return 0; }, @@ -139,7 +146,8 @@ let stampedKeys: Set; beforeAll(async () => { const engine = new ObjectQL(); - engine.registerDriver(makeStubDriver(), true); + const driver = makeStubDriver(); + engine.registerDriver(driver, true); await engine.init(); engine.registry.registerObject(contactObject as any, HARNESS_PACKAGE); // The REAL sys_audit_log object under test — not a hand-copied stand-in — @@ -165,9 +173,14 @@ beforeAll(async () => { }); await writer.flush(); + // The read still sees exactly the one row the writer persisted… const rows = (await engine.find('sys_audit_log', {})) as Array>; expect(rows).toHaveLength(1); - stampedKeys = new Set(Object.keys(rows[0])); + // …and the stamped keys are the STORED row's own: a read answers an + // unwritten declared column as `null` (#29408), which is not a stamp. + const stored = driver.storedRows('sys_audit_log'); + expect(stored).toHaveLength(1); + stampedKeys = new Set(Object.keys(stored[0])); }); /** The columns the shipped `record_views` list view declares. */ diff --git a/packages/runtime/src/declared-column-null-on-read.test.ts b/packages/runtime/src/declared-column-null-on-read.test.ts new file mode 100644 index 00000000000..3dff936bc21 --- /dev/null +++ b/packages/runtime/src/declared-column-null-on-read.test.ts @@ -0,0 +1,180 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#29408] One record reads the same on every driver: a declared column the + * read selected and the write never touched is `null`, not a missing key. + * + * Measured on `origin/main` before the fix, through the engine, a row inserted + * as `{ title }` on an object that also declares `note`: + * + * | read | driver-sql (SQLite) | driver-memory | + * |:-----------------------------|:--------------------|:--------------| + * | findOne, no projection | `note: null` | no `note` key | + * | findOne `fields` naming note | `note: null` | no `note` key | + * | findOne `fields` without it | no `note` key | no `note` key | + * | find, no projection | `note: null` | no `note` key | + * + * driver-mongodb answers as driver-memory does (its insert doors store no key + * for a field given no value); it is pinned in its own package, + * `mongodb-declared-column-null-on-read.test.ts`. driver-memory is not bound + * here: a new test consumer of it is a maintainer ruling + * (`check:driver-memory-census`), so its shape is pinned through the engine by + * the sparse driver in `@objectstack/objectql`'s + * `selected-declared-columns-answered.test.ts`, and the table's memory column + * was measured once, with a probe that is not kept. + * + * The engine answers the selected column `null` in one place + * (`selectedStoredColumns` in `@objectstack/objectql`'s + * `declared-read-columns.ts`), so every driver answers as SQL does. The security + * plugin is composed with a member who may not read `secret`: field-level + * security's refusal must keep the key ABSENT — a refused column must never + * reappear as `null`, which would tell the member the field exists and is + * empty. A system read of the same row is the control: there `secret` IS a + * selected, readable, unwritten column, and is answered `null`. + * + * Reads `@objectstack/objectql` from its built `dist/`, as every runtime suite + * does: rebuild objectql before trusting a run of this file. + */ + +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; +import { PermissionSetSchema } from '@objectstack/spec/security'; +import type { EngineQueryOptions } from '@objectstack/spec/data'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { SecurityPlugin } from '@objectstack/plugin-security'; +import { installDefaultOrganization } from './default-organization.testkit.js'; + +const LEDGER = 'nk_null_ledger'; +const ROW_ID = 'nk1'; + +const SYS_CTX = { isSystem: true, userId: 'usr_system' }; + +const MEMBER_SET = PermissionSetSchema.parse({ + name: 'member_default', + label: 'Member', + objects: { '*': { allowRead: true, allowCreate: true, allowEdit: true, allowDelete: true } }, + fields: { [`${LEDGER}.secret`]: { readable: false, editable: false } }, +}); + +const MEMBER_CTX = { userId: 'usr_member', positions: [] as string[], permissions: [MEMBER_SET.name], posture: 'MEMBER' as const }; + +type Ctx = EngineQueryOptions['context']; + +const own = (row: unknown, key: string): boolean => + row != null && typeof row === 'object' && Object.prototype.hasOwnProperty.call(row, key); + +/** + * The driver with the composition it serves: driver-sql under a real `single` + * install's Default Organization (its tenant column is NOT NULL). + */ +const DRIVERS: Array<[string, () => unknown, boolean]> = [ + ['driver-sql (SQLite)', () => new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true } as never), true], +]; + +describe.each(DRIVERS)('[#29408] a selected, unwritten declared column reads null — %s', (_label, makeDriver, withOrganization) => { + let engine: ObjectQL; + + beforeAll(async () => { + engine = new ObjectQL(); + engine.registerDriver(makeDriver() as never, true); + await engine.init(); + engine.registerApp({ + id: 'com.objectstack.qa.declared-column-null-on-read', + name: 'Declared column null on read', + version: '1.0.0', + type: 'plugin', + scope: 'system', + objects: [ + { + name: LEDGER, + label: 'Ledger', + sharingModel: 'public_read_write', + fields: { + title: { name: 'title', type: 'text' }, + note: { name: 'note', type: 'text' }, + secret: { name: 'secret', type: 'text' }, + }, + }, + ], + } as never); + await engine.syncSchemas(); + if (withOrganization) await installDefaultOrganization(engine); + + const services: Record = { + manifest: { register: vi.fn() }, + objectql: engine, + metadata: { + get: async (_type: string, name: string) => engine.getSchema(name) ?? null, + list: async () => [MEMBER_SET], + }, + }; + const ctx = { + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, + registerService: vi.fn(), + getService: (name: string) => { + if (!(name in services)) throw new Error(`service not registered: ${name}`); + return services[name]; + }, + }; + const plugin = new SecurityPlugin({ fallbackPermissionSet: 'member_default' }); + await plugin.init(ctx as never); + await plugin.start(ctx as never); + vi.spyOn((engine as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => undefined); + + // Only `title` is written: `note` and `secret` are never given a value. + await engine.insert(LEDGER, { id: ROW_ID, title: 'written' }, { context: SYS_CTX } as never); + }, 60_000); + + afterAll(async () => { + try { await engine?.destroy(); } catch { /* noop */ } + }); + + const byId = (context: Ctx, fields?: string[]): EngineQueryOptions => + ({ where: { id: ROW_ID }, ...(fields ? { fields } : {}), context }); + + it('whole-record read (findOne, no projection): the unwritten column is null', async () => { + const row = await engine.findOne(LEDGER, byId(MEMBER_CTX)); + expect(row).toMatchObject({ id: ROW_ID, title: 'written' }); + expect(own(row, 'note')).toBe(true); + expect(row!.note).toBeNull(); + }); + + it('list read (find, no projection): the unwritten column is null', async () => { + const rows = await engine.find(LEDGER, { context: MEMBER_CTX }); + expect(rows).toHaveLength(1); + expect(own(rows[0], 'note')).toBe(true); + expect(rows[0]!.note).toBeNull(); + }); + + it('a projection naming the column answers it null, on findOne and find', async () => { + const one = await engine.findOne(LEDGER, byId(MEMBER_CTX, ['title', 'note'])); + expect(own(one, 'note')).toBe(true); + expect(one!.note).toBeNull(); + const rows = await engine.find(LEDGER, { fields: ['title', 'note'], context: MEMBER_CTX }); + expect(own(rows[0], 'note')).toBe(true); + expect(rows[0]!.note).toBeNull(); + }); + + it('NOT SELECTED stays absent: a projection that does not name the column never answers it', async () => { + const one = await engine.findOne(LEDGER, byId(MEMBER_CTX, ['title'])); + expect(own(one, 'note')).toBe(false); + const rows = await engine.find(LEDGER, { fields: ['title'], context: MEMBER_CTX }); + expect(own(rows[0], 'note')).toBe(false); + }); + + it('FLS-REFUSED stays absent: a column the member may not read never reappears as null', async () => { + const whole = await engine.findOne(LEDGER, byId(MEMBER_CTX)); + expect(own(whole, 'secret')).toBe(false); + const listed = await engine.find(LEDGER, { context: MEMBER_CTX }); + expect(own(listed[0], 'secret')).toBe(false); + const named = await engine.findOne(LEDGER, byId(MEMBER_CTX, ['title', 'note', 'secret'])); + expect(own(named, 'secret')).toBe(false); + expect(own(named, 'note')).toBe(true); + }); + + it('control: the same column, read by the system, is a selected unwritten column and answers null', async () => { + const row = await engine.findOne(LEDGER, byId(SYS_CTX)); + expect(own(row, 'secret')).toBe(true); + expect(row!.secret).toBeNull(); + }); +});