From 0420214f52d7bd46a7a6ec224c7e0808e01e55a5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 10:52:40 +0000 Subject: [PATCH 1/7] fix(spec): FilterConditionSchema refuses on save every comparand slot the query faces refuse The save door asks the comparand-shape face about each slot of a field entry (read-only) and refuses what it refuses, in the schema door's words, plus a non-boolean $null / $exists flag, which every query face refuses. Claude-Session: https://claude.ai/code/session_01Rjy9MeetSfq34PKn81CRiN Co-authored-by: Claude --- packages/spec/src/data/filter.zod.ts | 313 +++++++++++++++++++++++---- 1 file changed, 273 insertions(+), 40 deletions(-) diff --git a/packages/spec/src/data/filter.zod.ts b/packages/spec/src/data/filter.zod.ts index 774debf8f6d..2925c497344 100644 --- a/packages/spec/src/data/filter.zod.ts +++ b/packages/spec/src/data/filter.zod.ts @@ -7,8 +7,11 @@ import { assertListComparandShapes } from './filter-comparand-shape'; // face so the save door and the query door print one sentence (ruling A, // record 5805248669: "one constant, two doors"). import { + IN_OPERATOR_SPELLINGS, + NIN_OPERATOR_SPELLINGS, arrayEqualityComparandMessage, arrayInequalityComparandMessage, + shapePreview, } from './filter-comparand-refusal-text'; import { normalizeFilterComparandTypes } from './filter-comparand-type'; import { bareDateRangePresetComparandMessage, isDateRangePresetName } from './date-range-presets'; @@ -312,8 +315,9 @@ const equalityComparandSchema = () => * * ⚠️ Scope: this is the OPERATOR slot. `FilterConditionSchema` (every stored * filter carrier) does not parse a field's operator map through - * `FieldOperatorsSchema`; its own walk, `checkFilterConditionComparands`, does - * not judge `$ne`, and the ruling names this slot, not that walk. + * `FieldOperatorsSchema`. [#20116] Its own walk, `checkFilterConditionComparands`, + * refuses the same shape by asking the comparand-shape face, and prints this + * sentence with the field named. * * ⚠️ `z.toJSONSchema()` has no arm for a custom check, so the published JSON * Schema still reads `{}` here. The site is declared in @@ -1652,11 +1656,253 @@ function isPlainFilterNode(value: unknown): value is Record { ); } +// ── [#20116] The query faces' comparand verdicts, asked at the save door ────── + +/** + * [#20116] Ask the comparand-shape face (`assertListComparandShapes`, + * `./filter-comparand-shape.ts`) about ONE comparand slot — a one-entry node + * holding either an implicit comparand (`{ stage: [...] }`) or a single operator + * (`{ stage: { $in: 'won' } }`). Returns the face's refusal, or `undefined` when + * the face accepts. + * + * The face is the JUDGE here, called read-only, so the save door refuses + * exactly the cells the query door refuses and no others: an arm the face gains + * later is refused on save the day it lands. It is handed one slot at a time + * because it throws on the first refusal it meets, and the save door reports + * every refused slot of a document, each at its own path. + * + * Only the face's own envelope (`INVALID_FILTER`) is read as a verdict. + * Anything else it throws is a defect in the face, not a refused filter, and is + * rethrown rather than reported as one. + */ +function comparandShapeFaceRefusal(slot: Record): Error | undefined { + try { + assertListComparandShapes(slot); + } catch (error) { + if ((error as { code?: unknown }).code === 'INVALID_FILTER') return error as Error; + throw error; + } + return undefined; +} + +/** `string` / `number` / `null` / `array` … — the face's `describeOperand`, for the two sentences below. */ +function describeComparandKind(value: unknown): string { + if (value === null) return 'null'; + if (value === undefined) return 'undefined'; + if (Array.isArray(value)) return 'array'; + if (value instanceof Date) return 'Date'; + return typeof value; +} + +/** + * [#20116] `$in` / `$nin` whose comparand is not a list — the face's sentence + * (`nonListComparandError`), less the ` at ` location only the face can + * write. The issue this door raises carries that location as its own `path`. + * The operator slot (`setMembershipSchema`) has only zod's generic wording for + * this shape, so the face's is the one sentence the platform has for it. + */ +function nonListComparandMessage(op: '$in' | '$nin', field: string, value: unknown): string { + const spellings = op === '$in' ? IN_OPERATOR_SPELLINGS : NIN_OPERATOR_SPELLINGS; + const alternative = op === '$in' ? '"=" ($eq)' : '"!=" ($ne)'; + return ( + `Operator "${op}" on field "${field}" requires an ARRAY of values. ` + + `Received ${describeComparandKind(value)} (${shapePreview(value)}). ` + + `"${op}" tests membership of a list — write ${shapePreview([value])} for a single value, ` + + `or use ${alternative} to compare against it. Authoring spellings: ${spellings.join(', ')}. ` + + 'The filter was NOT applied, and an unapplied filter would have returned the UNFILTERED ' + + 'result set.' + ); +} + +/** + * [#20116] `$between` whose comparand is not a two-element list — the face's + * sentence (`malformedRangeComparandError`), less its ` at ` location, for + * the reason {@link nonListComparandMessage} gives. + */ +function malformedRangeComparandMessage(field: string, value: unknown): string { + return ( + `Operator "$between" on field "${field}" requires a [min, max] value array. ` + + `Received ${describeComparandKind(value)} (${shapePreview(value)}). ` + + 'A range needs exactly two bounds, in order; the authoring spelling that lowers to ' + + '"$between" is "between". The filter was NOT applied, and an unapplied filter would have ' + + 'returned the UNFILTERED result set.' + ); +} + +/** The four ordering operators — the positions of the face's `null` ordering-comparand arm. */ +const ORDERING_OPERATORS: ReadonlySet = new Set(['$gt', '$gte', '$lt', '$lte']); + +/** Where a refusal sits below its operator (`[]`, or a member / endpoint index), and its words. */ +type SaveDoorRefusal = { readonly at: readonly number[]; readonly message: string }; + +/** + * [#20116] The save door's words for a slot the face refused. The VERDICT is + * the face's ({@link comparandShapeFaceRefusal}); this only picks the sentence, + * following the face's own order of checks so the sentence names the defect + * the face stopped at: + * + * - an array in the equality slot (implicit or `$eq`) and under `$ne` — the + * face's sentence, from the builder both doors import + * (`./filter-comparand-refusal-text.ts`); + * - a `null` ordering comparand, a `null` list member or `$between` endpoint, a + * blank endpoint and a `{ $field }` endpoint — the sentence the enforced + * operator slot (`FieldOperatorsSchema`) already prints for the same + * comparand, so one condition reads one way at the schema door; + * - a non-list `$in` / `$nin` and a `$between` that is not a pair — the face's + * sentence less its location, since no schema-door sentence exists for them. + * + * A refusal none of those arms recognises — an arm the face gained after this + * was written — is reported in the face's own words, location included, rather + * than accepted. `filter-save-door-face-parity.test.ts` fails on that text, so + * the new arm is worded here before it ships. + */ +function comparandShapeRefusalAtSave( + field: string, + op: string | undefined, + comparand: unknown, + face: Error, +): SaveDoorRefusal { + if (op === undefined) return { at: [], message: arrayEqualityComparandMessage(comparand, { field }) }; + if (op === '$eq') return { at: [], message: arrayEqualityComparandMessage(comparand, { op, field }) }; + if (op === '$ne') return { at: [], message: arrayInequalityComparandMessage(comparand, { field }) }; + if (comparand === null && ORDERING_OPERATORS.has(op)) { + return { at: [], message: nullOrderingComparandMessage(op) }; + } + if (op === '$in' || op === '$nin') { + if (!Array.isArray(comparand)) return { at: [], message: nonListComparandMessage(op, field, comparand) }; + const nullMember = comparand.indexOf(null); + if (nullMember !== -1) { + return { at: [nullMember], message: nullListComparandMemberMessage(`${op} member at index ${nullMember}`) }; + } + } + if (op === '$between') { + if (!Array.isArray(comparand) || comparand.length !== 2) { + return { at: [], message: malformedRangeComparandMessage(field, comparand) }; + } + const nullBound = comparand.indexOf(null); + if (nullBound !== -1) { + return { at: [nullBound], message: nullListComparandMemberMessage(`$between endpoint at index ${nullBound}`) }; + } + const blankBound = comparand.findIndex((bound) => bound === '' || bound === undefined); + if (blankBound === 0 || blankBound === 1) return { at: [blankBound], message: blankRangeBoundMessage(blankBound) }; + const referenceBound = comparand.findIndex(isFieldReferenceShape); + if (referenceBound !== -1) { + return { + at: [referenceBound], + message: listPositionFieldReferenceMessage(`$between endpoint at index ${referenceBound}`), + }; + } + } + return { at: [], message: face.message }; +} + +/** + * [#20116] The two flags `FieldOperatorsSchema` declares `z.boolean()`. A + * non-boolean one is refused on every query face — `driver-sql`, `driver-memory` + * and `driver-mongodb` (`nonBooleanNullComparandError`), the read-scope + * compiler and the analytics `where` door — under the #5347 / #5369 rulings: + * refused in every position, because the backends read one in opposite + * directions. The comparand-shape face does not judge flags, so this door + * judges them with the predicate every one of those faces uses: + * `typeof comparand !== 'boolean'`. + */ +const BOOLEAN_FLAG_OPERATORS: ReadonlySet = new Set(['$null', '$exists']); + +/** What arrived where a flag's boolean belongs — the analytics door's `describeFlagComparand`. */ +function describeFlagComparand(value: unknown): string { + if (value === null) return 'null'; + if (value === undefined) return 'undefined'; + if (typeof value === 'bigint') return `a bigint (${value}n)`; + if (Array.isArray(value)) return `an array (${shapePreview(value)})`; + if (value instanceof Date) return `a Date (${shapePreview(value)})`; + if (isFieldReferenceShape(value)) return `a field reference (${shapePreview(value)})`; + return `a ${typeof value} (${shapePreview(value)})`; +} + +/** + * [#20116] The refusal of a non-boolean `$null` / `$exists` flag, as the query + * faces give it. The first sentence is `driver-sql`'s word for word through + * "(true or false)", which the analytics `where` door also keeps; the reason and + * the prescription are the analytics door's, less the location and the history + * of what that door used to do. The field is named because this door can see + * it; the issue's own `path` carries the location. + */ +function nonBooleanFlagComparandMessage(op: string, field: string, value: unknown): string { + const [whenTrue, whenFalse] = op === '$null' ? ['has no value', 'has a value'] : ['has a value', 'has no value']; + return ( + `Operator "${op}" on field "${field}" requires a boolean comparand (true or false). ` + + `Received ${describeFlagComparand(value)}. @objectstack/spec FieldOperatorsSchema declares ` + + `${op} as a boolean, and a non-boolean is refused rather than coerced because the backends ` + + 'read one in OPPOSITE directions — one as IS NULL, another as IS NOT NULL. Write the ' + + `boolean itself: "${op}": true matches rows whose "${field}" ${whenTrue}, "${op}": false ` + + `rows whose "${field}" ${whenFalse}. The filter was NOT applied.` + ); +} + +/** + * [#20116] Raise, as `custom` issues under `slotPath`, every refusal the query + * faces give for ONE comparand slot, in this door's words: the comparand-shape + * face's verdict on the slot, and — for the two boolean flags, which that face + * does not judge — the flag rule. `op` is `undefined` for an implicit-equality + * comparand, and `slotPath` is then the field's own path. + */ +function reportQueryFaceRefusals( + ctx: z.RefinementCtx, + slotPath: (string | number)[], + field: string, + op: string | undefined, + comparand: unknown, +): void { + const refusals: SaveDoorRefusal[] = []; + const face = comparandShapeFaceRefusal(op === undefined ? { [field]: comparand } : { [field]: { [op]: comparand } }); + if (face) refusals.push(comparandShapeRefusalAtSave(field, op, comparand, face)); + if (op !== undefined && BOOLEAN_FLAG_OPERATORS.has(op) && typeof comparand !== 'boolean') { + refusals.push({ at: [], message: nonBooleanFlagComparandMessage(op, field, comparand) }); + } + for (const refusal of refusals) { + ctx.addIssue({ code: 'custom', path: [...slotPath, ...refusal.at], message: refusal.message }); + } +} + /** * Walk one condition node and report every comparand this authoring door * refuses — the bare date-range PRESET names in an ordering position (#8793), * the `$icontains` comparands the platform's own conformance table declares - * refused (#19514), and an ARRAY in the EQUALITY slot (#19889). + * refused (#19514), an ARRAY in the EQUALITY slot (#19889), and, since #20116, + * every comparand slot the query faces refuse. + * + * ## Every slot the query faces refuse is refused on save (#20116) + * + * The comparand-shape face (`assertListComparandShapes`) refuses, on every + * query, more than the equality slot: an array under `$ne` (ruling A on + * #19886), a `null` ordering comparand (2026-09-01), a non-list `$in` / `$nin` + * and a `$between` that is not a pair (#5869), a `null` list member or + * `$between` endpoint (2026-08-31), and a blank or `{ $field }` `$between` + * endpoint (#19071, #7596). Every query face refuses a non-boolean `$null` / + * `$exists` flag (#5347 / #5369). Measured on `origin/main` `af32cf9a` before + * this arm: `DatasetSchema`'s filter and measure filter, a dashboard widget + * `filter` and a report `runtimeFilter` each saved every one of them clean, + * while the face refused each shape with `INVALID_FILTER` / 400 — so a stored + * filter published and then failed every query built on it. + * + * - **The judge is the face itself**, asked about one slot at a time + * ({@link comparandShapeFaceRefusal}), so this door refuses exactly what the + * query door refuses and nothing else — `null` in the equality and `$ne` + * slots (the null predicate), a `{ $field }` reference as a whole comparand, + * `$in: []` / `$nin: []` and a whitespace endpoint all keep passing, because + * the face passes them. The flags, which that face does not judge, use the + * one predicate every flag face uses: `typeof comparand !== 'boolean'`. + * - **The words** are chosen by {@link comparandShapeRefusalAtSave}: the face's + * own sentence where the two doors already share a builder, the enforced + * operator slot's sentence where `FieldOperatorsSchema` already prints one for + * the same comparand, and the face's sentence less its location where neither + * door had one. None carries the face's ` at `; the issue's `path` does. + * - **The reach** is the face's, the equality arm's below: this node's own + * field entries and its `$and` / `$or` / `$not` members, and NOT a field spec + * with no `$` key. The drivers' flag checks stop at the same place. The + * analytics `where` door does descend a nested relation (it flattens one to a + * dotted member), so those positions belong to the analytics carriers' own + * refinement, never to this shared walk, which every other carrier reads. * * ## The equality-slot arm answers the FACE, in the face's words (#19889) * @@ -1683,12 +1929,11 @@ function isPlainFilterNode(value: unknown): value is Record { * door accepts, which is the split this arm exists to close. * - **Not dropped.** The refusal fails the parse; nothing is stripped from the * document. A dropped filter would show MORE rows than the author asked for. - * - ⛔ `$ne` is not judged by this walk (the ruling names equality), and the - * list operators keep their lists, `$in: []` / `$nin: []` included. [#19886] - * Ruling A refuses an array under `$ne` at the face and at the OPERATOR slot - * `FieldOperatorsSchema.$ne` (`inequalityComparandSchema`); it names neither - * this walk nor `FilterConditionSchema`, so a stored carrier still saves a - * `$ne` list and the face refuses it at query time. + * - The list operators keep their lists, `$in: []` / `$nin: []` included. + * [#20116] `$ne` carrying a list is refused on save too, since #20116, by the + * face-parity arm above in `arrayInequalityComparandMessage`'s words: the + * same reach as this arm and the sentence the face and + * `FieldOperatorsSchema.$ne` print (route A, the `$ne` member of #20116). * * Descends non-`$` keys only (operator specs and nested relations): the * `$and` / `$or` / `$not` members are re-parsed by {@link FilterConditionSchema} @@ -1740,22 +1985,16 @@ function checkFilterConditionComparands( for (const [key, value] of Object.entries(node)) { if (key.startsWith('$')) continue; // $and/$or/$not re-parse; other $ keys stay unjudged - // [#19889] The EQUALITY slot, implicit form `{ field: [...] }` — the empty - // list included. Judged on this node's OWN field entries only (`depth` 0), - // the face's exact reach: its walk never descends a field spec that has no - // `$` key, so an array inside a nested-relation condition is not refused - // there and is not refused here. See the docblock's third arm. - if (Array.isArray(value)) { - if (depth === 0) { - ctx.addIssue({ - code: 'custom', - path: [...path, key], - message: arrayEqualityComparandMessage(value, { field: key }), - }); - } + // [#19889, #20116] An IMPLICIT comparand — a scalar, a `Date` or an array + // (the equality slot's `{ field: [...] }`, the empty list included) — asked + // of the query faces. Judged on this node's OWN field entries only + // (`depth` 0), the face's exact reach: its walk never descends a field spec + // that has no `$` key, so nothing inside a nested-relation condition is + // refused there, and nothing is refused here. See the docblock. + if (!isPlainFilterNode(value)) { + if (depth === 0) reportQueryFaceRefusals(ctx, [...path, key], key, undefined, value); continue; } - if (!isPlainFilterNode(value)) continue; // a scalar implicit-equality comparand — not judged const hasOperatorKeys = Object.keys(value).some((k) => k.startsWith('$')); if (!hasOperatorKeys) { // Nested relation / deep equality — the schema does not re-parse these, @@ -1764,19 +2003,12 @@ function checkFilterConditionComparands( continue; } for (const [op, comparand] of Object.entries(value)) { - // [#19889] The EQUALITY slot, explicit form `{ field: { $eq: [...] } }`, - // on the same reach as the implicit form above. `null`, every scalar and a - // `{ $field }` reference are not arrays and pass. - if (op === '$eq') { - if (depth === 0 && Array.isArray(comparand)) { - ctx.addIssue({ - code: 'custom', - path: [...path, key, op], - message: arrayEqualityComparandMessage(comparand, { op: '$eq', field: key }), - }); - } - continue; - } + // [#20116] Every operator slot asked of the query faces, on the same + // reach as the implicit form above: the comparand-shape face's verdict + // (the equality and `$ne` slots, the ordering `null` carve-out, the list + // operators' shape, null-member and endpoint rules) and the boolean + // flags'. An operator neither judges passes through untouched. + if (depth === 0) reportQueryFaceRefusals(ctx, [...path, key, op], key, op, comparand); if (op === FILTER_TEXT_COMPARAND_OPERATOR && isRefusedTextComparand(comparand)) { ctx.addIssue({ code: 'custom', @@ -1927,7 +2159,7 @@ export const FilterConditionSchema: z.ZodType $or: z.array(FilterConditionSchema).optional(), $not: FilterConditionSchema.optional(), }) - // Three comparand refusals ride one walk — see its docblock for why. [#8793] + // The comparand refusals ride one walk — see its docblock for why. [#8793] // Bare date-range preset names are refused from ordering comparands (the // § 3.35 block above carries the ruling, the measured defect and the // ordering-only boundary); [#19514] `$icontains` comparands are refused on @@ -1935,9 +2167,10 @@ export const FilterConditionSchema: z.ZodType // stops admitting the document its own conformance table says will 400; // [#19889] an ARRAY in the equality slot, implicit or `$eq`, is refused in // the comparand-shape face's own words, so a stored filter the query door - // refuses is refused on save. The refinement judges this node's own field - // entries; `$and` / `$or` / `$not` members re-enter the schema and are - // judged by their own pass with nested issue paths. + // refuses is refused on save; [#20116] and so is every other slot the query + // faces refuse, the face itself deciding. The refinement judges this node's + // own field entries; `$and` / `$or` / `$not` members re-enter the schema and + // are judged by their own pass with nested issue paths. ).superRefine((node, ctx) => checkFilterConditionComparands(node, ctx)) ); From 231e4ebe29ee85ff614c2c03971b445acefdb745 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 11:08:34 +0000 Subject: [PATCH 2/7] test(spec): enumerate the save door against the query faces; register the ADR-0087 entry and changeset Claude-Session: https://claude.ai/code/session_01Rjy9MeetSfq34PKn81CRiN Co-authored-by: Claude --- .../20116-filter-save-door-face-parity.md | 57 +++ .../data/filter-save-door-face-parity.test.ts | 419 ++++++++++++++++++ packages/spec/src/data/filter.test.ts | 28 +- ...r-query-face-comparands-refused-at-save.ts | 83 ++++ 4 files changed, 576 insertions(+), 11 deletions(-) create mode 100644 .changeset/20116-filter-save-door-face-parity.md create mode 100644 packages/spec/src/data/filter-save-door-face-parity.test.ts create mode 100644 packages/spec/src/migrations/entries/semantic/18.filter-query-face-comparands-refused-at-save.ts diff --git a/.changeset/20116-filter-save-door-face-parity.md b/.changeset/20116-filter-save-door-face-parity.md new file mode 100644 index 00000000000..e3c9e54828a --- /dev/null +++ b/.changeset/20116-filter-save-door-face-parity.md @@ -0,0 +1,57 @@ +--- +"@objectstack/spec": minor +--- + +fix(spec)!: a filter carrying a comparand the query faces refuse is refused when it is saved (#20116) + +**BREAKING** — an accept-set narrowing of published authoring schemas, shipped as `minor` under the repo's launch-window convention for accept-set narrowings. The save door narrows to exactly what the query faces already refuse. The hand-migration prescription is registered under protocol major 18 as `filter-query-face-comparands-refused-at-save`. + +## What changes + +`FilterConditionSchema` now refuses, at parse, every comparand slot the query faces refuse: + +- a `$null` or `$exists` flag that is not a boolean — `"false"`, `"x"`, `null`, `1`; +- a `null` comparand of `$gt` / `$gte` / `$lt` / `$lte`; +- an `$in` or `$nin` comparand that is not a list, or a list holding `null`; +- a `$between` comparand that is not a two-element list, or whose endpoint is `null`, blank (`""`) or a `{ $field }` reference; +- an array under `$ne`. + +It judges the field entries of a condition and of every `$and` / `$or` / `$not` member. The shared comparand-shape face (`assertListComparandShapes`) is called read-only as the judge for each slot, so the save door refuses exactly what that face refuses on every query. The two boolean flags, which that face does not judge, are refused on the predicate every flag face uses (`driver-sql`, `driver-memory`, `driver-mongodb`, the read-scope compiler and the analytics `where` door): the comparand is not a boolean. + +Measured on `origin/main` `af32cf9a` before the change: a dataset `filter`, a dataset measure `filter`, a dashboard widget `filter` and a report `runtimeFilter` each parsed with `success: true` for one instance of every shape above. The comparand-shape face refused each one but the flags with `INVALID_FILTER` / 400, and the analytics `where` door refused all of them. So such a document published clean and then failed every chart built on it. + +Every schema that carries a `FilterCondition` refuses on parse. That covers the dataset `filter` and measure `filter`, the dashboard widget `filter` and options-source `filter`, the report and joined-report-block `runtimeFilter`, the field `relatedListFilter` and rollup `summaryOperations.filter`, the solution-blueprint summary `filter`, the analytics query `where`, the dataset selection `runtimeFilter`, the query `where` and `having`, the data-engine aggregate call's `having`, the aggregation `filter`, and the query-filter `where`. So `defineStack`, `os validate` and a save through the metadata protocol (`422 INVALID_METADATA`) refuse such a document at the slot's path, for example `filter.stage.$null` or `measures.0.filter.amount.$between.0`. + +The words are the query face's. For an array under `$ne`, a non-list `$in` / `$nin` and a malformed `$between`, the refusal is the face's sentence without its location clause (`at where..`), because the issue's path carries the location. For a `null` ordering comparand, a `null` list member or endpoint, and a blank or `{ $field }` endpoint, it is the sentence the enforced operator slot (`FieldOperatorsSchema`) already prints for the same comparand. A non-boolean flag gets the query faces' sentence: `Operator "$null" on field "stage" requires a boolean comparand (true or false).`, then the received value and the prescription. + +This also changes the `$ne` note of the equality-slot change earlier in this release: an array under `$ne` is now refused on save too, in the sentence `FieldOperatorsSchema.$ne` and the face print. + +## What does NOT change + +- **Nothing stored is rewritten, and nothing is dropped.** The parse fails and strips nothing. The read path does not re-validate stored rows, so a stored document keeps loading, and its next save is refused. Such a filter has failed every query since the runtime refusal of its shape, so the refusal is a repair. +- **The reach is the face's, and no wider.** A field spec with no `$` key, such as the nested-relation condition `{ account: { region: { $in: ["a", null] } } }`, is not judged, because neither the face nor the drivers' flag checks descend one. The analytics `where` door does refuse that shape when an analytics carrier is charted. +- **What the face passes still passes:** `$eq: null` and `$ne: null` (the null predicate), a `{ $field }` reference as the whole comparand of a scalar comparison, `$in: []` and `$nin: []`, a whitespace-only `$between` endpoint, and falsy endpoints such as `[0, 0]`. +- **The data-engine calls' `where` option still parses.** Its type is a union whose first arm is an open record. The face refuses the shape when the call runs. +- **No key, export or JSON Schema changes.** The published JSON Schema cannot state a refinement, and `FilterCondition`'s already could not state the equality-slot one. + +## FROM → TO + +| you wrote | write instead | +|:--|:--| +| `{ stage: { $null: "true" } }`, `{ stage: { $null: 1 } }` | `{ stage: { $null: true } }` ("has no value") | +| `{ stage: { $exists: "false" } }`, `{ stage: { $null: null } }` | the boolean you meant: `$exists: false` is "has no value", `$null: false` is "has a value" | +| `{ amount: { $gt: null } }` | `{ amount: { $eq: null } }` ("has no value") or `{ amount: { $ne: null } }` ("has a value") | +| `{ stage: { $in: "won" } }` | `{ stage: { $in: ["won"] } }` or `{ stage: "won" }` | +| `{ stage: { $in: ["won", null] } }` | `{ $or: [{ stage: { $in: ["won"] } }, { stage: { $null: true } }] }` | +| `{ amount: { $between: [null, 5] } }`, `{ amount: { $between: ["", 5] } }` | `{ amount: { $lte: 5 } }`, or the bound you meant | +| `{ amount: { $between: [{ $field: "floor" }, 5] } }` | `{ amount: { $gte: { $field: "floor" }, $lte: 5 } }` | +| `{ amount: { $between: 5 } }`, `{ amount: { $between: [1] } }` | `{ amount: { $between: [1, 5] } }` | +| `{ stage: { $ne: ["won", "lost"] } }` | `{ stage: { $nin: ["won", "lost"] } }` | + +## Who is affected, measured + +A literal-comparand scan of every member shape, with a lit control per shape, over `examples/**` and the non-test `packages/**` of this repository at `af32cf9a`, the console repository at its pinned commit `f8a9d0fb05`, and the cloud repository's `main` at `48d70663ab`, found authored filters carrying one in one place. The console's filter-condition widget writes "is empty" as `{ field: { $in: [null, ""] } }` and "is not empty" as `{ field: { $nin: [null, ""] } }`. That widget edits a field's `relatedListFilter` and a rollup's `summaryOperations.filter` in the Studio field designer, and a sharing rule's criteria. Both shapes carry a `null` list member, which the face has refused on every query since the 2026-08-31 ruling, so a filter saved that way has been failing its related list or rollup since then. After this change, the Studio save is refused instead, with the `$or` / `$null` prescription. Every other hit is prose, a type table or a test fixture. Deployed datasets, dashboards and reports were NOT measured. Validating each stack, or re-saving each document, finds every instance the surface above lists. + +Clause-②: no (narrowing) — nothing is widened. No key is added, removed or renamed, no exported symbol moves, and the operator vocabulary is unchanged. Comparand shapes that every query face already refused are now refused on save as well. + + diff --git a/packages/spec/src/data/filter-save-door-face-parity.test.ts b/packages/spec/src/data/filter-save-door-face-parity.test.ts new file mode 100644 index 00000000000..d918792166a --- /dev/null +++ b/packages/spec/src/data/filter-save-door-face-parity.test.ts @@ -0,0 +1,419 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20116] The SAVE door (`FilterConditionSchema`) refuses exactly the comparand + * slots the QUERY faces refuse — at the top level and in every `$and` / `$or` / + * `$not` member, and not inside a nested-relation condition, which the face + * never descends. + * + * Measured on `origin/main` `af32cf9a` before the change: `DatasetSchema` + * (filter and measure filter), a dashboard widget `filter` and a report + * `runtimeFilter` each answered `success: true` for `{ stage: { $null: 'x' } }`, + * `{ stage: { $exists: 'false' } }`, `{ stage: { $null: null } }`, + * `{ amount: { $gt: null } }`, `{ stage: { $in: 'won' } }`, + * `{ stage: { $in: ['won', null] } }`, `{ amount: { $between: [null, 5] } }` + * and `{ stage: { $ne: ['won', 'lost'] } }`, while `assertListComparandShapes` + * refused every one but the flags with `INVALID_FILTER` / 400 and every query + * face refuses the flags (#5347 / #5369). + * + * ## What would make these pins worthless, and what stops it + * + * - **A hand list of arms.** §1 derives the operators from + * `FieldOperatorsSchema`'s declared shape, and the face-judged cells by + * asking the face about every operator × comparand in a shape battery. A new + * operator, or a new face arm on any battery shape, lands in the table by + * itself, and the door has to answer it the face's way. + * - **A door that refuses everything.** §1 asserts EQUALITY of verdicts per + * cell, so an over-wide door is as red as a narrow one, and §4 runs the + * controls through both doors. + * - **A new face arm refused in the face's raw words.** The door falls back to + * the face's own message for an arm it has no sentence for; §2 fails on the + * face's location clause (` at where.`) in any door message. + */ + +import { describe, expect, it } from 'vitest'; + +import { StandardErrorCode } from '../api/errors.zod'; +import { DashboardSchema } from '../ui/dashboard.zod'; +import { DatasetSchema } from '../ui/dataset.zod'; +import { ReportSchema } from '../ui/report.zod'; +import { assertListComparandShapes } from './filter-comparand-shape'; +import { isRefusedTextComparand } from './filter-text-comparand'; +import { FieldOperatorsSchema, FilterConditionSchema } from './filter.zod'; + +type Issue = { code: string; path: PropertyKey[]; message: string }; +type Parsed = { success: boolean; error?: { issues: readonly Issue[] } }; + +/** The face's refusal of `where`, or `undefined` when it accepts. */ +function faceRefusal(where: unknown): (Error & { code?: string; status?: number }) | undefined { + try { + assertListComparandShapes(where); + } catch (error) { + return error as Error & { code?: string; status?: number }; + } + return undefined; +} + +/** Issues whose path starts with `prefix` (dot-joined). */ +function issuesUnder(result: Parsed, prefix: string): Issue[] { + if (result.success) return []; + return result.error!.issues.filter((i) => { + const joined = i.path.join('.'); + return joined === prefix || joined.startsWith(`${prefix}.`); + }); +} + +/** The one issue at exactly `path`, failing loudly on none or several. */ +function issueAt(result: Parsed, path: string): Issue { + expect(result.success, `expected a refusal at ${path}`).toBe(false); + const issues = result.error!.issues.filter((i) => i.path.join('.') === path); + expect(issues, `issues raised: ${JSON.stringify(result.error!.issues.map((i) => i.path))}`).toHaveLength(1); + return issues[0]!; +} + +/** Unwrap `.optional()` layers to the slot's own type name. */ +function slotType(schema: unknown): string | undefined { + let node = schema as { def?: { type?: string; innerType?: unknown } } | undefined; + while (node?.def?.type === 'optional') node = node.def.innerType as typeof node; + return node?.def?.type; +} + +// --------------------------------------------------------------------------- +// The table: every declared operator × a battery of comparand shapes +// --------------------------------------------------------------------------- + +/** The declared operator vocabulary — the enforced copy's own keys. */ +const OPERATORS = Object.keys((FieldOperatorsSchema as unknown as { shape: Record }).shape); + +/** The slots `FieldOperatorsSchema` declares `z.boolean()` — the flag arm's operators. */ +const BOOLEAN_SLOTS = OPERATORS.filter( + (op) => slotType((FieldOperatorsSchema as unknown as { shape: Record }).shape[op]) === 'boolean', +); + +const DAY = new Date('2026-07-01T00:00:00.000Z'); + +/** One comparand of every shape class a face arm keys on, and the neighbours it must not. */ +const BATTERY: ReadonlyArray = [ + ['a string', 'won'], + ['the string "false"', 'false'], + ['the empty string', ''], + ['a number', 5], + ['zero', 0], + ['true', true], + ['false', false], + ['null', null], + ['undefined', undefined], + ['a Date', DAY], + ['a { $field } reference', { $field: 'budget' }], + ['a plain object', { a: 1 }], + ['an empty list', []], + ['a one-member list', ['won']], + ['a pair', [1, 5]], + ['a whitespace pair', [' ', 'M']], + ['a triple', [1, 2, 3]], + ['a list holding null', ['won', null]], + ['a pair with a null MIN', [null, 5]], + ['a pair with a blank MIN', ['', 5]], + ['a pair with a { $field } MIN', [{ $field: 'a' }, 5]], + ['a list of one { $field }', [{ $field: 'a' }]], + ['a nested list', [[1]]], +]; + +/** What the query faces answer for one operator slot: the shape face, and the flag rule. */ +function queryFacesRefuse(op: string, comparand: unknown): boolean { + if (faceRefusal({ f: { [op]: comparand } })) return true; + return BOOLEAN_SLOTS.includes(op) && typeof comparand !== 'boolean'; +} + +/** The pre-existing `$icontains` arm (#19514), which judges at any depth and is not this card's. */ +function textArmRefuses(op: string, comparand: unknown): boolean { + return op === '$icontains' && isRefusedTextComparand(comparand); +} + +/** The positions the door must answer the face's way, and where the slot then sits. */ +const POSITIONS: ReadonlyArray) => unknown, prefix: string]> = [ + ['top level', (e) => e, 'f'], + ['an $and member', (e) => ({ $and: [{ g: 1 }, e] }), '$and.1.f'], + ['an $or member', (e) => ({ $or: [e] }), '$or.0.f'], + ['under $not', (e) => ({ $not: e }), '$not.f'], +]; + +describe('#20116 §1 — the enumeration: the save door refuses exactly what the query faces refuse', () => { + it('the table is derived, not hand-listed, and covers every arm the face and the flag rule judge', () => { + // The vocabulary is the enforced copy's, so a new operator joins the table. + expect(OPERATORS).toEqual(expect.arrayContaining(['$eq', '$ne', '$gt', '$in', '$nin', '$between', '$null', '$exists'])); + expect(BOOLEAN_SLOTS.sort()).toEqual(['$exists', '$null']); + // Every operator the face judges today refuses at least one battery shape — + // the guard against a battery that silently stopped reaching an arm. + const faceJudged = OPERATORS.filter((op) => BATTERY.some(([, c]) => faceRefusal({ f: { [op]: c } }))); + expect(faceJudged.sort()).toEqual(['$between', '$eq', '$gt', '$gte', '$in', '$lt', '$lte', '$ne', '$nin']); + }); + + for (const [position, wrap, prefix] of POSITIONS) { + it(`every operator × comparand cell, ${position}`, () => { + const mismatches: string[] = []; + let refused = 0; + for (const op of OPERATORS) { + for (const [label, comparand] of BATTERY) { + const expected = queryFacesRefuse(op, comparand) || textArmRefuses(op, comparand); + const got = issuesUnder(FilterConditionSchema.safeParse(wrap({ f: { [op]: comparand } })), `${prefix}.${op}`); + if (expected) refused += 1; + if ((got.length > 0) !== expected) { + mismatches.push(`${op} ← ${label}: faces ${expected ? 'REFUSE' : 'ACCEPT'}, door ${got.length > 0 ? 'REFUSE' : 'ACCEPT'}`); + } + } + } + expect(mismatches).toEqual([]); + // Guards the loop against passing vacuously. + expect(refused).toBeGreaterThan(40); + }); + } + + it('the implicit-equality slot, every battery shape that is a comparand there', () => { + for (const [label, comparand] of BATTERY) { + if (comparand !== null && typeof comparand === 'object' && !Array.isArray(comparand) && !(comparand instanceof Date)) { + continue; // a plain object in this slot is a nested condition, not a comparand + } + const expected = faceRefusal({ f: comparand }) !== undefined; + const got = issuesUnder(FilterConditionSchema.safeParse({ f: comparand }), 'f'); + expect(got.length > 0, label).toBe(expected); + } + }); + + it('inside a nested-relation condition the door answers as the face does — it never descends one', () => { + for (const op of OPERATORS) { + for (const [label, comparand] of BATTERY) { + const where = { acct: { f: { [op]: comparand } } }; + // The face leaves the relation alone, whatever sits in it. + expect(faceRefusal(where), `${op} ← ${label}`).toBeUndefined(); + const got = issuesUnder(FilterConditionSchema.safeParse(where), `acct.f.${op}`); + // Only the #19514 text arm, which walks nested relations on purpose, may fire here. + expect(got.length > 0, `${op} ← ${label}`).toBe(textArmRefuses(op, comparand)); + } + } + }); +}); + +// --------------------------------------------------------------------------- +// §2 The envelope and the words, per arm +// --------------------------------------------------------------------------- + +describe('#20116 §2 — each refusal: issue code, path and the prescription', () => { + /** The face's message for `where`, with its ` at .` location replaced by `.`. */ + function faceSentenceWithoutLocation(where: unknown, facePath: string): string { + const face = faceRefusal(where); + expect(face, `the face accepted ${JSON.stringify(where)}`).toBeDefined(); + expect(face!.code).toBe(StandardErrorCode.enum.INVALID_FILTER); + expect(face!.status).toBe(400); + const location = ` at ${facePath}.`; + // The clause is really there, exactly once, so removing it is not vacuous. + expect(face!.message.split(location)).toHaveLength(2); + return face!.message.replace(location, '.'); + } + + it.each([ + ['a non-list $in', { stage: { $in: 'won' } }, 'stage.$in', 'where.stage.$in'], + ['a non-list $nin', { stage: { $nin: 7 } }, 'stage.$nin', 'where.stage.$nin'], + ['a scalar $between', { amount: { $between: 5 } }, 'amount.$between', 'where.amount.$between'], + ['a one-bound $between', { amount: { $between: [1] } }, 'amount.$between', 'where.amount.$between'], + ['a three-bound $between', { amount: { $between: [1, 2, 3] } }, 'amount.$between', 'where.amount.$between'], + ['a $ne list — the appended member', { stage: { $ne: ['won', 'lost'] } }, 'stage.$ne', 'where.stage.$ne'], + ['an empty $ne list', { stage: { $ne: [] } }, 'stage.$ne', 'where.stage.$ne'], + ['an implicit list (#19889, unchanged)', { stage: ['won'] }, 'stage', 'where.stage'], + ['a $eq list (#19889, unchanged)', { stage: { $eq: ['won'] } }, 'stage.$eq', 'where.stage.$eq'], + ])('%s — the face\'s sentence, less its location', (_label, where, issuePath, facePath) => { + const issue = issueAt(FilterConditionSchema.safeParse(where), issuePath); + expect(issue.code).toBe('custom'); + expect(issue.message).toBe(faceSentenceWithoutLocation(where, facePath)); + }); + + it('the $in prescription names the list spelling, the equality alternative and the authoring spelling', () => { + const { message } = issueAt(FilterConditionSchema.safeParse({ stage: { $in: 'won' } }), 'stage.$in'); + expect(message).toMatch(/^Operator "\$in" on field "stage" requires an ARRAY of values\. Received string \("won"\)\. /); + expect(message).toContain('write ["won"] for a single value, or use "=" ($eq) to compare against it'); + expect(message).toContain('Authoring spellings: in.'); + expect(message).not.toContain(' at where.'); + }); + + it('the $ne prescription is $nin, and the field is named', () => { + const { message } = issueAt(FilterConditionSchema.safeParse({ stage: { $ne: ['won', 'lost'] } }), 'stage.$ne'); + expect(message).toMatch(/^Operator "\$ne" on field "stage" requires a single comparable value, but received an array/); + expect(message).toContain('{"$nin": […]}'); + expect(message).not.toContain('{"$in": […]}'); + }); + + it.each([ + ['$gt: null', { amount: { $gt: null } }, 'amount.$gt', { $gt: null }, '$gt'], + ['$gte: null', { amount: { $gte: null } }, 'amount.$gte', { $gte: null }, '$gte'], + ['$lt: null', { amount: { $lt: null } }, 'amount.$lt', { $lt: null }, '$lt'], + ['$lte: null', { amount: { $lte: null } }, 'amount.$lte', { $lte: null }, '$lte'], + ['a null $in member', { stage: { $in: ['won', null] } }, 'stage.$in.1', { $in: ['won', null] }, '$in.1'], + ['a null $nin member', { stage: { $nin: [null] } }, 'stage.$nin.0', { $nin: [null] }, '$nin.0'], + ['a null $between MIN', { amount: { $between: [null, 5] } }, 'amount.$between.0', { $between: [null, 5] }, '$between.0'], + ['a null $between MAX', { amount: { $between: [1, null] } }, 'amount.$between.1', { $between: [1, null] }, '$between.1'], + ['a blank $between MIN', { amount: { $between: ['', 5] } }, 'amount.$between.0', { $between: ['', 5] }, '$between.0'], + ['a { $field } $between MAX', { amount: { $between: [1, { $field: 'b' }] } }, 'amount.$between.1', { $between: [1, { $field: 'b' }] }, '$between.1'], + ])('%s — the enforced operator slot\'s sentence, at the member or endpoint', (_label, where, issuePath, slotInput, slotPath) => { + // The face refuses it on every query… + const face = faceRefusal(where); + expect(face?.code).toBe(StandardErrorCode.enum.INVALID_FILTER); + expect(face?.status).toBe(400); + // …and the save door refuses it at the slot, in the words + // `FieldOperatorsSchema` already prints for the same comparand. + const issue = issueAt(FilterConditionSchema.safeParse(where), issuePath); + expect(issue.code).toBe('custom'); + expect(issue.message).toBe(issueAt(FieldOperatorsSchema.safeParse(slotInput), slotPath).message); + }); + + it('the null-ordering and null-member prescriptions are the null predicate', () => { + expect(issueAt(FilterConditionSchema.safeParse({ amount: { $gt: null } }), 'amount.$gt').message) + .toContain('{"$eq": null} is "has no value", {"$ne": null} is "has a value"'); + expect(issueAt(FilterConditionSchema.safeParse({ stage: { $in: ['won', null] } }), 'stage.$in.1').message) + .toContain('{"$or": [{"$in": […]}, {"$null": true}]}'); + }); + + it.each([ + ['$null: "x"', { stage: { $null: 'x' } }, 'stage.$null', 'a string ("x")'], + ['$exists: "false" — the string, truthy', { stage: { $exists: 'false' } }, 'stage.$exists', 'a string ("false")'], + ['$null: null', { stage: { $null: null } }, 'stage.$null', 'null'], + ['$exists: 1', { stage: { $exists: 1 } }, 'stage.$exists', 'a number (1)'], + ['$null: an array', { stage: { $null: [true] } }, 'stage.$null', 'an array ([true])'], + ])('a non-boolean flag, %s — the query faces\' sentence and prescription', (_label, where, issuePath, received) => { + const op = issuePath.split('.').pop()!; + const issue = issueAt(FilterConditionSchema.safeParse(where), issuePath); + expect(issue.code).toBe('custom'); + // `driver-sql`'s first sentence, word for word, which the analytics door keeps too. + expect(issue.message).toMatch( + new RegExp(`^Operator "\\${op}" on field "stage" requires a boolean comparand \\(true or false\\)\\. `), + ); + expect(issue.message).toContain(`Received ${received}.`); + expect(issue.message).toContain(`@objectstack/spec FieldOperatorsSchema declares ${op} as a boolean`); + const [whenTrue, whenFalse] = op === '$null' ? ['has no value', 'has a value'] : ['has a value', 'has no value']; + expect(issue.message).toContain( + `Write the boolean itself: "${op}": true matches rows whose "stage" ${whenTrue}, "${op}": false rows whose "stage" ${whenFalse}.`, + ); + expect(issue.message.length).toBeLessThan(500); + }); + + it('no door message carries the face\'s location — every face arm has a save-door sentence', () => { + for (const op of OPERATORS) { + for (const [, comparand] of BATTERY) { + const result = FilterConditionSchema.safeParse({ f: { [op]: comparand } }); + for (const issue of issuesUnder(result, `f.${op}`)) { + expect(issue.message, `${op} ← ${JSON.stringify(comparand)}`).not.toContain(' at where.'); + } + } + } + }); + + it('every refused slot of one document is reported, each at its own path', () => { + const result = FilterConditionSchema.safeParse({ + stage: { $null: 'x', $in: 'won' }, + amount: { $gt: null, $between: [null, 5] }, + $or: [{ owner: { $ne: ['a'] } }], + }); + expect(result.success).toBe(false); + expect(result.error!.issues.map((i) => i.path.join('.')).sort()).toEqual([ + '$or.0.owner.$ne', + 'amount.$between.0', + 'amount.$gt', + 'stage.$in', + 'stage.$null', + ]); + }); +}); + +// --------------------------------------------------------------------------- +// §3 The stored carriers refuse on save, at their own paths +// --------------------------------------------------------------------------- + +describe('#20116 §3 — the stored carriers', () => { + const dataset = (extra: Record) => ({ + name: 'deals_ds', + label: 'Deals', + object: 'deal', + dimensions: [{ name: 'stage', field: 'stage', type: 'string' }], + measures: [{ name: 'deal_count', aggregate: 'count' }], + ...extra, + }); + const dashboard = (filter: unknown) => ({ + name: 'sales', + label: 'Sales', + widgets: [{ id: 'won_deals', type: 'metric', dataset: 'deals', values: ['total'], filter }], + }); + const report = (runtimeFilter: unknown) => ({ + name: 'pipeline', label: 'Pipeline', type: 'summary', + dataset: 'sales', rows: ['stage'], values: ['revenue'], runtimeFilter, + }); + + /** The collector's members, each with the slot path it is refused at. */ + const MEMBERS: ReadonlyArray, slot: string]> = [ + [{ stage: { $null: 'x' } }, 'stage.$null'], + [{ stage: { $exists: 'false' } }, 'stage.$exists'], + [{ stage: { $null: null } }, 'stage.$null'], + [{ amount: { $gt: null } }, 'amount.$gt'], + [{ stage: { $in: 'won' } }, 'stage.$in'], + [{ stage: { $in: ['won', null] } }, 'stage.$in.1'], + [{ amount: { $between: [null, 5] } }, 'amount.$between.0'], + [{ stage: { $ne: ['won', 'lost'] } }, 'stage.$ne'], + ]; + + it.each(MEMBERS)('%j — refused by the dataset filter, a measure filter, a widget filter and a report runtimeFilter', (where, slot) => { + const expected = issueAt(FilterConditionSchema.safeParse(where), slot).message; + expect(issueAt(DatasetSchema.safeParse(dataset({ filter: where })), `filter.${slot}`).message).toBe(expected); + const measure = dataset({ measures: [{ name: 'deal_count', aggregate: 'count', filter: where }] }); + expect(issueAt(DatasetSchema.safeParse(measure), `measures.0.filter.${slot}`).message).toBe(expected); + expect(issueAt(DashboardSchema.safeParse(dashboard(where)), `widgets.0.filter.${slot}`).message).toBe(expected); + expect(issueAt(ReportSchema.safeParse(report(where)), `runtimeFilter.${slot}`).message).toBe(expected); + }); + + it('CONTROL — the same carriers publish the boolean flags and keep them', () => { + for (const where of [{ stage: { $null: true } }, { stage: { $null: false } }, { stage: { $exists: true } }, { stage: { $exists: false } }]) { + const parsed = DatasetSchema.safeParse(dataset({ filter: where })); + expect(parsed.success, JSON.stringify(parsed.error?.issues)).toBe(true); + expect(parsed.data!.filter).toEqual(where); + } + }); +}); + +// --------------------------------------------------------------------------- +// §4 CONTROLS — what the face judges but passes stays accepted at BOTH doors +// --------------------------------------------------------------------------- + +describe('#20116 §4 — what stays accepted, at both doors', () => { + it.each([ + ['$null: true', { stage: { $null: true } }], + ['$null: false', { stage: { $null: false } }], + ['$exists: true', { stage: { $exists: true } }], + ['$exists: false', { stage: { $exists: false } }], + ['$eq: null — the has-no-value predicate', { stage: { $eq: null } }], + ['$ne: null — the has-a-value predicate', { stage: { $ne: null } }], + ['$ne: a scalar', { stage: { $ne: 'lost' } }], + ['$eq: a { $field } reference', { amount: { $eq: { $field: 'budget' } } }], + ['$ne: a { $field } reference', { amount: { $ne: { $field: 'budget' } } }], + ['$gt: a { $field } reference', { amount: { $gt: { $field: 'budget' } } }], + ['$lte: a { $field } reference', { amount: { $lte: { $field: 'budget' } } }], + ['a column-to-column range as two bounds', { amount: { $gte: { $field: 'a' }, $lte: { $field: 'b' } } }], + ['$gt: a Date', { closed_at: { $gt: DAY } }], + ['$gte: an ISO day', { closed_at: { $gte: '2026-01-01' } }], + ['$in: [] — matches nothing', { stage: { $in: [] } }], + ['$nin: [] — matches everything', { stage: { $nin: [] } }], + ['$in keeps its list', { stage: { $in: ['won', 'lost'] } }], + ['$in: a { $field } member — the face does not judge members', { stage: { $in: [{ $field: 'x' }] } }], + ['$between keeps its pair', { amount: { $between: [1, 9] } }], + ['$between: a whitespace endpoint — the schema door never judged one', { code: { $between: [' ', 'M'] } }], + ['$between: falsy endpoints are endpoints', { amount: { $between: [0, 0] } }], + ['relation traversal with no comparand operator', { acct: { region: 'NA' } }], + ['a member shape INSIDE a nested relation — the face never descends one', { acct: { stage: { $in: ['won', null] } } }], + ['a flag INSIDE a nested relation — the drivers never descend one either', { acct: { stage: { $null: 'x' } } }], + ['every one of the above under $and / $or / $not', { + $and: [{ stage: { $ne: null } }], $or: [{ amount: { $gt: { $field: 'b' } } }], $not: { stage: { $in: [] } }, + }], + ])('%s', (_label, where) => { + expect(faceRefusal(where)).toBeUndefined(); + const result = FilterConditionSchema.safeParse(where); + expect(result.success, JSON.stringify(result.error?.issues)).toBe(true); + // Accepted means KEPT: the door returns the document it was given. + expect(result.data).toEqual(where); + }); +}); diff --git a/packages/spec/src/data/filter.test.ts b/packages/spec/src/data/filter.test.ts index 4c02c4caa5b..b007454a56b 100644 --- a/packages/spec/src/data/filter.test.ts +++ b/packages/spec/src/data/filter.test.ts @@ -669,18 +669,24 @@ describe('RangeOperatorSchema', () => { }); /** - * `FilterConditionSchema` is `z.record(z.string(), z.unknown())` at every - * field position, so it judges no comparand at all — measured here against - * the ALREADY-RULED `{ $field }` endpoint (#7596), which it also lets - * through. That control is the point: the green below is this schema's - * standing shape and NOT a hole this narrowing opened, and the enforcement - * lives where it always did (`FieldOperatorsSchema` / the normalized AST). + * [#20116] `FilterConditionSchema` is `z.record(z.string(), z.unknown())` at + * every field position, and until #20116 it judged neither endpoint — the + * blank one nor the ALREADY-RULED `{ $field }` one (#7596) — while the + * comparand-shape face refused both on every query. Its walk now asks the + * face about each slot, so both are refused on save, at the endpoint, in the + * sentence the enforced operator slot prints for the same pair. */ - it('is not judged by the loose FilterConditionSchema — and neither is the #7596 shape', () => { - expect(FilterConditionSchema.safeParse({ age: { $between: [18, ''] } }).success).toBe(true); - expect(FilterConditionSchema.safeParse({ - age: { $between: [18, { $field: 'cap' }] }, - }).success).toBe(true); + it('is refused by FilterConditionSchema too, at the endpoint, in the operator slot\'s words (#20116)', () => { + const blank = FilterConditionSchema.safeParse({ age: { $between: [18, ''] } }); + expect(blank.success).toBe(false); + expect(issuesOf(blank).map((i) => i.path)).toEqual([['age', '$between', 1]]); + expect(issuesOf(blank)[0]?.message) + .toBe(issuesOf(FieldOperatorsSchema.safeParse({ $between: [18, ''] }))[0]?.message); + expect(issuesOf(blank)[0]?.message).toContain('the MAX bound'); + const reference = FilterConditionSchema.safeParse({ age: { $between: [18, { $field: 'cap' }] } }); + expect(reference.success).toBe(false); + expect(issuesOf(reference).map((i) => i.path)).toEqual([['age', '$between', 1]]); + expect(issuesOf(reference)[0]?.message).toContain('A { "$field": … } reference is not a valid $between endpoint at index 1'); }); it('narrows the blank endpoint and NOTHING wider — the falsy and short values stay', () => { diff --git a/packages/spec/src/migrations/entries/semantic/18.filter-query-face-comparands-refused-at-save.ts b/packages/spec/src/migrations/entries/semantic/18.filter-query-face-comparands-refused-at-save.ts new file mode 100644 index 00000000000..05929426aa7 --- /dev/null +++ b/packages/spec/src/migrations/entries/semantic/18.filter-query-face-comparands-refused-at-save.ts @@ -0,0 +1,83 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import type { SemanticMigration } from '../../types.js'; + +// The SCHEMA door's half of every comparand refusal the query faces already +// give. The shared comparand-shape face and the query faces' boolean-flag checks +// refuse these slots on every query; this entry records that the same slots are +// now refused where a filter is SAVED, so a stored filter stops publishing clean +// and failing later for someone else. The face itself is the judge at the save +// door, so the two doors cannot drift apart. +export const entry: SemanticMigration = { + id: 'filter-query-face-comparands-refused-at-save', + // No backticks in `surface` — build-upgrade-guide renders it inside a code + // span already, and a nested backtick would close it. + surface: + 'data.FilterCondition — every comparand slot the query faces refuse, now refused when the ' + + 'document is PARSED: a $null or $exists flag that is not a boolean (a string such as ' + + '"false", null, a number); a null $gt / $gte / $lt / $lte comparand; an $in or $nin ' + + 'comparand that is not a list, or a list holding null; a $between comparand that is not a ' + + 'two-element list, or whose endpoint is null, blank or a { $field } reference; and an ' + + 'array under $ne. On every schema that carries a FilterCondition: a dataset filter and a ' + + 'dataset measure filter, a dashboard widget filter and an options-source filter, a report ' + + 'and joined-report-block runtimeFilter, a field relatedListFilter and a rollup ' + + 'summaryOperations filter, a solution-blueprint summary filter, an analytics query where, ' + + 'a dataset selection runtimeFilter, a query where and having, the data-engine aggregate ' + + 'call\'s having, an aggregation filter and a query-filter where', + replacement: + 'the spelling the refusal prescribes, which is the one the query faces already prescribe. ' + + 'A flag is the boolean itself: $null true is "has no value", $null false is "has a value", ' + + 'and $exists is the inverse. Absence is the null predicate, never null in an ordering or ' + + 'list position: $eq null is "has no value", $ne null is "has a value", and "one of these ' + + 'values OR has no value" is an $or of an $in and a $null true. A single value for $in is a ' + + 'one-member list, or plain equality. A range is two bounds in a two-element list; a range ' + + 'bounded on one side is a $gte or a $lte; a column-to-column range is a $gte and a $lte ' + + 'whose comparands are { $field } references. "None of these values" is $nin, never $ne ' + + 'with a list. The null predicate itself, a { $field } reference as a whole comparand, ' + + 'an empty $in or $nin list and a whitespace endpoint are untouched', + reason: + 'The save door narrows to exactly what the query faces already refuse (#20116, the ' + + 'collector for its family; the $ne member is route A, the same reach and the same one ' + + 'sentence as the equality slot of filter-equality-array-comparand-refused-at-save). The ' + + 'shared comparand-shape face refuses on every query a null ordering comparand (ruled ' + + '2026-09-01), a non-list $in / $nin and a malformed $between range, a null list member or ' + + 'endpoint (ruled 2026-08-31), a blank endpoint (ruled 2026-09-20), a { $field } endpoint ' + + '(ruled 2026-08-11) and an array under $ne (ruled 2026-09-24); every query face refuses a ' + + 'non-boolean $null / $exists flag, because the backends read one in opposite directions. ' + + 'Measured on origin/main af32cf9a before the change: a dataset filter, a dataset measure ' + + 'filter, a dashboard widget filter and a report runtimeFilter each parsed GREEN for one ' + + 'instance of every shape the surface names, while the face refused each one with ' + + 'INVALID_FILTER / 400 and the analytics where door refused every one of them, the flags ' + + 'included. So such a document published clean and then failed every chart built on it. ' + + 'The save door now asks the face itself about each slot, so it refuses exactly what the ' + + 'face refuses and passes what the face passes; the words are the face\'s, or the ' + + 'sentence the enforced operator slot already prints for the same comparand, never the ' + + 'face\'s location clause, which the issue\'s path carries instead. The reach is the ' + + 'face\'s and no wider: the field entries of a condition and of every $and / $or / $not ' + + 'member, and NOT a field spec with no $ key (a nested-relation condition), which neither ' + + 'the face nor the drivers\' flag checks descend. ⚠️ So one position still refuses only at ' + + 'execution: a refused shape INSIDE a nested-relation condition on an analytics carrier ' + + '(a dataset filter or measure filter, a dashboard widget filter, a report runtimeFilter). ' + + 'The analytics where door flattens that relation to a dotted member and refuses it when ' + + 'the carrier is charted. Metadata AT REST is not rewritten and this entry adds no D2 ' + + 'conversion: none of these shapes has a single honest meaning (that is why each was ' + + 'refused), and a conversion would have to pick one. The read path does not re-validate ' + + 'stored rows, so a stored document keeps loading; re-saving it through the metadata ' + + 'protocol (422 INVALID_METADATA), defineStack or os validate is refused at the filter\'s ' + + 'path. Such a filter has failed every query since the runtime refusal of its shape, so ' + + 'the refusal is a repair and not a loss. ADR-0049 / ADR-0087 / ADR-0112.', + acceptanceCriteria: + 'Validate every stack and re-save every stored document that carries a filter: os validate ' + + 'or defineStack, and a save through the metadata protocol, report each refused slot by ' + + 'path with the operator, the field and the prescription, so the sweep is mechanical for ' + + 'the carriers the surface lists. Decide per filter what it meant and write that spelling; ' + + 'on most backends the filter had been failing every query, so re-check what the surface ' + + 'is supposed to show rather than assuming the old rows were right. One producer was ' + + 'measured before the change: a filter builder that writes "is empty" / "is not empty" as ' + + 'an $in / $nin list holding null and the empty string (the Studio filter-condition widget, ' + + 'at the console pin of that date); what it wrote is refused on its next save. ⛔ A clean ' + + 'save is NOT a complete sweep for the one position the reason names: search the analytics ' + + 'carriers for a nested-relation condition whose inner field carries one of these shapes, ' + + 'and chart it, where the analytics where door refuses with INVALID_FILTER / 400 naming ' + + 'the field and the path.', +}; From ef703ccc2c31368d64c77ba97f4d428ca8fc8261 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 11:52:08 +0000 Subject: [PATCH 3/7] chore(spec): regenerate the migration registry for filter-query-face-comparands-refused-at-save Claude-Session: https://claude.ai/code/session_01Rjy9MeetSfq34PKn81CRiN Co-authored-by: Claude --- packages/spec/src/migrations/registry.ts | 79 ++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/packages/spec/src/migrations/registry.ts b/packages/spec/src/migrations/registry.ts index 7dde143ab33..b398ffe6bfd 100644 --- a/packages/spec/src/migrations/registry.ts +++ b/packages/spec/src/migrations/registry.ts @@ -9535,6 +9535,85 @@ const step18: MigrationStep = { + 'door, 400 after), so re-check what the surface was supposed to show rather than ' + 'assuming the old result set was correct.', }, + // The SCHEMA door's half of every comparand refusal the query faces already + // give. The shared comparand-shape face and the query faces' boolean-flag checks + // refuse these slots on every query; this entry records that the same slots are + // now refused where a filter is SAVED, so a stored filter stops publishing clean + // and failing later for someone else. The face itself is the judge at the save + // door, so the two doors cannot drift apart. + { + id: 'filter-query-face-comparands-refused-at-save', + // No backticks in `surface` — build-upgrade-guide renders it inside a code + // span already, and a nested backtick would close it. + surface: + 'data.FilterCondition — every comparand slot the query faces refuse, now refused when the ' + + 'document is PARSED: a $null or $exists flag that is not a boolean (a string such as ' + + '"false", null, a number); a null $gt / $gte / $lt / $lte comparand; an $in or $nin ' + + 'comparand that is not a list, or a list holding null; a $between comparand that is not a ' + + 'two-element list, or whose endpoint is null, blank or a { $field } reference; and an ' + + 'array under $ne. On every schema that carries a FilterCondition: a dataset filter and a ' + + 'dataset measure filter, a dashboard widget filter and an options-source filter, a report ' + + 'and joined-report-block runtimeFilter, a field relatedListFilter and a rollup ' + + 'summaryOperations filter, a solution-blueprint summary filter, an analytics query where, ' + + 'a dataset selection runtimeFilter, a query where and having, the data-engine aggregate ' + + 'call\'s having, an aggregation filter and a query-filter where', + replacement: + 'the spelling the refusal prescribes, which is the one the query faces already prescribe. ' + + 'A flag is the boolean itself: $null true is "has no value", $null false is "has a value", ' + + 'and $exists is the inverse. Absence is the null predicate, never null in an ordering or ' + + 'list position: $eq null is "has no value", $ne null is "has a value", and "one of these ' + + 'values OR has no value" is an $or of an $in and a $null true. A single value for $in is a ' + + 'one-member list, or plain equality. A range is two bounds in a two-element list; a range ' + + 'bounded on one side is a $gte or a $lte; a column-to-column range is a $gte and a $lte ' + + 'whose comparands are { $field } references. "None of these values" is $nin, never $ne ' + + 'with a list. The null predicate itself, a { $field } reference as a whole comparand, ' + + 'an empty $in or $nin list and a whitespace endpoint are untouched', + reason: + 'The save door narrows to exactly what the query faces already refuse (#20116, the ' + + 'collector for its family; the $ne member is route A, the same reach and the same one ' + + 'sentence as the equality slot of filter-equality-array-comparand-refused-at-save). The ' + + 'shared comparand-shape face refuses on every query a null ordering comparand (ruled ' + + '2026-09-01), a non-list $in / $nin and a malformed $between range, a null list member or ' + + 'endpoint (ruled 2026-08-31), a blank endpoint (ruled 2026-09-20), a { $field } endpoint ' + + '(ruled 2026-08-11) and an array under $ne (ruled 2026-09-24); every query face refuses a ' + + 'non-boolean $null / $exists flag, because the backends read one in opposite directions. ' + + 'Measured on origin/main af32cf9a before the change: a dataset filter, a dataset measure ' + + 'filter, a dashboard widget filter and a report runtimeFilter each parsed GREEN for one ' + + 'instance of every shape the surface names, while the face refused each one with ' + + 'INVALID_FILTER / 400 and the analytics where door refused every one of them, the flags ' + + 'included. So such a document published clean and then failed every chart built on it. ' + + 'The save door now asks the face itself about each slot, so it refuses exactly what the ' + + 'face refuses and passes what the face passes; the words are the face\'s, or the ' + + 'sentence the enforced operator slot already prints for the same comparand, never the ' + + 'face\'s location clause, which the issue\'s path carries instead. The reach is the ' + + 'face\'s and no wider: the field entries of a condition and of every $and / $or / $not ' + + 'member, and NOT a field spec with no $ key (a nested-relation condition), which neither ' + + 'the face nor the drivers\' flag checks descend. ⚠️ So one position still refuses only at ' + + 'execution: a refused shape INSIDE a nested-relation condition on an analytics carrier ' + + '(a dataset filter or measure filter, a dashboard widget filter, a report runtimeFilter). ' + + 'The analytics where door flattens that relation to a dotted member and refuses it when ' + + 'the carrier is charted. Metadata AT REST is not rewritten and this entry adds no D2 ' + + 'conversion: none of these shapes has a single honest meaning (that is why each was ' + + 'refused), and a conversion would have to pick one. The read path does not re-validate ' + + 'stored rows, so a stored document keeps loading; re-saving it through the metadata ' + + 'protocol (422 INVALID_METADATA), defineStack or os validate is refused at the filter\'s ' + + 'path. Such a filter has failed every query since the runtime refusal of its shape, so ' + + 'the refusal is a repair and not a loss. ADR-0049 / ADR-0087 / ADR-0112.', + acceptanceCriteria: + 'Validate every stack and re-save every stored document that carries a filter: os validate ' + + 'or defineStack, and a save through the metadata protocol, report each refused slot by ' + + 'path with the operator, the field and the prescription, so the sweep is mechanical for ' + + 'the carriers the surface lists. Decide per filter what it meant and write that spelling; ' + + 'on most backends the filter had been failing every query, so re-check what the surface ' + + 'is supposed to show rather than assuming the old rows were right. One producer was ' + + 'measured before the change: a filter builder that writes "is empty" / "is not empty" as ' + + 'an $in / $nin list holding null and the empty string (the Studio filter-condition widget, ' + + 'at the console pin of that date); what it wrote is refused on its next save. ⛔ A clean ' + + 'save is NOT a complete sweep for the one position the reason names: search the analytics ' + + 'carriers for a nested-relation condition whose inner field carries one of these shapes, ' + + 'and chart it, where the analytics where door refuses with INVALID_FILTER / 400 naming ' + + 'the field and the path.', + }, { id: 'filter-text-operator-declared-type-refused', surface: 'a STORED filter body the engine executes, where a text operator names a ' From 70e9a6100ea97d041e2293a39a9c70b28a90738e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 11:56:08 +0000 Subject: [PATCH 4/7] fix(spec): one save-door judge for both reaches; the dataset carriers refuse every face-refused slot inside a nested relation The judge moves to a non-barrel module so the dataset carriers' nested-relation walk asks the same function FilterConditionSchema's walk asks. Claude-Session: https://claude.ai/code/session_01Rjy9MeetSfq34PKn81CRiN Co-authored-by: Claude --- .../20116-filter-save-door-face-parity.md | 8 +- .../data/filter-save-door-face-parity.test.ts | 87 +++++- .../src/data/filter-save-door-refusals.ts | 295 ++++++++++++++++++ packages/spec/src/data/filter.zod.ts | 235 +------------- ...r-query-face-comparands-refused-at-save.ts | 24 +- ...ataset-filter-nested-relation-list.test.ts | 18 +- packages/spec/src/ui/dataset.zod.ts | 82 ++--- 7 files changed, 479 insertions(+), 270 deletions(-) create mode 100644 packages/spec/src/data/filter-save-door-refusals.ts diff --git a/.changeset/20116-filter-save-door-face-parity.md b/.changeset/20116-filter-save-door-face-parity.md index e3c9e54828a..3ffee5db98f 100644 --- a/.changeset/20116-filter-save-door-face-parity.md +++ b/.changeset/20116-filter-save-door-face-parity.md @@ -16,20 +16,22 @@ fix(spec)!: a filter carrying a comparand the query faces refuse is refused when - a `$between` comparand that is not a two-element list, or whose endpoint is `null`, blank (`""`) or a `{ $field }` reference; - an array under `$ne`. -It judges the field entries of a condition and of every `$and` / `$or` / `$not` member. The shared comparand-shape face (`assertListComparandShapes`) is called read-only as the judge for each slot, so the save door refuses exactly what that face refuses on every query. The two boolean flags, which that face does not judge, are refused on the predicate every flag face uses (`driver-sql`, `driver-memory`, `driver-mongodb`, the read-scope compiler and the analytics `where` door): the comparand is not a boolean. +It judges the field entries of a condition and of every `$and` / `$or` / `$not` member. The dataset `filter` and measure `filter` also judge the same slots INSIDE a nested-relation condition, through the nested-relation walk they already carry, because the analytics `where` door flattens a relation and judges its entries. The shared comparand-shape face (`assertListComparandShapes`) is called read-only as the judge for each slot, so the save door refuses exactly what that face refuses on every query. The two boolean flags, which that face does not judge, are refused on the predicate every flag face uses (`driver-sql`, `driver-memory`, `driver-mongodb`, the read-scope compiler and the analytics `where` door): the comparand is not a boolean. -Measured on `origin/main` `af32cf9a` before the change: a dataset `filter`, a dataset measure `filter`, a dashboard widget `filter` and a report `runtimeFilter` each parsed with `success: true` for one instance of every shape above. The comparand-shape face refused each one but the flags with `INVALID_FILTER` / 400, and the analytics `where` door refused all of them. So such a document published clean and then failed every chart built on it. +Measured on `origin/main` `af32cf9a` before the change: a dataset `filter`, a dataset measure `filter`, a dashboard widget `filter` and a report `runtimeFilter` each parsed with `success: true` for one instance of every shape above. The comparand-shape face refused each one but the flags with `INVALID_FILTER` / 400, and the analytics `where` door refused all of them, inside a nested relation too. So such a document published clean and then failed every chart built on it. Every schema that carries a `FilterCondition` refuses on parse. That covers the dataset `filter` and measure `filter`, the dashboard widget `filter` and options-source `filter`, the report and joined-report-block `runtimeFilter`, the field `relatedListFilter` and rollup `summaryOperations.filter`, the solution-blueprint summary `filter`, the analytics query `where`, the dataset selection `runtimeFilter`, the query `where` and `having`, the data-engine aggregate call's `having`, the aggregation `filter`, and the query-filter `where`. So `defineStack`, `os validate` and a save through the metadata protocol (`422 INVALID_METADATA`) refuse such a document at the slot's path, for example `filter.stage.$null` or `measures.0.filter.amount.$between.0`. The words are the query face's. For an array under `$ne`, a non-list `$in` / `$nin` and a malformed `$between`, the refusal is the face's sentence without its location clause (`at where..`), because the issue's path carries the location. For a `null` ordering comparand, a `null` list member or endpoint, and a blank or `{ $field }` endpoint, it is the sentence the enforced operator slot (`FieldOperatorsSchema`) already prints for the same comparand. A non-boolean flag gets the query faces' sentence: `Operator "$null" on field "stage" requires a boolean comparand (true or false).`, then the received value and the prescription. +Two request doors parse these carriers, and they now answer before the analytics compiler does. The REST dataset selection (its `runtimeFilter`, parsed against `DatasetSelectionSchema`) and the analytics query body (its `where`, parsed against `AnalyticsQueryRequestSchema`) answer `VALIDATION_FAILED` / 400 with the sentence at the field, for a top-level or combinator slot. Before, the compiler answered `INVALID_FILTER` / 400 for the same filter. + This also changes the `$ne` note of the equality-slot change earlier in this release: an array under `$ne` is now refused on save too, in the sentence `FieldOperatorsSchema.$ne` and the face print. ## What does NOT change - **Nothing stored is rewritten, and nothing is dropped.** The parse fails and strips nothing. The read path does not re-validate stored rows, so a stored document keeps loading, and its next save is refused. Such a filter has failed every query since the runtime refusal of its shape, so the refusal is a repair. -- **The reach is the face's, and no wider.** A field spec with no `$` key, such as the nested-relation condition `{ account: { region: { $in: ["a", null] } } }`, is not judged, because neither the face nor the drivers' flag checks descend one. The analytics `where` door does refuse that shape when an analytics carrier is charted. +- **The shared reach is the face's, and no wider.** On every carrier but the two dataset ones, a field spec with no `$` key, such as the nested-relation condition `{ account: { region: { $in: ["a", null] } } }`, is not judged, because neither the face nor the drivers' flag checks descend one. A dashboard widget `filter` and a report `runtimeFilter` therefore still save that shape, and the analytics `where` door refuses it when they are charted. - **What the face passes still passes:** `$eq: null` and `$ne: null` (the null predicate), a `{ $field }` reference as the whole comparand of a scalar comparison, `$in: []` and `$nin: []`, a whitespace-only `$between` endpoint, and falsy endpoints such as `[0, 0]`. - **The data-engine calls' `where` option still parses.** Its type is a union whose first arm is an open record. The face refuses the shape when the call runs. - **No key, export or JSON Schema changes.** The published JSON Schema cannot state a refinement, and `FilterCondition`'s already could not state the equality-slot one. diff --git a/packages/spec/src/data/filter-save-door-face-parity.test.ts b/packages/spec/src/data/filter-save-door-face-parity.test.ts index d918792166a..a9d0a8af5f4 100644 --- a/packages/spec/src/data/filter-save-door-face-parity.test.ts +++ b/packages/spec/src/data/filter-save-door-face-parity.test.ts @@ -4,7 +4,9 @@ * [#20116] The SAVE door (`FilterConditionSchema`) refuses exactly the comparand * slots the QUERY faces refuse — at the top level and in every `$and` / `$or` / * `$not` member, and not inside a nested-relation condition, which the face - * never descends. + * never descends. The two dataset carriers, charted through the analytics + * `where` door that DOES descend a relation, refuse the same slots inside one + * (§5), asking the same function. * * Measured on `origin/main` `af32cf9a` before the change: `DatasetSchema` * (filter and measure filter), a dashboard widget `filter` and a report @@ -35,7 +37,7 @@ import { describe, expect, it } from 'vitest'; import { StandardErrorCode } from '../api/errors.zod'; import { DashboardSchema } from '../ui/dashboard.zod'; -import { DatasetSchema } from '../ui/dataset.zod'; +import { DatasetMeasureSchema, DatasetSchema } from '../ui/dataset.zod'; import { ReportSchema } from '../ui/report.zod'; import { assertListComparandShapes } from './filter-comparand-shape'; import { isRefusedTextComparand } from './filter-text-comparand'; @@ -417,3 +419,84 @@ describe('#20116 §4 — what stays accepted, at both doors', () => { expect(result.data).toEqual(where); }); }); + +// --------------------------------------------------------------------------- +// §5 Inside a nested relation, the ANALYTICS carriers answer as the analytics door +// --------------------------------------------------------------------------- + +describe('#20116 §5 — inside a nested relation, the dataset carriers refuse what the analytics door refuses', () => { + // The analytics `where` door flattens a nested relation to dotted members and + // hands each entry to the same query faces it hands a top-level entry. The + // dataset carriers' own walk (`refuseNestedRelationComparands`, #20207's) + // reaches those entries and asks the same function the shared walk asks, so + // the table is §1's, one relation down, on both carriers. + const dataset = (filter: unknown) => ({ + name: 'deals_ds', + label: 'Deals', + object: 'deal', + dimensions: [{ name: 'stage', field: 'stage', type: 'string' }], + measures: [{ name: 'deal_count', aggregate: 'count' }], + filter, + }); + + it.each([ + ['one hop', (e: Record) => ({ acct: e }), 'acct.f'], + ['two hops, under $or', (e: Record) => ({ $or: [{ acct: { owner: e } }] }), '$or.0.acct.owner.f'], + ] as const)('every operator × comparand cell, %s', (_label, wrap, prefix) => { + const mismatches: string[] = []; + let refused = 0; + for (const op of OPERATORS) { + for (const [label, comparand] of BATTERY) { + const expected = queryFacesRefuse(op, comparand) || textArmRefuses(op, comparand); + const filter = wrap({ f: { [op]: comparand } }); + const scoped = issuesUnder(DatasetSchema.safeParse(dataset(filter)), `filter.${prefix}.${op}`); + const measure = issuesUnder( + DatasetMeasureSchema.safeParse({ name: 'deal_count', aggregate: 'count', filter }), + `filter.${prefix}.${op}`, + ); + if (expected) refused += 1; + if ((scoped.length > 0) !== expected || (measure.length > 0) !== expected) { + mismatches.push(`${op} ← ${label}: faces ${expected ? 'REFUSE' : 'ACCEPT'}, dataset ${scoped.length > 0 ? 'REFUSE' : 'ACCEPT'}, measure ${measure.length > 0 ? 'REFUSE' : 'ACCEPT'}`); + } + } + } + expect(mismatches).toEqual([]); + expect(refused).toBeGreaterThan(40); + }); + + it.each([ + [{ acct: { stage: { $null: 'x' } } }, 'acct.stage.$null'], + [{ acct: { stage: { $exists: 'false' } } }, 'acct.stage.$exists'], + [{ acct: { stage: { $null: null } } }, 'acct.stage.$null'], + [{ acct: { amount: { $gt: null } } }, 'acct.amount.$gt'], + [{ acct: { stage: { $in: 'won' } } }, 'acct.stage.$in'], + [{ acct: { stage: { $in: ['won', null] } } }, 'acct.stage.$in.1'], + [{ acct: { amount: { $between: [null, 5] } } }, 'acct.amount.$between.0'], + [{ acct: { stage: { $ne: ['won', 'lost'] } } }, 'acct.stage.$ne'], + ] as const)('the collector\'s nested member %j — one issue per carrier, in the top-level sentence', (filter, slot) => { + // The same words as the top-level form: the field named is the leaf, as + // the analytics door names it, and the issue path carries the relation. + const leaf = slot.split('.').slice(1).join('.'); + const topLevel = { [leaf.split('.')[0]!]: (filter.acct as Record)[leaf.split('.')[0]!] }; + const expected = issueAt(FilterConditionSchema.safeParse(topLevel), leaf).message; + const scoped = issueAt(DatasetSchema.safeParse(dataset(filter)), `filter.${slot}`); + expect(scoped.code).toBe('custom'); + expect(scoped.message).toBe(expected); + const measure = issueAt(DatasetMeasureSchema.safeParse({ name: 'deal_count', aggregate: 'count', filter }), `filter.${slot}`); + expect(measure.message).toBe(expected); + }); + + it('CONTROL — the null predicate, references, empty lists and flags pass inside a relation, and are kept', () => { + const filter = { + acct: { + stage: { $ne: null, $in: [] }, + owner: { region: { $eq: null } }, + amount: { $gt: { $field: 'floor' }, $between: [1, 9] }, + active: { $null: false, $exists: true }, + }, + }; + const parsed = DatasetSchema.safeParse(dataset(filter)); + expect(parsed.success, JSON.stringify(parsed.error?.issues)).toBe(true); + expect(parsed.data!.filter).toEqual(filter); + }); +}); diff --git a/packages/spec/src/data/filter-save-door-refusals.ts b/packages/spec/src/data/filter-save-door-refusals.ts new file mode 100644 index 00000000000..2160a2f3321 --- /dev/null +++ b/packages/spec/src/data/filter-save-door-refusals.ts @@ -0,0 +1,295 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20116] The SAVE door's verdict on ONE comparand slot: every refusal the + * QUERY faces give for it, in the schema door's words. One function, called by + * the two walks that reach a slot when a filter is saved: + * + * - `FilterConditionSchema`'s own walk (`checkFilterConditionComparands`, + * `./filter.zod.ts`), over the field entries of a condition and of every + * `$and` / `$or` / `$not` member — the shared comparand face's reach, which + * every schema carrying a `FilterCondition` gets; + * - the analytics carriers' nested-relation walk + * (`refuseNestedRelationComparands`, `../ui/dataset.zod.ts`), over the + * entries INSIDE a nested-relation condition, which the analytics `where` + * door flattens to dotted members and judges like any other entry. + * + * So a slot is judged one way whichever reach finds it, and a rule added here + * reaches both. + * + * ## The judge is the query face itself + * + * The comparand-shape face (`assertListComparandShapes`, + * `./filter-comparand-shape.ts`) is called read-only on a one-slot node, so the + * save door refuses exactly the cells the query door refuses: an array in the + * equality or `$ne` slot, a `null` ordering comparand, a non-list `$in` / + * `$nin`, a malformed `$between`, a `null` list member or endpoint, and a blank + * or `{ $field }` endpoint — and passes what the face passes (the null + * predicate, a `{ $field }` reference as a whole comparand, `$in: []`, a + * whitespace endpoint). An arm the face gains later is refused on save the day + * it lands. The two boolean flags, which that face does not judge, are judged + * with the one predicate every flag face uses (`driver-sql`, `driver-memory`, + * `driver-mongodb`, the read-scope compiler, the analytics `where` door): the + * comparand is not a boolean (#5347 / #5369). + * + * ## The words + * + * - The equality and `$ne` slots: the face's own sentence, from the builders + * both doors import (`./filter-comparand-refusal-text.ts`). + * - A `null` ordering comparand, a `null` list member or endpoint, a blank or + * `{ $field }` endpoint: the sentence the enforced operator slot + * (`FieldOperatorsSchema`) prints for the same comparand, read off that slot + * rather than restated, so one condition reads one way at the schema door. + * - A non-list `$in` / `$nin` and a malformed `$between`: the face's sentence + * less its location, because the operator slot has only zod's generic + * wording for those shapes. + * - A non-boolean flag: the query faces' sentence (see + * {@link nonBooleanFlagComparandMessage}). + * + * None carries the face's ` at ` clause: the issue's own `path` carries + * the location, which a refinement cannot see from inside the document. + * + * ## Why a module of its own, and why the operator slots are handed in + * + * `./filter.zod.ts` is re-exported whole by the `data` barrel, so a function + * exported from it would be published API. This module is not in the barrel, + * like `./filter-comparand-refusal-text.ts`. It cannot import `./filter.zod.ts` + * either, because that module imports THIS one — the cycle the face's + * `LIST_COMPARAND_OPERATORS` docblock records. So the enforced operator slots, + * whose sentences four arms print, are handed in by the caller as + * `FieldOperatorsSchema` ({@link OperatorSlots}). + */ + +import type { z } from 'zod'; +import { assertListComparandShapes } from './filter-comparand-shape'; +import { + IN_OPERATOR_SPELLINGS, + NIN_OPERATOR_SPELLINGS, + arrayEqualityComparandMessage, + arrayInequalityComparandMessage, + shapePreview, +} from './filter-comparand-refusal-text'; + +/** One parse result, as far as this module reads it. */ +type SlotParse = { + readonly success: boolean; + readonly error?: { readonly issues: ReadonlyArray<{ readonly path: readonly PropertyKey[]; readonly message: string }> }; +}; + +/** + * The slice of `FieldOperatorsSchema` this module reads: each operator's own + * enforced slot, keyed by operator. Handed in by the caller — see the module + * note for why this module does not import it. + */ +export interface OperatorSlots { + readonly shape: Readonly>; +} + +/** + * Ask the comparand-shape face about ONE slot — a one-entry node holding either + * an implicit comparand (`{ stage: [...] }`) or a single operator + * (`{ stage: { $in: 'won' } }`). Returns the face's refusal, or `undefined` when + * the face accepts. It is handed one slot at a time because it throws on the + * first refusal it meets, and the save door reports every refused slot of a + * document, each at its own path. + * + * Only the face's own envelope (`INVALID_FILTER`) is read as a verdict. + * Anything else it throws is a defect in the face, not a refused filter, and is + * rethrown rather than reported as one. + */ +function comparandShapeFaceRefusal(slot: Record): Error | undefined { + try { + assertListComparandShapes(slot); + } catch (error) { + if ((error as { code?: unknown }).code === 'INVALID_FILTER') return error as Error; + throw error; + } + return undefined; +} + +/** + * The face's `{ $field }` recogniser (`isFieldReferenceShape`): SHAPE only, a + * non-array object carrying a `$field` key. Spelled here because this module + * cannot import the schema door's copy (see the module note); it only locates + * the endpoint the face already refused, and never decides a verdict. + */ +function isFieldReferenceShape(value: unknown): boolean { + return !!value && typeof value === 'object' && !Array.isArray(value) && '$field' in value; +} + +/** `string` / `number` / `null` / `array` … — the face's `describeOperand`, for the two sentences below. */ +function describeComparandKind(value: unknown): string { + if (value === null) return 'null'; + if (value === undefined) return 'undefined'; + if (Array.isArray(value)) return 'array'; + if (value instanceof Date) return 'Date'; + return typeof value; +} + +/** + * `$in` / `$nin` whose comparand is not a list — the face's sentence + * (`nonListComparandError`), less the ` at ` location only the face can + * write. The operator slot (`setMembershipSchema`) has only zod's generic + * wording for this shape, so the face's is the one sentence the platform has. + */ +function nonListComparandMessage(op: '$in' | '$nin', field: string, value: unknown): string { + const spellings = op === '$in' ? IN_OPERATOR_SPELLINGS : NIN_OPERATOR_SPELLINGS; + const alternative = op === '$in' ? '"=" ($eq)' : '"!=" ($ne)'; + return ( + `Operator "${op}" on field "${field}" requires an ARRAY of values. ` + + `Received ${describeComparandKind(value)} (${shapePreview(value)}). ` + + `"${op}" tests membership of a list — write ${shapePreview([value])} for a single value, ` + + `or use ${alternative} to compare against it. Authoring spellings: ${spellings.join(', ')}. ` + + 'The filter was NOT applied, and an unapplied filter would have returned the UNFILTERED ' + + 'result set.' + ); +} + +/** + * `$between` whose comparand is not a two-element list — the face's sentence + * (`malformedRangeComparandError`), less its ` at ` location, for the + * reason {@link nonListComparandMessage} gives. + */ +function malformedRangeComparandMessage(field: string, value: unknown): string { + return ( + `Operator "$between" on field "${field}" requires a [min, max] value array. ` + + `Received ${describeComparandKind(value)} (${shapePreview(value)}). ` + + 'A range needs exactly two bounds, in order; the authoring spelling that lowers to ' + + '"$between" is "between". The filter was NOT applied, and an unapplied filter would have ' + + 'returned the UNFILTERED result set.' + ); +} + +/** The four ordering operators — the positions of the face's `null` ordering-comparand arm. */ +const ORDERING_OPERATORS: ReadonlySet = new Set(['$gt', '$gte', '$lt', '$lte']); + +/** Where a refusal sits below its operator (`[]`, or a member / endpoint index), and its words. */ +type SaveDoorRefusal = { readonly at: readonly number[]; readonly message: string }; + +/** + * The sentence the enforced operator slot prints for `comparand` at `at` — the + * schema door's established words for that condition — or `undefined` when the + * slot raises nothing there. + */ +function operatorSlotSentence( + slots: OperatorSlots, + op: string, + comparand: unknown, + at: readonly number[], +): string | undefined { + const parsed = slots.shape[op]?.safeParse(comparand); + const where = at.join('.'); + return parsed?.error?.issues.find((issue) => issue.path.join('.') === where)?.message; +} + +/** + * Locate the defect the face stopped at, following the face's own order of + * checks (`assertFieldListComparands`), and return where it sits and the words + * for it. The VERDICT is already the face's; this only picks the sentence. + * + * A refusal none of these arms recognises — an arm the face gained after this + * was written — is reported in the face's own words, location included, rather + * than accepted. `filter-save-door-face-parity.test.ts` fails on that text, so + * the new arm is worded here before it ships. + */ +function comparandShapeRefusalAtSave( + slots: OperatorSlots, + field: string, + op: string | undefined, + comparand: unknown, + face: Error, +): SaveDoorRefusal { + const fromSlot = (at: readonly number[]): SaveDoorRefusal | undefined => { + const message = op === undefined ? undefined : operatorSlotSentence(slots, op, comparand, at); + return message === undefined ? undefined : { at, message }; + }; + let refusal: SaveDoorRefusal | undefined; + if (op === undefined) { + refusal = { at: [], message: arrayEqualityComparandMessage(comparand, { field }) }; + } else if (op === '$eq') { + refusal = { at: [], message: arrayEqualityComparandMessage(comparand, { op, field }) }; + } else if (op === '$ne') { + refusal = { at: [], message: arrayInequalityComparandMessage(comparand, { field }) }; + } else if (comparand === null && ORDERING_OPERATORS.has(op)) { + refusal = fromSlot([]); + } else if (op === '$in' || op === '$nin') { + refusal = Array.isArray(comparand) + ? fromSlot([comparand.indexOf(null)]) + : { at: [], message: nonListComparandMessage(op, field, comparand) }; + } else if (op === '$between') { + if (!Array.isArray(comparand) || comparand.length !== 2) { + refusal = { at: [], message: malformedRangeComparandMessage(field, comparand) }; + } else { + const nullBound = comparand.indexOf(null); + const blankBound = comparand.findIndex((bound) => bound === '' || bound === undefined); + const referenceBound = comparand.findIndex(isFieldReferenceShape); + const bound = nullBound !== -1 ? nullBound : blankBound !== -1 ? blankBound : referenceBound; + if (bound !== -1) refusal = fromSlot([bound]); + } + } + return refusal ?? { at: [], message: face.message }; +} + +/** + * The two flags `FieldOperatorsSchema` declares `z.boolean()`. A non-boolean one + * is refused on every query face under the #5347 / #5369 rulings, in every + * position, because the backends read one in opposite directions. + */ +const BOOLEAN_FLAG_OPERATORS: ReadonlySet = new Set(['$null', '$exists']); + +/** What arrived where a flag's boolean belongs — the analytics door's `describeFlagComparand`. */ +function describeFlagComparand(value: unknown): string { + if (value === null) return 'null'; + if (value === undefined) return 'undefined'; + if (typeof value === 'bigint') return `a bigint (${value}n)`; + if (Array.isArray(value)) return `an array (${shapePreview(value)})`; + if (value instanceof Date) return `a Date (${shapePreview(value)})`; + if (isFieldReferenceShape(value)) return `a field reference (${shapePreview(value)})`; + return `a ${typeof value} (${shapePreview(value)})`; +} + +/** + * The refusal of a non-boolean `$null` / `$exists` flag, as the query faces + * give it. The first sentence is `driver-sql`'s word for word through "(true or + * false)", which the analytics `where` door also keeps; the reason and the + * prescription are the analytics door's, less the location and the history of + * what that door used to do. The field is named because this door can see it; + * the issue's own `path` carries the location. + */ +function nonBooleanFlagComparandMessage(op: string, field: string, value: unknown): string { + const [whenTrue, whenFalse] = op === '$null' ? ['has no value', 'has a value'] : ['has a value', 'has no value']; + return ( + `Operator "${op}" on field "${field}" requires a boolean comparand (true or false). ` + + `Received ${describeFlagComparand(value)}. @objectstack/spec FieldOperatorsSchema declares ` + + `${op} as a boolean, and a non-boolean is refused rather than coerced because the backends ` + + 'read one in OPPOSITE directions — one as IS NULL, another as IS NOT NULL. Write the ' + + `boolean itself: "${op}": true matches rows whose "${field}" ${whenTrue}, "${op}": false ` + + `rows whose "${field}" ${whenFalse}. The filter was NOT applied.` + ); +} + +/** + * Raise, as `custom` issues under `slotPath`, every refusal the query faces + * give for ONE comparand slot: the comparand-shape face's verdict on the slot, + * and — for the two boolean flags, which that face does not judge — the flag + * rule. `op` is `undefined` for an implicit-equality comparand, and `slotPath` + * is then the field's own path. `slots` is `FieldOperatorsSchema`. + */ +export function reportQueryFaceRefusals( + ctx: z.RefinementCtx, + slotPath: readonly (string | number)[], + field: string, + op: string | undefined, + comparand: unknown, + slots: OperatorSlots, +): void { + const refusals: SaveDoorRefusal[] = []; + const face = comparandShapeFaceRefusal(op === undefined ? { [field]: comparand } : { [field]: { [op]: comparand } }); + if (face) refusals.push(comparandShapeRefusalAtSave(slots, field, op, comparand, face)); + if (op !== undefined && BOOLEAN_FLAG_OPERATORS.has(op) && typeof comparand !== 'boolean') { + refusals.push({ at: [], message: nonBooleanFlagComparandMessage(op, field, comparand) }); + } + for (const refusal of refusals) { + ctx.addIssue({ code: 'custom', path: [...slotPath, ...refusal.at], message: refusal.message }); + } +} diff --git a/packages/spec/src/data/filter.zod.ts b/packages/spec/src/data/filter.zod.ts index 2925c497344..63e8e25cb54 100644 --- a/packages/spec/src/data/filter.zod.ts +++ b/packages/spec/src/data/filter.zod.ts @@ -7,12 +7,13 @@ import { assertListComparandShapes } from './filter-comparand-shape'; // face so the save door and the query door print one sentence (ruling A, // record 5805248669: "one constant, two doors"). import { - IN_OPERATOR_SPELLINGS, - NIN_OPERATOR_SPELLINGS, arrayEqualityComparandMessage, arrayInequalityComparandMessage, - shapePreview, } from './filter-comparand-refusal-text'; +// [#20116] The save door's verdict on one comparand slot — the query faces' +// refusals, the comparand-shape face deciding — shared with the analytics +// carriers' nested-relation walk (`../ui/dataset.zod.ts`). +import { reportQueryFaceRefusals } from './filter-save-door-refusals'; import { normalizeFilterComparandTypes } from './filter-comparand-type'; import { bareDateRangePresetComparandMessage, isDateRangePresetName } from './date-range-presets'; // [#19514] The text-comparand door this package publishes for the @@ -1656,214 +1657,6 @@ function isPlainFilterNode(value: unknown): value is Record { ); } -// ── [#20116] The query faces' comparand verdicts, asked at the save door ────── - -/** - * [#20116] Ask the comparand-shape face (`assertListComparandShapes`, - * `./filter-comparand-shape.ts`) about ONE comparand slot — a one-entry node - * holding either an implicit comparand (`{ stage: [...] }`) or a single operator - * (`{ stage: { $in: 'won' } }`). Returns the face's refusal, or `undefined` when - * the face accepts. - * - * The face is the JUDGE here, called read-only, so the save door refuses - * exactly the cells the query door refuses and no others: an arm the face gains - * later is refused on save the day it lands. It is handed one slot at a time - * because it throws on the first refusal it meets, and the save door reports - * every refused slot of a document, each at its own path. - * - * Only the face's own envelope (`INVALID_FILTER`) is read as a verdict. - * Anything else it throws is a defect in the face, not a refused filter, and is - * rethrown rather than reported as one. - */ -function comparandShapeFaceRefusal(slot: Record): Error | undefined { - try { - assertListComparandShapes(slot); - } catch (error) { - if ((error as { code?: unknown }).code === 'INVALID_FILTER') return error as Error; - throw error; - } - return undefined; -} - -/** `string` / `number` / `null` / `array` … — the face's `describeOperand`, for the two sentences below. */ -function describeComparandKind(value: unknown): string { - if (value === null) return 'null'; - if (value === undefined) return 'undefined'; - if (Array.isArray(value)) return 'array'; - if (value instanceof Date) return 'Date'; - return typeof value; -} - -/** - * [#20116] `$in` / `$nin` whose comparand is not a list — the face's sentence - * (`nonListComparandError`), less the ` at ` location only the face can - * write. The issue this door raises carries that location as its own `path`. - * The operator slot (`setMembershipSchema`) has only zod's generic wording for - * this shape, so the face's is the one sentence the platform has for it. - */ -function nonListComparandMessage(op: '$in' | '$nin', field: string, value: unknown): string { - const spellings = op === '$in' ? IN_OPERATOR_SPELLINGS : NIN_OPERATOR_SPELLINGS; - const alternative = op === '$in' ? '"=" ($eq)' : '"!=" ($ne)'; - return ( - `Operator "${op}" on field "${field}" requires an ARRAY of values. ` - + `Received ${describeComparandKind(value)} (${shapePreview(value)}). ` - + `"${op}" tests membership of a list — write ${shapePreview([value])} for a single value, ` - + `or use ${alternative} to compare against it. Authoring spellings: ${spellings.join(', ')}. ` - + 'The filter was NOT applied, and an unapplied filter would have returned the UNFILTERED ' - + 'result set.' - ); -} - -/** - * [#20116] `$between` whose comparand is not a two-element list — the face's - * sentence (`malformedRangeComparandError`), less its ` at ` location, for - * the reason {@link nonListComparandMessage} gives. - */ -function malformedRangeComparandMessage(field: string, value: unknown): string { - return ( - `Operator "$between" on field "${field}" requires a [min, max] value array. ` - + `Received ${describeComparandKind(value)} (${shapePreview(value)}). ` - + 'A range needs exactly two bounds, in order; the authoring spelling that lowers to ' - + '"$between" is "between". The filter was NOT applied, and an unapplied filter would have ' - + 'returned the UNFILTERED result set.' - ); -} - -/** The four ordering operators — the positions of the face's `null` ordering-comparand arm. */ -const ORDERING_OPERATORS: ReadonlySet = new Set(['$gt', '$gte', '$lt', '$lte']); - -/** Where a refusal sits below its operator (`[]`, or a member / endpoint index), and its words. */ -type SaveDoorRefusal = { readonly at: readonly number[]; readonly message: string }; - -/** - * [#20116] The save door's words for a slot the face refused. The VERDICT is - * the face's ({@link comparandShapeFaceRefusal}); this only picks the sentence, - * following the face's own order of checks so the sentence names the defect - * the face stopped at: - * - * - an array in the equality slot (implicit or `$eq`) and under `$ne` — the - * face's sentence, from the builder both doors import - * (`./filter-comparand-refusal-text.ts`); - * - a `null` ordering comparand, a `null` list member or `$between` endpoint, a - * blank endpoint and a `{ $field }` endpoint — the sentence the enforced - * operator slot (`FieldOperatorsSchema`) already prints for the same - * comparand, so one condition reads one way at the schema door; - * - a non-list `$in` / `$nin` and a `$between` that is not a pair — the face's - * sentence less its location, since no schema-door sentence exists for them. - * - * A refusal none of those arms recognises — an arm the face gained after this - * was written — is reported in the face's own words, location included, rather - * than accepted. `filter-save-door-face-parity.test.ts` fails on that text, so - * the new arm is worded here before it ships. - */ -function comparandShapeRefusalAtSave( - field: string, - op: string | undefined, - comparand: unknown, - face: Error, -): SaveDoorRefusal { - if (op === undefined) return { at: [], message: arrayEqualityComparandMessage(comparand, { field }) }; - if (op === '$eq') return { at: [], message: arrayEqualityComparandMessage(comparand, { op, field }) }; - if (op === '$ne') return { at: [], message: arrayInequalityComparandMessage(comparand, { field }) }; - if (comparand === null && ORDERING_OPERATORS.has(op)) { - return { at: [], message: nullOrderingComparandMessage(op) }; - } - if (op === '$in' || op === '$nin') { - if (!Array.isArray(comparand)) return { at: [], message: nonListComparandMessage(op, field, comparand) }; - const nullMember = comparand.indexOf(null); - if (nullMember !== -1) { - return { at: [nullMember], message: nullListComparandMemberMessage(`${op} member at index ${nullMember}`) }; - } - } - if (op === '$between') { - if (!Array.isArray(comparand) || comparand.length !== 2) { - return { at: [], message: malformedRangeComparandMessage(field, comparand) }; - } - const nullBound = comparand.indexOf(null); - if (nullBound !== -1) { - return { at: [nullBound], message: nullListComparandMemberMessage(`$between endpoint at index ${nullBound}`) }; - } - const blankBound = comparand.findIndex((bound) => bound === '' || bound === undefined); - if (blankBound === 0 || blankBound === 1) return { at: [blankBound], message: blankRangeBoundMessage(blankBound) }; - const referenceBound = comparand.findIndex(isFieldReferenceShape); - if (referenceBound !== -1) { - return { - at: [referenceBound], - message: listPositionFieldReferenceMessage(`$between endpoint at index ${referenceBound}`), - }; - } - } - return { at: [], message: face.message }; -} - -/** - * [#20116] The two flags `FieldOperatorsSchema` declares `z.boolean()`. A - * non-boolean one is refused on every query face — `driver-sql`, `driver-memory` - * and `driver-mongodb` (`nonBooleanNullComparandError`), the read-scope - * compiler and the analytics `where` door — under the #5347 / #5369 rulings: - * refused in every position, because the backends read one in opposite - * directions. The comparand-shape face does not judge flags, so this door - * judges them with the predicate every one of those faces uses: - * `typeof comparand !== 'boolean'`. - */ -const BOOLEAN_FLAG_OPERATORS: ReadonlySet = new Set(['$null', '$exists']); - -/** What arrived where a flag's boolean belongs — the analytics door's `describeFlagComparand`. */ -function describeFlagComparand(value: unknown): string { - if (value === null) return 'null'; - if (value === undefined) return 'undefined'; - if (typeof value === 'bigint') return `a bigint (${value}n)`; - if (Array.isArray(value)) return `an array (${shapePreview(value)})`; - if (value instanceof Date) return `a Date (${shapePreview(value)})`; - if (isFieldReferenceShape(value)) return `a field reference (${shapePreview(value)})`; - return `a ${typeof value} (${shapePreview(value)})`; -} - -/** - * [#20116] The refusal of a non-boolean `$null` / `$exists` flag, as the query - * faces give it. The first sentence is `driver-sql`'s word for word through - * "(true or false)", which the analytics `where` door also keeps; the reason and - * the prescription are the analytics door's, less the location and the history - * of what that door used to do. The field is named because this door can see - * it; the issue's own `path` carries the location. - */ -function nonBooleanFlagComparandMessage(op: string, field: string, value: unknown): string { - const [whenTrue, whenFalse] = op === '$null' ? ['has no value', 'has a value'] : ['has a value', 'has no value']; - return ( - `Operator "${op}" on field "${field}" requires a boolean comparand (true or false). ` - + `Received ${describeFlagComparand(value)}. @objectstack/spec FieldOperatorsSchema declares ` - + `${op} as a boolean, and a non-boolean is refused rather than coerced because the backends ` - + 'read one in OPPOSITE directions — one as IS NULL, another as IS NOT NULL. Write the ' - + `boolean itself: "${op}": true matches rows whose "${field}" ${whenTrue}, "${op}": false ` - + `rows whose "${field}" ${whenFalse}. The filter was NOT applied.` - ); -} - -/** - * [#20116] Raise, as `custom` issues under `slotPath`, every refusal the query - * faces give for ONE comparand slot, in this door's words: the comparand-shape - * face's verdict on the slot, and — for the two boolean flags, which that face - * does not judge — the flag rule. `op` is `undefined` for an implicit-equality - * comparand, and `slotPath` is then the field's own path. - */ -function reportQueryFaceRefusals( - ctx: z.RefinementCtx, - slotPath: (string | number)[], - field: string, - op: string | undefined, - comparand: unknown, -): void { - const refusals: SaveDoorRefusal[] = []; - const face = comparandShapeFaceRefusal(op === undefined ? { [field]: comparand } : { [field]: { [op]: comparand } }); - if (face) refusals.push(comparandShapeRefusalAtSave(field, op, comparand, face)); - if (op !== undefined && BOOLEAN_FLAG_OPERATORS.has(op) && typeof comparand !== 'boolean') { - refusals.push({ at: [], message: nonBooleanFlagComparandMessage(op, field, comparand) }); - } - for (const refusal of refusals) { - ctx.addIssue({ code: 'custom', path: [...slotPath, ...refusal.at], message: refusal.message }); - } -} - /** * Walk one condition node and report every comparand this authoring door * refuses — the bare date-range PRESET names in an ordering position (#8793), @@ -1885,15 +1678,17 @@ function reportQueryFaceRefusals( * while the face refused each shape with `INVALID_FILTER` / 400 — so a stored * filter published and then failed every query built on it. * - * - **The judge is the face itself**, asked about one slot at a time - * ({@link comparandShapeFaceRefusal}), so this door refuses exactly what the - * query door refuses and nothing else — `null` in the equality and `$ne` + * - **The judge is the face itself**, asked about one slot at a time by + * `reportQueryFaceRefusals` (`./filter-save-door-refusals.ts`, the one + * function the analytics carriers' nested-relation walk calls too), so this + * door refuses exactly what the query door refuses and nothing else — `null` + * in the equality and `$ne` * slots (the null predicate), a `{ $field }` reference as a whole comparand, * `$in: []` / `$nin: []` and a whitespace endpoint all keep passing, because * the face passes them. The flags, which that face does not judge, use the * one predicate every flag face uses: `typeof comparand !== 'boolean'`. - * - **The words** are chosen by {@link comparandShapeRefusalAtSave}: the face's - * own sentence where the two doors already share a builder, the enforced + * - **The words** are chosen in that module: the face's own sentence where + * the two doors already share a builder, the enforced * operator slot's sentence where `FieldOperatorsSchema` already prints one for * the same comparand, and the face's sentence less its location where neither * door had one. None carries the face's ` at `; the issue's `path` does. @@ -1902,7 +1697,9 @@ function reportQueryFaceRefusals( * with no `$` key. The drivers' flag checks stop at the same place. The * analytics `where` door does descend a nested relation (it flattens one to a * dotted member), so those positions belong to the analytics carriers' own - * refinement, never to this shared walk, which every other carrier reads. + * walk (`refuseNestedRelationComparands`, `../ui/dataset.zod.ts`), which + * asks the same function — never to this shared walk, which every other + * carrier reads. * * ## The equality-slot arm answers the FACE, in the face's words (#19889) * @@ -1992,7 +1789,7 @@ function checkFilterConditionComparands( // that has no `$` key, so nothing inside a nested-relation condition is // refused there, and nothing is refused here. See the docblock. if (!isPlainFilterNode(value)) { - if (depth === 0) reportQueryFaceRefusals(ctx, [...path, key], key, undefined, value); + if (depth === 0) reportQueryFaceRefusals(ctx, [...path, key], key, undefined, value, FieldOperatorsSchema); continue; } const hasOperatorKeys = Object.keys(value).some((k) => k.startsWith('$')); @@ -2008,7 +1805,7 @@ function checkFilterConditionComparands( // (the equality and `$ne` slots, the ordering `null` carve-out, the list // operators' shape, null-member and endpoint rules) and the boolean // flags'. An operator neither judges passes through untouched. - if (depth === 0) reportQueryFaceRefusals(ctx, [...path, key, op], key, op, comparand); + if (depth === 0) reportQueryFaceRefusals(ctx, [...path, key, op], key, op, comparand, FieldOperatorsSchema); if (op === FILTER_TEXT_COMPARAND_OPERATOR && isRefusedTextComparand(comparand)) { ctx.addIssue({ code: 'custom', diff --git a/packages/spec/src/migrations/entries/semantic/18.filter-query-face-comparands-refused-at-save.ts b/packages/spec/src/migrations/entries/semantic/18.filter-query-face-comparands-refused-at-save.ts index 05929426aa7..15dadf1355a 100644 --- a/packages/spec/src/migrations/entries/semantic/18.filter-query-face-comparands-refused-at-save.ts +++ b/packages/spec/src/migrations/entries/semantic/18.filter-query-face-comparands-refused-at-save.ts @@ -23,7 +23,8 @@ export const entry: SemanticMigration = { + 'and joined-report-block runtimeFilter, a field relatedListFilter and a rollup ' + 'summaryOperations filter, a solution-blueprint summary filter, an analytics query where, ' + 'a dataset selection runtimeFilter, a query where and having, the data-engine aggregate ' - + 'call\'s having, an aggregation filter and a query-filter where', + + 'call\'s having, an aggregation filter and a query-filter where; and, on a dataset filter ' + + 'and a dataset measure filter only, the same slots INSIDE a nested-relation condition', replacement: 'the spelling the refusal prescribes, which is the one the query faces already prescribe. ' + 'A flag is the boolean itself: $null true is "has no value", $null false is "has a value", ' @@ -55,11 +56,14 @@ export const entry: SemanticMigration = { + 'face\'s location clause, which the issue\'s path carries instead. The reach is the ' + 'face\'s and no wider: the field entries of a condition and of every $and / $or / $not ' + 'member, and NOT a field spec with no $ key (a nested-relation condition), which neither ' - + 'the face nor the drivers\' flag checks descend. ⚠️ So one position still refuses only at ' - + 'execution: a refused shape INSIDE a nested-relation condition on an analytics carrier ' - + '(a dataset filter or measure filter, a dashboard widget filter, a report runtimeFilter). ' - + 'The analytics where door flattens that relation to a dotted member and refuses it when ' - + 'the carrier is charted. Metadata AT REST is not rewritten and this entry adds no D2 ' + + 'the face nor the drivers\' flag checks descend. The analytics where door DOES descend ' + + 'one (it flattens the relation to dotted members and judges each), so the two dataset ' + + 'carriers, whose own nested-relation walk already refused an equality list there ' + + '(dataset-filter-nested-relation-equality-array-refused-at-save), now ask the same ' + + 'judge about every slot inside a relation. ⚠️ So one position still refuses only at ' + + 'execution: a refused shape INSIDE a nested-relation condition on a dashboard widget ' + + 'filter or a report runtimeFilter, which reach the analytics where door too but carry ' + + 'the shared schema\'s reach only. Metadata AT REST is not rewritten and this entry adds no D2 ' + 'conversion: none of these shapes has a single honest meaning (that is why each was ' + 'refused), and a conversion would have to pick one. The read path does not re-validate ' + 'stored rows, so a stored document keeps loading; re-saving it through the metadata ' @@ -76,8 +80,8 @@ export const entry: SemanticMigration = { + 'measured before the change: a filter builder that writes "is empty" / "is not empty" as ' + 'an $in / $nin list holding null and the empty string (the Studio filter-condition widget, ' + 'at the console pin of that date); what it wrote is refused on its next save. ⛔ A clean ' - + 'save is NOT a complete sweep for the one position the reason names: search the analytics ' - + 'carriers for a nested-relation condition whose inner field carries one of these shapes, ' - + 'and chart it, where the analytics where door refuses with INVALID_FILTER / 400 naming ' - + 'the field and the path.', + + 'save is NOT a complete sweep for the one position the reason names: search dashboard ' + + 'widget filters and report runtimeFilters for a nested-relation condition whose inner ' + + 'field carries one of these shapes, and chart it, where the analytics where door refuses ' + + 'with INVALID_FILTER / 400 naming the field and the path.', }; diff --git a/packages/spec/src/ui/dataset-filter-nested-relation-list.test.ts b/packages/spec/src/ui/dataset-filter-nested-relation-list.test.ts index 28e5f89d8ce..1a39066a23b 100644 --- a/packages/spec/src/ui/dataset-filter-nested-relation-list.test.ts +++ b/packages/spec/src/ui/dataset-filter-nested-relation-list.test.ts @@ -206,7 +206,6 @@ describe('#20080 §4 — what a nested relation may still carry', () => { ['$in: [] stays the declared predicate', { account: { region: { $in: [] } } }], ['$nin keeps its list', { account: { region: { $nin: ['a'] } } }], ['$between keeps its pair', { account: { score: { $between: [1, 9] } } }], - ['$ne carrying a list — refused at the shared face (ruling A, #19886), not yet at this save door (#20116)', { account: { region: { $ne: ['a'] } } }], ['a scalar under $and', { $and: [{ account: { region: 'a' } }] }], ])('%s', (_label, filter) => { const scoped = DatasetSchema.safeParse(withScope(filter)); @@ -218,6 +217,23 @@ describe('#20080 §4 — what a nested relation may still carry', () => { expect(measure.data!.filter).toEqual(filter); }); + it('[#20116] $ne carrying a list inside a relation is refused on both carriers, in the door\'s $ne sentence', () => { + // This row sat in the table above as "not yet at this save door (#20116)". + // #20116 routes every slot this walk reaches through the query faces' + // verdict, so the shape the analytics door refuses on chart is refused here. + const filter = { account: { region: { $ne: ['a'] } } }; + const door = analyticsDoorRefusal('region', { $ne: ['a'] }, 'where.account'); + expect(door.code).toBe(StandardErrorCode.enum.INVALID_FILTER); + expect(door.status).toBe(400); + const location = ' at where.account.region.$ne.'; + expect(door.message.split(location)).toHaveLength(2); + const scoped = issueAt(DatasetSchema.safeParse(withScope(filter)), 'filter.account.region.$ne'); + expect(scoped.message).toBe(door.message.replace(location, '.')); + expect(scoped.message).toContain('{"$nin": […]}'); + const measure = issueAt(DatasetMeasureSchema.safeParse({ name: 'deal_count', aggregate: 'count', filter }), 'filter.account.region.$ne'); + expect(measure.message).toBe(scoped.message); + }); + it('the shared FilterConditionSchema keeps its own reach — ruling A is untouched', () => { for (const filter of [{ account: { region: ['a'] } }, { account: { region: { $eq: ['a'] } } }]) { const result = FilterConditionSchema.safeParse(filter); diff --git a/packages/spec/src/ui/dataset.zod.ts b/packages/spec/src/ui/dataset.zod.ts index 1f5a2c9700f..a9b21cb11b5 100644 --- a/packages/spec/src/ui/dataset.zod.ts +++ b/packages/spec/src/ui/dataset.zod.ts @@ -5,8 +5,8 @@ import { lazySchema } from '../shared/lazy-schema'; import { strictObject } from '../shared/strict-object'; import { ProtectionSchema } from '../shared/protection.zod'; import { MetadataProtectionFields } from '../kernel/metadata-protection.zod'; -import { FilterConditionSchema } from '../data/filter.zod'; -import { arrayEqualityComparandMessage } from '../data/filter-comparand-refusal-text'; +import { FieldOperatorsSchema, FilterConditionSchema } from '../data/filter.zod'; +import { reportQueryFaceRefusals } from '../data/filter-save-door-refusals'; import { SnakeCaseIdentifierSchema } from '../shared/identifiers.zod'; import { I18nLabelSchema } from './i18n.zod'; import { AggregationFunction, DateGranularity } from '../data/query.zod'; @@ -109,6 +109,10 @@ function isAnalyticsNestedRelationSpec(spec: unknown): spec is Record - refuseNestedRelationEqualityLists(member, ctx, [...path, key, index], insideRelation)); + refuseNestedRelationComparands(member, ctx, [...path, key, index], insideRelation)); } continue; } if (key === '$not') { - refuseNestedRelationEqualityLists(spec, ctx, [...path, key], insideRelation); + refuseNestedRelationComparands(spec, ctx, [...path, key], insideRelation); continue; } if (key.startsWith('$')) continue; if (isAnalyticsNestedRelationSpec(spec)) { - refuseNestedRelationEqualityLists(spec, ctx, [...path, key], true); + refuseNestedRelationComparands(spec, ctx, [...path, key], true); continue; } if (!insideRelation) continue; // `FilterConditionSchema` judges this entry itself - if (Array.isArray(spec)) { - ctx.addIssue({ - code: 'custom', - path: [...path, key], - message: arrayEqualityComparandMessage(spec, { field: key }), - }); - } else if (isAnalyticsFilterObject(spec) && Array.isArray(spec.$eq)) { - ctx.addIssue({ - code: 'custom', - path: [...path, key, '$eq'], - message: arrayEqualityComparandMessage(spec.$eq, { op: '$eq', field: key }), - }); + // [#20116] Every slot the analytics door hands to the query faces, asked of + // the one function `FilterConditionSchema`'s own walk asks: an implicit + // comparand, or each operator of an operator map. + if (!isAnalyticsFilterObject(spec)) { + reportQueryFaceRefusals(ctx, [...path, key], key, undefined, spec, FieldOperatorsSchema); + continue; + } + for (const [op, comparand] of Object.entries(spec)) { + if (!op.startsWith('$')) continue; + reportQueryFaceRefusals(ctx, [...path, key, op], key, op, comparand, FieldOperatorsSchema); } } } @@ -211,7 +223,7 @@ function refuseNestedRelationEqualityLists( /** * The optional filter both analytics carriers declare — `DatasetSchema.filter` * and `DatasetMeasureSchema.filter` — which is `FilterConditionSchema` plus - * {@link refuseNestedRelationEqualityLists}. Every other schema that carries a + * {@link refuseNestedRelationComparands}. Every other schema that carries a * `FilterCondition` keeps the shared schema's reach. * * The check sits on the OPTIONAL wrapper, not on `FilterConditionSchema` @@ -226,7 +238,7 @@ function refuseNestedRelationEqualityLists( */ function analyticsCarrierFilter() { return FilterConditionSchema.optional().superRefine((filter, ctx) => - refuseNestedRelationEqualityLists(filter, ctx)); + refuseNestedRelationComparands(filter, ctx)); } /** @@ -341,7 +353,7 @@ export const DatasetMeasureSchema = lazySchema(() => strictObject({ * Measure-scoped filter (e.g. only won deals for "won_amount"). [#20080] A * list in the equality slot inside a nested relation is refused on save, as * the analytics door refuses it on chart — see - * {@link refuseNestedRelationEqualityLists}. + * {@link refuseNestedRelationComparands}. */ filter: analyticsCarrierFilter().meta({ title: 'Filter' }), /** @@ -504,7 +516,7 @@ export const DatasetSchema = lazySchema(() => strictObject({ * Definition-level filter (the dataset's intrinsic scope, e.g. non-deleted). * [#20080] A list in the equality slot inside a nested relation is refused * on save, as the analytics door refuses it on chart — see - * {@link refuseNestedRelationEqualityLists}. + * {@link refuseNestedRelationComparands}. */ filter: analyticsCarrierFilter().describe('Intrinsic dataset scope filter'), From 31ddfa40c33eaded68434a7f162be26890ad1305 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 13:00:45 +0000 Subject: [PATCH 5/7] chore(spec): regenerate the migration registry for the nested-relation reach Claude-Session: https://claude.ai/code/session_01Rjy9MeetSfq34PKn81CRiN Co-authored-by: Claude --- packages/spec/src/migrations/registry.ts | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/spec/src/migrations/registry.ts b/packages/spec/src/migrations/registry.ts index 165b60dafaa..8547bfc6a0e 100644 --- a/packages/spec/src/migrations/registry.ts +++ b/packages/spec/src/migrations/registry.ts @@ -9628,7 +9628,8 @@ const step18: MigrationStep = { + 'and joined-report-block runtimeFilter, a field relatedListFilter and a rollup ' + 'summaryOperations filter, a solution-blueprint summary filter, an analytics query where, ' + 'a dataset selection runtimeFilter, a query where and having, the data-engine aggregate ' - + 'call\'s having, an aggregation filter and a query-filter where', + + 'call\'s having, an aggregation filter and a query-filter where; and, on a dataset filter ' + + 'and a dataset measure filter only, the same slots INSIDE a nested-relation condition', replacement: 'the spelling the refusal prescribes, which is the one the query faces already prescribe. ' + 'A flag is the boolean itself: $null true is "has no value", $null false is "has a value", ' @@ -9660,11 +9661,14 @@ const step18: MigrationStep = { + 'face\'s location clause, which the issue\'s path carries instead. The reach is the ' + 'face\'s and no wider: the field entries of a condition and of every $and / $or / $not ' + 'member, and NOT a field spec with no $ key (a nested-relation condition), which neither ' - + 'the face nor the drivers\' flag checks descend. ⚠️ So one position still refuses only at ' - + 'execution: a refused shape INSIDE a nested-relation condition on an analytics carrier ' - + '(a dataset filter or measure filter, a dashboard widget filter, a report runtimeFilter). ' - + 'The analytics where door flattens that relation to a dotted member and refuses it when ' - + 'the carrier is charted. Metadata AT REST is not rewritten and this entry adds no D2 ' + + 'the face nor the drivers\' flag checks descend. The analytics where door DOES descend ' + + 'one (it flattens the relation to dotted members and judges each), so the two dataset ' + + 'carriers, whose own nested-relation walk already refused an equality list there ' + + '(dataset-filter-nested-relation-equality-array-refused-at-save), now ask the same ' + + 'judge about every slot inside a relation. ⚠️ So one position still refuses only at ' + + 'execution: a refused shape INSIDE a nested-relation condition on a dashboard widget ' + + 'filter or a report runtimeFilter, which reach the analytics where door too but carry ' + + 'the shared schema\'s reach only. Metadata AT REST is not rewritten and this entry adds no D2 ' + 'conversion: none of these shapes has a single honest meaning (that is why each was ' + 'refused), and a conversion would have to pick one. The read path does not re-validate ' + 'stored rows, so a stored document keeps loading; re-saving it through the metadata ' @@ -9681,10 +9685,10 @@ const step18: MigrationStep = { + 'measured before the change: a filter builder that writes "is empty" / "is not empty" as ' + 'an $in / $nin list holding null and the empty string (the Studio filter-condition widget, ' + 'at the console pin of that date); what it wrote is refused on its next save. ⛔ A clean ' - + 'save is NOT a complete sweep for the one position the reason names: search the analytics ' - + 'carriers for a nested-relation condition whose inner field carries one of these shapes, ' - + 'and chart it, where the analytics where door refuses with INVALID_FILTER / 400 naming ' - + 'the field and the path.', + + 'save is NOT a complete sweep for the one position the reason names: search dashboard ' + + 'widget filters and report runtimeFilters for a nested-relation condition whose inner ' + + 'field carries one of these shapes, and chart it, where the analytics where door refuses ' + + 'with INVALID_FILTER / 400 naming the field and the path.', }, { id: 'filter-text-operator-declared-type-refused', From dfca00f01db0f2895a52b27065d8072ff0190ea3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 14:54:05 +0000 Subject: [PATCH 6/7] test(rest): the one-bound $between is refused at the analytics route's schema door now The row moves from the normalizer-refusal table to the #17551 door table, asserting 400 VALIDATION_FAILED, the member path and the face's sentence; the changeset gains the HTTP-door FROM -> TO rows. Claude-Session: https://claude.ai/code/session_01Rjy9MeetSfq34PKn81CRiN Co-authored-by: Claude --- .../20116-filter-save-door-face-parity.md | 12 ++++ .../analytics-filter-refusal-envelope.test.ts | 72 +++++++++++++------ 2 files changed, 63 insertions(+), 21 deletions(-) diff --git a/.changeset/20116-filter-save-door-face-parity.md b/.changeset/20116-filter-save-door-face-parity.md index 3ffee5db98f..475a3727a51 100644 --- a/.changeset/20116-filter-save-door-face-parity.md +++ b/.changeset/20116-filter-save-door-face-parity.md @@ -50,6 +50,18 @@ This also changes the `$ne` note of the equality-slot change earlier in this rel | `{ amount: { $between: 5 } }`, `{ amount: { $between: [1] } }` | `{ amount: { $between: [1, 5] } }` | | `{ stage: { $ne: ["won", "lost"] } }` | `{ stage: { $nin: ["won", "lost"] } }` | +### FROM → TO at the HTTP doors + +Same status, different code: the refusal now comes from the route's schema door, located on the member, instead of from the analytics filter normalizer. + +| request | before | after | +|:--|:--|:--| +| `POST /analytics/dataset/query` with `selection.runtimeFilter: { amount: { $between: [10] } }` | `400 INVALID_FILTER` from the analytics normalizer, in the comparand-shape face's sentence | `400 VALIDATION_FAILED`, `details.fields[]` entry `selection.runtimeFilter.amount.$between` with the sentence `Operator "$between" on field "amount" requires a [min, max] value array. Received array ([10]). …` | +| the same route, any other slot above in `selection.runtimeFilter` (top level or in `$and` / `$or` / `$not`) | `400 INVALID_FILTER` | `400 VALIDATION_FAILED`, located on the slot, with that slot's sentence | +| `POST /analytics/query` (`AnalyticsQueryRequestSchema`) with the same shape in `where` | `400 INVALID_FILTER` | refused by the request schema at `where.amount.$between`, answered `400 VALIDATION_FAILED` | + +A client that branches on `INVALID_FILTER` for these shapes reads `VALIDATION_FAILED` instead. Both are 400 and both name the field. + ## Who is affected, measured A literal-comparand scan of every member shape, with a lit control per shape, over `examples/**` and the non-test `packages/**` of this repository at `af32cf9a`, the console repository at its pinned commit `f8a9d0fb05`, and the cloud repository's `main` at `48d70663ab`, found authored filters carrying one in one place. The console's filter-condition widget writes "is empty" as `{ field: { $in: [null, ""] } }` and "is not empty" as `{ field: { $nin: [null, ""] } }`. That widget edits a field's `relatedListFilter` and a rollup's `summaryOperations.filter` in the Studio field designer, and a sharing rule's criteria. Both shapes carry a `null` list member, which the face has refused on every query since the 2026-08-31 ruling, so a filter saved that way has been failing its related list or rollup since then. After this change, the Studio save is refused instead, with the `$or` / `$null` prescription. Every other hit is prose, a type table or a test fixture. Deployed datasets, dashboards and reports were NOT measured. Validating each stack, or re-saving each document, finds every instance the surface above lists. diff --git a/packages/rest/src/analytics-filter-refusal-envelope.test.ts b/packages/rest/src/analytics-filter-refusal-envelope.test.ts index 8cec51ea3cd..d2a139b4d6e 100644 --- a/packages/rest/src/analytics-filter-refusal-envelope.test.ts +++ b/packages/rest/src/analytics-filter-refusal-envelope.test.ts @@ -193,19 +193,10 @@ describe('[#5352] POST /analytics/dataset/query — a filter refusal reaches the runtimeFilter: { stage: {} }, message: /carries a field constraint with zero operators/, }, - { - // [#20010] RE-WORDED, same verdict: still a real normalizer refusal, - // still 400 INVALID_FILTER through this seam. PR #20032 hands every field - // entry of an object-form `where` to the shared comparand-shape face - // (`assertListComparandShapes`) before any node is built, and the face's - // own rule answers first — a list operator takes a list, `$between` a - // two-element [min, max] (#5869, moved to the face by #9228). So the - // message is the face's, the same bytes the FilterArray spelling gets; - // the analytics "needs a two-element" sentence is no longer reached. - name: 'a $between with one bound', - runtimeFilter: { amount: { $between: [10] } }, - message: /Operator "\$between" on field "amount" requires a \[min, max\] value array/, - }, + // [#20116] `a $between with one bound` sat here. It no longer reaches the + // normalizer: `FilterConditionSchema` now refuses every slot the shared + // comparand-shape face refuses, so this route's schema door answers it + // first. It moved, with its #20010 note, to the #17551 block below. { name: 'an unsupported top-level operator', runtimeFilter: { $nor: [{ stage: 'won' }] }, @@ -227,7 +218,9 @@ describe('[#5352] POST /analytics/dataset/query — a filter refusal reaches the /** * [#17551] Three spellings that used to reach the normalizer now stop one layer * earlier — at the route's schema door — and the block above no longer claims - * them. + * them. [#20116] A fourth joined them: the one-bound `$between`, once + * `FilterConditionSchema` began refusing on save every comparand slot the + * shared comparand-shape face refuses on query. * * ⚠️ This is a CODE change on a live wire surface, so it is recorded with the * measurement that justifies it rather than as a test edit. Since #17551 the @@ -242,14 +235,17 @@ describe('[#5352] POST /analytics/dataset/query — a filter refusal reaches the * | `{ $or: [{…}, 'nope'] }` | refused at the schema | refused at the schema | * | `{ $not: 5 }` | refused at the schema | refused at the schema | * | `{ stage: {} }` | passes the schema | passes the schema | - * | `{ amount: { $between: [10] } }` | passes the schema | passes the schema | + * | `{ amount: { $between: [10] } }` | refused at the schema (#20116) | refused at the schema (#20116) | * | `{ $nor: [{…}] }` | passes the schema | passes the schema | * | `{ $or: [] }` | passes the schema | passes the schema | * * ⇒ the two routes now answer this field IDENTICALLY, which is the whole reason * the door exists ("one family, two postures" was the defect). The three rows - * that changed changed because the dataset route used to be the LOOSER of the - * two, not because anything narrowed past `FilterCondition`. + * that changed under #17551 changed because the dataset route used to be the + * LOOSER of the two, not because anything narrowed past `FilterCondition`. The + * `$between` row changed under #20116 on BOTH routes at once, because + * `FilterCondition` itself narrowed: the schema now refuses what the face + * refused on every query, in the face's sentence less its location. * * ⛔ Nothing here weakens #5352's subject: the seam it exists for — a real * `AnalyticsService`, a real `normalizeAnalyticsFilterTree` refusal, and this @@ -257,10 +253,23 @@ describe('[#5352] POST /analytics/dataset/query — a filter refusal reaches the * driven by every case left in the block above, `$sortOf` included. */ describe('[#17551] the structurally-malformed filter spellings are refused at the door', () => { - const AT_THE_DOOR: Array<{ name: string; runtimeFilter: unknown }> = [ + const AT_THE_DOOR: Array<{ name: string; runtimeFilter: unknown; member?: string; sentence?: RegExp }> = [ { name: 'an $or that is not an array', runtimeFilter: { $or: 'won' } }, { name: 'an $or branch that is not a filter object', runtimeFilter: { $or: [{ stage: 'won' }, 'nope'] } }, { name: 'a $not of a non-object', runtimeFilter: { $not: 5 } }, + { + // [#20010] The face's sentence has named this shape since PR #20032 handed + // every field entry of an object-form `where` to the shared comparand- + // shape face (`assertListComparandShapes`) — a list operator takes a + // list, `$between` a two-element [min, max] (#5869, moved to the face by + // #9228) — and it still does. [#20116] What moved is WHERE: the schema + // door now asks that face on save and prints its sentence less the + // `at where…` location, so the refusal is located on the member instead. + name: 'a $between with one bound', + runtimeFilter: { amount: { $between: [10] } }, + member: 'selection.runtimeFilter.amount.$between', + sentence: /^Operator "\$between" on field "amount" requires a \[min, max\] value array\. Received array \(\[10\]\)\. A range needs exactly two bounds/, + }, ]; for (const c of AT_THE_DOOR) { @@ -274,8 +283,17 @@ describe('[#17551] the structurally-malformed filter spellings are refused at th expect(res.body.code).not.toBe('ANALYTICS_QUERY_FAILED'); expect(res.body.code).toBe('VALIDATION_FAILED'); // …and it says WHICH member, which the deeper refusal never did. - const fields: Array<{ field: string }> = res.body.details.fields; + const fields: Array<{ field: string; message: string }> = res.body.details.fields; expect(fields.map((f) => f.field).some((f) => f.startsWith('selection.runtimeFilter'))).toBe(true); + if (c.member) { + // [#20116] Exactly one entry, at the slot, carrying the face's sentence. + const atMember = fields.filter((f) => f.field === c.member); + expect(atMember, JSON.stringify(fields)).toHaveLength(1); + expect(atMember[0]!.message).toMatch(c.sentence!); + expect(atMember[0]!.message).not.toContain(' at where.'); + // The normalizer never ran: the refusal is the door's, not INVALID_FILTER. + expect(res.body.code).not.toBe('INVALID_FILTER'); + } }); } @@ -288,12 +306,24 @@ describe('[#17551] the structurally-malformed filter spellings are refused at th where: c.runtimeFilter, }); expect(parsed.success, `${c.name} must be refused by the sibling schema too`).toBe(false); + if (c.member) { + // [#20116] …at the same slot, in the same sentence, one level up: the + // sibling body carries the filter as `where`, not `selection.runtimeFilter`. + const issues: Array<{ path: PropertyKey[]; message: string }> = parsed.error.issues; + const slot = c.member.replace(/^selection\.runtimeFilter\./, 'where.'); + const atSlot = issues.filter((i) => i.path.join('.') === slot); + expect(atSlot, JSON.stringify(issues.map((i) => i.path))).toHaveLength(1); + expect(atSlot[0]!.message).toMatch(c.sentence!); + } } }); - it('CONTROL — the four spellings the schema PASSES still cross the seam', async () => { + it('CONTROL — the three spellings the schema PASSES still cross the seam', async () => { + // [#20116] `{ amount: { $between: [10] } }` left this list: the schema + // refuses it now, on both routes (the AT_THE_DOOR row above, and the + // sibling-schema control, which iterates that table). const { AnalyticsQueryRequestSchema } = await import('@objectstack/spec/api'); - const passes = [{ stage: {} }, { amount: { $between: [10] } }, { $nor: [{ stage: 'won' }] }, { $or: [] }]; + const passes = [{ stage: {} }, { $nor: [{ stage: 'won' }] }, { $or: [] }]; for (const where of passes) { const parsed = (AnalyticsQueryRequestSchema as any).safeParse({ cube: 'opportunity', measures: ['revenue'], where, From ab146a9a3f4e3aa077594ba66b86880bb4ff933e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 15:01:32 +0000 Subject: [PATCH 7/7] test(spec): fulfil the $ne carrier-walk todo; changeset gains the $nin null-member rows Claude-Session: https://claude.ai/code/session_01Rjy9MeetSfq34PKn81CRiN Co-authored-by: Claude --- .../20116-filter-save-door-face-parity.md | 3 + .../data/filter-ne-array-schema-door.test.ts | 60 ++++++++++++++----- 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/.changeset/20116-filter-save-door-face-parity.md b/.changeset/20116-filter-save-door-face-parity.md index 475a3727a51..a3d584964c3 100644 --- a/.changeset/20116-filter-save-door-face-parity.md +++ b/.changeset/20116-filter-save-door-face-parity.md @@ -45,6 +45,9 @@ This also changes the `$ne` note of the equality-slot change earlier in this rel | `{ amount: { $gt: null } }` | `{ amount: { $eq: null } }` ("has no value") or `{ amount: { $ne: null } }` ("has a value") | | `{ stage: { $in: "won" } }` | `{ stage: { $in: ["won"] } }` or `{ stage: "won" }` | | `{ stage: { $in: ["won", null] } }` | `{ $or: [{ stage: { $in: ["won"] } }, { stage: { $null: true } }] }` | +| `{ stage: { $nin: ["lost", null] } }`, meaning "has a value and is not `lost`" | `{ stage: { $nin: ["lost"], $null: false } }` | +| `{ stage: { $in: [null, ""] } }`, a filter builder's "is empty" | `{ $or: [{ stage: { $null: true } }, { stage: "" }] }` | +| `{ stage: { $nin: [null, ""] } }`, a filter builder's "is not empty" | `{ stage: { $null: false, $ne: "" } }` | | `{ amount: { $between: [null, 5] } }`, `{ amount: { $between: ["", 5] } }` | `{ amount: { $lte: 5 } }`, or the bound you meant | | `{ amount: { $between: [{ $field: "floor" }, 5] } }` | `{ amount: { $gte: { $field: "floor" }, $lte: 5 } }` | | `{ amount: { $between: 5 } }`, `{ amount: { $between: [1] } }` | `{ amount: { $between: [1, 5] } }` | diff --git a/packages/spec/src/data/filter-ne-array-schema-door.test.ts b/packages/spec/src/data/filter-ne-array-schema-door.test.ts index 7b4f7e53709..9c327d38545 100644 --- a/packages/spec/src/data/filter-ne-array-schema-door.test.ts +++ b/packages/spec/src/data/filter-ne-array-schema-door.test.ts @@ -36,9 +36,11 @@ import { describe, expect, it } from 'vitest'; import { StandardErrorCode } from '../api/errors.zod'; import { assertListComparandShapes } from './filter-comparand-shape'; +import { DatasetSchema } from '../ui/dataset.zod'; import { EqualityOperatorSchema, FieldOperatorsSchema, + FilterConditionSchema, NormalizedFilterSchema, parseFilterAST, } from './filter.zod'; @@ -235,21 +237,49 @@ describe('#19886 §4 — what stays accepted, at BOTH doors', () => { }); // --------------------------------------------------------------------------- -// §5 The boundary this change does NOT move +// §5 The stored-filter carrier walk — refused on save since #20116 // --------------------------------------------------------------------------- -describe('#19886 §5 — the stored-filter carrier walk', () => { - // Why the operator slot's refusal does not reach a stored carrier: - // `FilterConditionSchema` parses a field entry as `unknown` and judges it - // with its own walk (`checkFilterConditionComparands`), which judges `$eq` - // and not `$ne`. The `$eq` side of that walk is pinned in - // `filter-equality-array-schema-door.test.ts`, including that its equality - // arm raises nothing for `$ne`. - it.todo( - 'FilterConditionSchema (every stored filter carrier: dataset, widget, report, rollup …) refusing ' - + '$ne: [...] on SAVE. Ruling A names the face and FieldOperatorsSchema.$ne, not the carrier walk, ' - + 'so the carrier still saves the shape and the face refuses it at query time. Reported to the ' - + 'seat for its own decision; ⛔ not pinned green here, because a green pin would read as a ' - + 'ruling nobody made', - ); +describe('#19886 §5 — the stored-filter carrier walk refuses $ne: [...] too (#20116)', () => { + // This section held an `it.todo`: ruling A named the face and + // `FieldOperatorsSchema.$ne`, not `FilterConditionSchema`'s carrier walk, so + // every stored carrier still saved the shape and the face refused it at query + // time. #20116's `$ne` member (route A: the equality arm's reach, the one + // sentence) routes every slot that walk reaches through the face itself, so + // the todo is fulfilled here. The full operator × comparand table, carriers + // and nested relations included, is `filter-save-door-face-parity.test.ts`. + it('FilterConditionSchema refuses it at the slot, in the face\'s sentence less its location', () => { + const where = { stage: { $ne: ['won', 'lost'] } }; + const face = faceRefusal(where); + expect(face.code).toBe(StandardErrorCode.enum.INVALID_FILTER); + expect(face.status).toBe(400); + const location = ' at where.stage.$ne.'; + expect(face.message.split(location)).toHaveLength(2); + const issue = issueAt(FilterConditionSchema.safeParse(where), 'stage.$ne'); + expect(issue.code).toBe('custom'); + expect(issue.message).toBe(face.message.replace(location, '.')); + expect(issue.message).toMatch(/^Operator "\$ne" on field "stage" requires a single comparable value/); + expect(issue.message).toContain(REMEDY); + expect(issue.message).toMatch(NOT_APPLIED); + }); + + it('a stored carrier refuses it at its own path — a dataset filter', () => { + const parsed = DatasetSchema.safeParse({ + name: 'deals_ds', + label: 'Deals', + object: 'deal', + dimensions: [{ name: 'stage', field: 'stage', type: 'string' }], + measures: [{ name: 'deal_count', aggregate: 'count' }], + filter: { $or: [{ stage: { $ne: [] } }] }, + }); + expect(issueAt(parsed, 'filter.$or.0.stage.$ne').message).toContain(REMEDY); + }); + + it('CONTROL — the null predicate and a scalar still save', () => { + for (const where of [{ stage: { $ne: null } }, { stage: { $ne: 'lost' } }]) { + const result = FilterConditionSchema.safeParse(where); + expect(result.success, JSON.stringify(result.error?.issues)).toBe(true); + expect(result.data).toEqual(where); + } + }); });