From 46ee85eda186e22e93afd8c988b0ac027fc4e6e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 14:30:20 +0000 Subject: [PATCH 1/3] test(service-analytics,runtime,rest): pin a structured-JSON dimension refused at the analytics door (red) Pins first, before the fix. On the base they are red: - POST /api/v1/analytics/query with a cube or dataset dimension on a json field answers 200 with one group per serialized document on SQLite and 500 DATABASE_ERROR on PostgreSQL 16; /analytics/sql builds the statement. - POST /api/v1/analytics/dataset/query answers the same two ways. - The service door pins (both strategy faces, every structured-JSON type, the member as written, the one-predicate GUARD) are red; their controls are green. Claude-Session: https://claude.ai/code/session_01XY5uCwTjZj7884yYtyur4H Co-authored-by: Claude --- ...lytics-dataset-json-dimension-door.test.ts | 215 +++++++++++++++ .../src/analytics-json-dimension-door.test.ts | 260 ++++++++++++++++++ .../dimension-structured-json-door.test.ts | 233 ++++++++++++++++ 3 files changed, 708 insertions(+) create mode 100644 packages/rest/src/analytics-dataset-json-dimension-door.test.ts create mode 100644 packages/runtime/src/analytics-json-dimension-door.test.ts create mode 100644 packages/services/service-analytics/src/__tests__/dimension-structured-json-door.test.ts diff --git a/packages/rest/src/analytics-dataset-json-dimension-door.test.ts b/packages/rest/src/analytics-dataset-json-dimension-door.test.ts new file mode 100644 index 00000000000..bdb750f29a0 --- /dev/null +++ b/packages/rest/src/analytics-dataset-json-dimension-door.test.ts @@ -0,0 +1,215 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * A dataset dimension on a structured-JSON field is refused at the dataset + * door — `POST /api/v1/analytics/dataset/query` answers `400 INVALID_FIELD`, + * naming the dataset dimension the selection wrote, before any SQL is built — + * over a real `SqlDriver`; and a `text` dimension (the control) is served + * unchanged. + * + * The cube face of the same door (`POST /api/v1/analytics/query`, served by + * `@objectstack/runtime`'s dispatcher, not by this package) is pinned in + * `packages/runtime/src/analytics-json-dimension-door.test.ts`. This file is + * the route this package serves: an INLINE dataset, compiled per request, + * whose dimensions reach the same cube query through `DatasetExecutor`. + * + * ## Measured on the base, through this door + * + * Three rows, `title` x, x, y, and a different `meta` document per row; a + * dataset declaring `meta_doc` over the `json` field `meta`: + * + * | `selection.dimensions` | SQLite | PostgreSQL 16 | + * |:--|:--|:--| + * | `title_dim` (text, the control) | 200, `x` 2 · `y` 1 | same | + * | `meta_doc` (json) | 200, one group per serialized document (3) | 500 | + * + * ## The composition, and the dialect axis of THIS file + * + * The analytics service is the one `AnalyticsServicePlugin` composes over a + * real `ObjectQL` engine — both auto-bridges live, so `NativeSQLStrategy` + * answers on a SQL driver. The SQLite cell always runs. The PostgreSQL cell + * runs where `OS_TEST_POSTGRES_URL` is set and is a named skip otherwise; no + * CI step provisions that variable for this package, so the live cell is + * red-capable and un-run in CI, and the PR that landed this file carries its + * local PostgreSQL 16 run. The live cell owns its table, dropped before and + * after. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { AnalyticsServicePlugin, type AnalyticsService } from '@objectstack/service-analytics'; +import { RestServer } from './rest-server'; + +const OBJECT = 'rest_dataset_json_dim_ledger'; + +const LEDGER = { + name: OBJECT, + label: 'Dataset JSON dimension ledger', + fields: { + title: { name: 'title', type: 'text' as const }, + meta: { name: 'meta', type: 'json' as const }, + }, +}; + +const ROWS = [ + { id: 'r1', title: 'x', meta: { a: 1 } }, + { id: 'r2', title: 'x', meta: { a: 2 } }, + { id: 'r3', title: 'y', meta: { b: 1 } }, +]; + +/** The inline dataset the request carries — as a Studio preview or a widget posts it. */ +const DATASET = { + name: 'json_dim_inline', + label: 'JSON dimension inline', + object: OBJECT, + dimensions: [ + { name: 'title_dim', field: 'title', type: 'string' }, + { name: 'meta_doc', field: 'meta', type: 'string' }, + ], + measures: [{ name: 'row_count', aggregate: 'count' }], +}; + +/** The route the refusal prescribes — asserted on the wire body. */ +const ROUTE = 'Group by a field that stores one scalar value: store the part you group on in a field of its own and group by that field.'; + +interface Cell { + id: 'sqlite' | 'pg'; + label: string; + env: string | null; + config: () => Record | null; +} + +const CELLS: readonly Cell[] = [ + { id: 'sqlite', label: 'sqlite', env: null, config: () => ({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) }, + { + id: 'pg', + label: 'live postgres', + env: 'OS_TEST_POSTGRES_URL', + config: () => (process.env.OS_TEST_POSTGRES_URL ? { client: 'pg', connection: process.env.OS_TEST_POSTGRES_URL } : null), + }, +]; + +const quiet = { debug() {}, info() {}, warn() {}, error() {}, child() { return quiet; } }; + +function createMockServer() { + const noop = () => {}; + return { get: noop, post: noop, put: noop, delete: noop, patch: noop, use: noop, listen: async () => {}, close: async () => {} }; +} + +function mockProtocol() { + return { + getDiscovery: async () => ({ version: 'v0', routes: { data: '', metadata: '' } }), + getMetaTypes: async () => [], + getMetaItems: async () => [], + }; +} + +function makeRes() { + const res: any = { + statusCode: 200, + body: undefined as any, + header: () => res, + status: (code: number) => { res.statusCode = code; return res; }, + json: (body: unknown) => { res.body = body; return res; }, + end: () => res, + }; + return res; +} + +for (const cell of CELLS) { + const config = cell.config(); + describe.skipIf(!config)( + `a dataset dimension on a json field at POST /api/v1/analytics/dataset/query — ${cell.label}${config ? '' : ` (skipped: set ${cell.env} to run this cell)`}`, + () => { + let driver: any; + let engine: ObjectQL; + /** Raw-SQL statements and engine aggregates that read THIS object. */ + const reads = { rawSql: 0, aggregate: 0 }; + let query: (selection: Record) => Promise<{ status: number; body: any }>; + + const dropTables = async () => { + if (cell.id === 'sqlite') return; + await driver?.execute(`drop table if exists ${OBJECT}`).catch(() => {}); + }; + + beforeAll(async () => { + driver = new SqlDriver(config as any); + await dropTables(); + engine = new ObjectQL({ logger: quiet } as any); + engine.registerDriver(driver, true); + await engine.init(); + engine.registry.registerObject(LEDGER as any); + await engine.syncSchemas(); + for (const row of ROWS) await engine.insert(OBJECT, { ...row } as any); + + const realExecute = (engine as any).execute.bind(engine); + (engine as any).execute = (sql: unknown, opts?: { object?: string }) => { + if (opts?.object === OBJECT) reads.rawSql += 1; + return realExecute(sql, opts); + }; + const realAggregate = engine.aggregate.bind(engine); + (engine as any).aggregate = (object: string, ...rest: unknown[]) => { + if (object === OBJECT) reads.aggregate += 1; + return (realAggregate as any)(object, ...rest); + }; + + // The plugin's own composition over the real engine: both auto-bridges. + const registered: Record = {}; + await new AnalyticsServicePlugin().init({ + getService: (name: string) => (name === 'data' ? engine : registered[name]), + registerService: (name: string, svc: unknown) => { registered[name] = svc; }, + replaceService: (name: string, svc: unknown) => { registered[name] = svc; }, + hook: () => {}, + logger: quiet, + } as never); + const analytics = registered.analytics as AnalyticsService; + + const rest = new RestServer( + createMockServer() as any, mockProtocol() as any, { api: { requireAuth: false } } as any, + undefined, undefined, undefined, undefined, undefined, undefined, undefined, + undefined, undefined, undefined, undefined, + async () => analytics, + ); + (rest as any).resolveExecCtx = async () => ({ userId: 'test-user' }); + rest.registerRoutes(); + const route = rest.getRoutes().find((r: any) => r.method === 'POST' && r.path === '/api/v1/analytics/dataset/query'); + expect(route).toBeDefined(); + query = async (selection) => { + const res = makeRes(); + // What the wire carries: JSON. + const body = JSON.parse(JSON.stringify({ dataset: DATASET, selection })); + await route!.handler({ method: 'POST', params: {}, headers: {}, body, query: {} } as any, res); + return { status: res.statusCode, body: res.body }; + }; + }); + + afterAll(async () => { + await dropTables(); + try { await engine?.destroy(); } catch { /* noop */ } + }); + + it('a dataset dimension on a json field answers 400 INVALID_FIELD naming the dataset dimension — no statement reaches the engine', async () => { + const before = { ...reads }; + const res = await query({ measures: ['row_count'], dimensions: ['meta_doc'] }); + expect(res.status, JSON.stringify(res.body)).toBe(400); + expect(res.body.code).toBe('INVALID_FIELD'); + expect(String(res.body.message)).toContain(`Dimension 'meta_doc' on cube '${DATASET.name}' groups by field 'meta'`); + expect(String(res.body.message)).toContain(`'${OBJECT}' declares as json`); + expect(String(res.body.message)).toContain(ROUTE); + expect(reads, 'no raw SQL and no engine aggregate for the object').toEqual(before); + }); + + it('CONTROL a text dataset dimension is served unchanged: one group per value, counted', async () => { + const before = { ...reads }; + const res = await query({ measures: ['row_count'], dimensions: ['title_dim'] }); + expect(res.status, JSON.stringify(res.body)).toBe(200); + const groups = (res.body.rows as Array<{ title_dim: string; row_count: number | string }>) + .map((r) => [r.title_dim, Number(r.row_count)] as const) + .sort(([a], [b]) => a.localeCompare(b)); + expect(groups).toEqual([['x', 2], ['y', 1]]); + expect(reads.rawSql - before.rawSql, 'the native strategy answered').toBeGreaterThanOrEqual(1); + }); + }, + ); +} diff --git a/packages/runtime/src/analytics-json-dimension-door.test.ts b/packages/runtime/src/analytics-json-dimension-door.test.ts new file mode 100644 index 00000000000..e7291ae33d5 --- /dev/null +++ b/packages/runtime/src/analytics-json-dimension-door.test.ts @@ -0,0 +1,260 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * A cube dimension and a dataset dimension on a structured-JSON field are + * refused at the public analytics door — `POST /api/v1/analytics/query` (and + * its dry-run twin `POST /api/v1/analytics/sql`) answer `400 INVALID_FIELD`, + * naming the member the caller wrote, before any SQL is built — over a real + * `SqlDriver`; and a `text` dimension (the control) is served unchanged. + * + * ## The composition is the shipped one + * + * `AnalyticsServicePlugin` is initialised over a real `ObjectQL` engine as its + * `'data'` service, so both of its auto-bridges are live: `executeRawSql` → + * `engine.execute` (which is what makes `NativeSQLStrategy` the strategy that + * answers on a SQL driver) and `executeAggregate` → `engine.aggregate`. The + * route is the real `dispatcher-plugin` mount. A dataset is registered with + * `registerDataset`, the configuration door, and queried by its name — the + * cube it compiles to. + * + * ## Measured on the base, through this door + * + * Three rows, `title` x, x, y, and a different `meta` document per row: + * + * | `dimensions` | SQLite | PostgreSQL 16 | + * |:--|:--|:--| + * | `title` (text, the control) | 200, `x` 2 · `y` 1 (native SQL) | same | + * | `meta` (json), a cube dimension | 200, one group per serialized document (3) | 500 | + * | `meta_doc` (a dataset dimension on `meta`) | 200, 3 groups | 500 | + * + * `engine.aggregate` was reached 0 times on those rows: the native strategy + * compiled `GROUP BY` itself, so the engine's own `groupBy` refusal never saw + * the query. + * + * ## The dialect axis of THIS file + * + * The SQLite cell always runs. The PostgreSQL cell runs where + * `OS_TEST_POSTGRES_URL` is set and is a named skip otherwise. No CI step + * provisions that variable for this file, so the live cell is red-capable and + * un-run in CI; the PR that landed this file carries its local PostgreSQL + * 16 run. The live cell owns its table, dropped before and after. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { AnalyticsServicePlugin, type AnalyticsService } from '@objectstack/service-analytics'; +import { DatasetSchema } from '@objectstack/spec/ui'; + +import { createDispatcherPlugin } from './dispatcher-plugin.js'; + +const OBJECT = 'analytics_json_dim_ledger'; +const CUBE = 'json_dim_ledger'; +const DATASET = 'json_dim_ledger_ds'; + +const LEDGER = { + name: OBJECT, + label: 'JSON dimension ledger', + fields: { + title: { name: 'title', type: 'text' as const }, + meta: { name: 'meta', type: 'json' as const }, + }, +}; + +const ROWS = [ + { id: 'j1', title: 'x', meta: { a: 1 } }, + { id: 'j2', title: 'x', meta: { a: 2 } }, + { id: 'j3', title: 'y', meta: { b: 1 } }, +]; + +/** An authored cube over the object: one text and one json dimension. */ +const LEDGER_CUBE = { + name: CUBE, + title: 'JSON dimension ledger', + sql: OBJECT, + public: true, + measures: { count: { label: 'Rows', type: 'count' as const, sql: '*' } }, + dimensions: { + title: { label: 'Title', type: 'string' as const, sql: 'title' }, + meta: { label: 'Meta', type: 'string' as const, sql: 'meta' }, + }, +}; + +/** A dataset whose dimensions compile to the same two columns, under names of their own. */ +const LEDGER_DATASET = DatasetSchema.parse({ + name: DATASET, + label: 'JSON dimension ledger dataset', + object: OBJECT, + dimensions: [ + { name: 'title_dim', field: 'title', type: 'string' }, + { name: 'meta_doc', field: 'meta', type: 'string' }, + ], + measures: [{ name: 'row_count', aggregate: 'count' }], +}); + +/** The route the refusal prescribes — asserted on the wire body. */ +const ROUTE = 'Group by a field that stores one scalar value: store the part you group on in a field of its own and group by that field.'; + +interface Cell { + id: 'sqlite' | 'pg'; + label: string; + env: string | null; + config: () => Record | null; +} + +const CELLS: readonly Cell[] = [ + { id: 'sqlite', label: 'sqlite', env: null, config: () => ({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) }, + { + id: 'pg', + label: 'live postgres', + env: 'OS_TEST_POSTGRES_URL', + config: () => (process.env.OS_TEST_POSTGRES_URL ? { client: 'pg', connection: process.env.OS_TEST_POSTGRES_URL } : null), + }, +]; + +const quiet = { debug() {}, info() {}, warn() {}, error() {}, child() { return quiet; } }; + +type Handler = (req: unknown, res: unknown) => unknown; + +function makeRes() { + const res: any = { + statusCode: undefined as number | undefined, + body: undefined as any, + status(c: number) { res.statusCode = c; return res; }, + header() { return res; }, + json(b: unknown) { res.body = b; return res; }, + }; + return res; +} + +for (const cell of CELLS) { + const config = cell.config(); + describe.skipIf(!config)( + `a dimension on a json field at POST /api/v1/analytics/query — ${cell.label}${config ? '' : ` (skipped: set ${cell.env} to run this cell)`}`, + () => { + let driver: any; + let engine: ObjectQL; + /** Raw-SQL statements and engine aggregates that read THIS object. */ + const reads = { rawSql: 0, aggregate: 0 }; + const handlers: Record = {}; + + const dropTables = async () => { + if (cell.id === 'sqlite') return; + await driver?.execute(`drop table if exists ${OBJECT}`).catch(() => {}); + }; + + async function post(sub: 'query' | 'sql', body: Record) { + const handler = handlers[`POST /api/v1/analytics/${sub}`]; + expect(handler, `POST /api/v1/analytics/${sub} must be mounted`).toBeTypeOf('function'); + const res = makeRes(); + // What the wire carries: JSON. + await handler({ body: JSON.parse(JSON.stringify(body)), query: {} }, res); + return { status: res.statusCode ?? 200, body: res.body }; + } + + beforeAll(async () => { + driver = new SqlDriver(config as any); + await dropTables(); + engine = new ObjectQL({ logger: quiet } as any); + engine.registerDriver(driver, true); + await engine.init(); + engine.registry.registerObject(LEDGER as any); + await engine.syncSchemas(); + for (const row of ROWS) await engine.insert(OBJECT, { ...row } as any); + + // Count what reaches the engine for THIS object, on both bridges. + const realExecute = (engine as any).execute.bind(engine); + (engine as any).execute = (sql: unknown, opts?: { object?: string }) => { + if (opts?.object === OBJECT) reads.rawSql += 1; + return realExecute(sql, opts); + }; + const realAggregate = engine.aggregate.bind(engine); + (engine as any).aggregate = (object: string, ...rest: unknown[]) => { + if (object === OBJECT) reads.aggregate += 1; + return (realAggregate as any)(object, ...rest); + }; + + // The plugin's own composition over the real engine: both auto-bridges. + const registered: Record = {}; + await new AnalyticsServicePlugin({ cubes: [LEDGER_CUBE as any] }).init({ + getService: (name: string) => (name === 'data' ? engine : registered[name]), + registerService: (name: string, svc: unknown) => { registered[name] = svc; }, + replaceService: (name: string, svc: unknown) => { registered[name] = svc; }, + hook: () => {}, + logger: quiet, + } as never); + const analytics = registered.analytics as AnalyticsService; + analytics.registerDataset(LEDGER_DATASET); + + const rec = (verb: string) => (path: string, handler: Handler) => { handlers[`${verb} ${path}`] = handler; }; + const server = { get: rec('GET'), post: rec('POST'), put: rec('PUT'), delete: rec('DELETE'), patch: rec('PATCH') }; + const kernel = { + getService: (name: string) => (name === 'analytics' ? analytics : undefined), + getServiceAsync: async (name: string) => (name === 'analytics' ? analytics : undefined), + }; + const plugin = createDispatcherPlugin({ prefix: '/api/v1', securityHeaders: false }); + await plugin.start?.({ + getKernel: () => kernel, + getService: (name: string) => (name === 'http.server' ? server : undefined), + environmentId: undefined, + logger: quiet, + hook: () => {}, + on: () => {}, + } as any); + }); + + afterAll(async () => { + await dropTables(); + try { await engine?.destroy(); } catch { /* noop */ } + }); + + it('a cube dimension on a json field answers 400 INVALID_FIELD naming the member the caller wrote — no statement reaches the engine', async () => { + const before = { ...reads }; + const res = await post('query', { cube: CUBE, measures: ['count'], dimensions: ['meta'] }); + expect(res.status, JSON.stringify(res.body)).toBe(400); + expect(res.body.error.code).toBe('INVALID_FIELD'); + expect(res.body.error.httpStatus).toBe(400); + expect(res.body.error.message).toContain(`Dimension 'meta' on cube '${CUBE}' groups by field 'meta'`); + expect(res.body.error.message).toContain(`'${OBJECT}' declares as json`); + expect(res.body.error.message).toContain(ROUTE); + expect(reads, 'no raw SQL and no engine aggregate for the object').toEqual(before); + }); + + it('a dataset dimension on a json field answers the same 400, naming the dataset dimension — no statement reaches the engine', async () => { + const before = { ...reads }; + const res = await post('query', { cube: DATASET, measures: ['row_count'], dimensions: ['meta_doc'] }); + expect(res.status, JSON.stringify(res.body)).toBe(400); + expect(res.body.error.code).toBe('INVALID_FIELD'); + expect(res.body.error.message).toContain(`Dimension 'meta_doc' on cube '${DATASET}' groups by field 'meta'`); + expect(res.body.error.message).toContain(ROUTE); + expect(reads, 'no raw SQL and no engine aggregate for the object').toEqual(before); + }); + + it('the dry-run door refuses the same dimension: no statement is built to show', async () => { + const res = await post('sql', { cube: CUBE, measures: ['count'], dimensions: ['meta'] }); + expect(res.status, JSON.stringify(res.body)).toBe(400); + expect(res.body.error.code).toBe('INVALID_FIELD'); + expect(res.body.error.message).toContain(`Dimension 'meta' on cube '${CUBE}'`); + }); + + it('CONTROL a text dimension is served unchanged by the native strategy: one group per value, counted', async () => { + const before = { ...reads }; + const res = await post('query', { cube: CUBE, measures: ['count'], dimensions: ['title'] }); + expect(res.status, JSON.stringify(res.body)).toBe(200); + const groups = (res.body.data.rows as Array<{ title: string; count: number | string }>) + .map((r) => [r.title, Number(r.count)] as const) + .sort(([a], [b]) => a.localeCompare(b)); + expect(groups).toEqual([['x', 2], ['y', 1]]); + expect(reads.rawSql - before.rawSql, 'the native strategy answered: one statement').toBe(1); + expect(reads.aggregate - before.aggregate, 'the engine aggregate was not asked').toBe(0); + + const ds = await post('query', { cube: DATASET, measures: ['row_count'], dimensions: ['title_dim'] }); + expect(ds.status, JSON.stringify(ds.body)).toBe(200); + const dsGroups = (ds.body.data.rows as Array<{ title_dim: string; row_count: number | string }>) + .map((r) => [r.title_dim, Number(r.row_count)] as const) + .sort(([a], [b]) => a.localeCompare(b)); + expect(dsGroups).toEqual([['x', 2], ['y', 1]]); + }); + }, + ); +} diff --git a/packages/services/service-analytics/src/__tests__/dimension-structured-json-door.test.ts b/packages/services/service-analytics/src/__tests__/dimension-structured-json-door.test.ts new file mode 100644 index 00000000000..fd0c7397b1f --- /dev/null +++ b/packages/services/service-analytics/src/__tests__/dimension-structured-json-door.test.ts @@ -0,0 +1,233 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * A GROUPED dimension on a structured-JSON field — `json`, `composite`, + * `repeater`, `record`, `location`, `address`, `vector` — is refused + * `INVALID_FIELD` / 400 at the analytics door, naming the member the caller + * wrote, before either strategy builds anything. + * + * What these pins hold, beside the HTTP pins on SQLite and PostgreSQL + * (`packages/runtime/src/analytics-json-dimension-door.test.ts` for + * `/api/v1/analytics/query` and `/sql`, + * `packages/rest/src/analytics-dataset-json-dimension-door.test.ts` for + * `/api/v1/analytics/dataset/query`): + * + * - one answer on BOTH strategy faces — `NativeSQLStrategy` (which compiled + * `GROUP BY` by hand and never reached the engine's own `groupBy` door) and + * `ObjectQLStrategy` (which reached it, and was refused there under the + * engine's `groupBy[0]` position rather than the member); + * - the member as the caller spelled it: a cube key that differs from its + * column, a cube-qualified spelling, a dataset dimension, an ad-hoc + * (inferred) cube, and a bucketed time dimension; + * - no statement and no aggregate reaches the host bridges on a refusal; + * - GUARD: the judged types are exactly `@objectstack/spec/data`'s + * `STRUCTURED_JSON_TYPES`, over every `FieldType` — the predicate the + * engine's door reads, never a second list. + */ + +import { describe, it, expect, vi } from 'vitest'; +import type { Cube } from '@objectstack/spec/data'; +import { FieldType, STRUCTURED_JSON_TYPES } from '@objectstack/spec/data'; +import type { Dataset } from '@objectstack/spec/ui'; +import type { AnalyticsQuery } from '@objectstack/spec/contracts'; +import { AnalyticsService } from '../analytics-service.js'; + +const silentLogger = { + info: vi.fn(), + debug: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + child: vi.fn().mockReturnThis(), +} as any; + +const OBJECT = 'ledger'; + +/** One field per `FieldType`, named `f_`, plus the text control `title`. */ +const FIELDS: Record = { + title: { type: 'text' }, + ...Object.fromEntries(FieldType.options.map((type) => [`f_${type}`, { type }])), +}; + +/** An authored cube: a text dimension, a json one filed under a key that is not its column, and a time one. */ +const LEDGER_CUBE: Cube = { + name: 'ledger_cube', + title: 'Ledger', + sql: OBJECT, + public: true, + measures: { count: { label: 'Rows', type: 'count', sql: '*' } }, + dimensions: { + title: { label: 'Title', type: 'string', sql: 'title' }, + doc: { label: 'Doc', type: 'string', sql: 'f_json' }, + stamped: { label: 'Stamped', type: 'time', sql: 'f_json', granularities: ['month'] }, + ...Object.fromEntries( + [...STRUCTURED_JSON_TYPES].map((type) => [`f_${type}`, { label: type, type: 'string' as const, sql: `f_${type}` }]), + ), + }, +} as Cube; + +const LEDGER_DATASET = { + name: 'ledger_ds', + label: 'Ledger dataset', + object: OBJECT, + dimensions: [ + { name: 'title_dim', field: 'title', type: 'string' }, + { name: 'meta_doc', field: 'f_json', type: 'string' }, + ], + measures: [{ name: 'row_count', aggregate: 'count' }], +} as unknown as Dataset; + +type Face = 'native' | 'objectql'; + +interface Refusal extends Error { + code?: string; + status?: number; + member?: string; + param?: string; + field?: string; + object?: string; + cube?: string; +} + +function makeService(face: Face, opts: { sourceFieldMeta?: boolean } = {}) { + const calls = { raw: [] as string[], aggregate: [] as unknown[] }; + const service = new AnalyticsService({ + logger: silentLogger, + cubes: [LEDGER_CUBE], + queryCapabilities: () => ({ nativeSql: face === 'native', objectqlAggregate: face === 'objectql', inMemory: false }), + executeRawSql: async (_object: string, sql: string) => { + calls.raw.push(sql); + return [{ title: 'x', count: 2 }]; + }, + executeAggregate: async (_object: string, options: unknown) => { + calls.aggregate.push(options); + return [{ title: 'x', count: 2 }]; + }, + isRegisteredObject: (n: string) => n === OBJECT, + getObjectFieldNames: (n: string) => (n === OBJECT ? Object.keys(FIELDS) : undefined), + ...(opts.sourceFieldMeta === false + ? {} + : { sourceFieldMeta: (o: string, f: string) => (o === OBJECT ? FIELDS[f] : undefined) }), + }); + return { service, calls }; +} + +/** The error a call rejected with — and a loud failure if it resolved. */ +async function rejection(call: Promise): Promise { + let resolved: unknown; + try { + resolved = await call; + } catch (e) { + return e as Refusal; + } + throw new Error(`expected a refusal, got ${JSON.stringify(resolved)}`); +} + +/** The ADR-0112 envelope plus the member the caller wrote. */ +function envelopeOf(err: Refusal) { + return { code: err.code, status: err.status, member: err.member, param: err.param, field: err.field, object: err.object }; +} + +describe('a dimension on a structured-JSON field is refused at the analytics door, on both strategy faces', () => { + for (const face of ['native', 'objectql'] as const) { + it(`${face}: every structured-JSON type answers INVALID_FIELD / 400 naming the member — no statement, no aggregate`, async () => { + const { service, calls } = makeService(face); + for (const type of STRUCTURED_JSON_TYPES) { + const member = `f_${type}`; + const err = await rejection(service.query({ cube: 'ledger_cube', measures: ['count'], dimensions: [member] })); + expect(envelopeOf(err), type).toEqual({ + code: 'INVALID_FIELD', status: 400, member, param: 'dimensions', field: member, object: OBJECT, + }); + expect(err.message, type).toContain(`Dimension '${member}' on cube 'ledger_cube' groups by field '${member}'`); + expect(err.message, type).toContain(`declares as ${type}`); + } + expect(calls.raw, 'no statement reached the raw-SQL bridge').toEqual([]); + expect(calls.aggregate, 'no aggregate reached the engine bridge').toEqual([]); + }); + } + + it('names the member as the caller spelled it: a cube key over another column, and the cube-qualified spelling', async () => { + const { service, calls } = makeService('native'); + const keyed = await rejection(service.query({ cube: 'ledger_cube', measures: ['count'], dimensions: ['title', 'doc'] })); + expect(envelopeOf(keyed)).toEqual({ code: 'INVALID_FIELD', status: 400, member: 'doc', param: 'dimensions', field: 'f_json', object: OBJECT }); + expect(keyed.message).toContain(`Dimension 'doc' on cube 'ledger_cube' groups by field 'f_json'`); + const qualified = await rejection(service.query({ cube: 'ledger_cube', measures: ['count'], dimensions: ['ledger_cube.doc'] })); + expect(envelopeOf(qualified)).toMatchObject({ code: 'INVALID_FIELD', status: 400, member: 'ledger_cube.doc', field: 'f_json' }); + expect(calls.raw).toEqual([]); + }); + + it('a bucketed time dimension on a json field is refused the same way, under timeDimensions — on the ObjectQL face too', async () => { + for (const face of ['native', 'objectql'] as const) { + const { service, calls } = makeService(face); + const err = await rejection(service.query({ + cube: 'ledger_cube', + measures: ['count'], + timeDimensions: [{ dimension: 'stamped', granularity: 'month' }], + })); + expect(envelopeOf(err), face).toEqual({ + code: 'INVALID_FIELD', status: 400, member: 'stamped', param: 'timeDimensions', field: 'f_json', object: OBJECT, + }); + expect(err.message, face).toContain(`Time dimension 'stamped' on cube 'ledger_cube' buckets field 'f_json'`); + expect(calls.aggregate, `${face}: the engine was not asked`).toEqual([]); + // …and the cube's declared default bucket makes a GROUPED time dimension out of a plain `dimensions` entry. + const defaulted = await rejection(service.query({ cube: 'ledger_cube', measures: ['count'], dimensions: ['stamped'] })); + expect(envelopeOf(defaulted), face).toMatchObject({ code: 'INVALID_FIELD', status: 400, member: 'stamped', param: 'dimensions' }); + } + }); + + it('a dataset dimension is refused naming the dataset dimension', async () => { + const { service, calls } = makeService('native'); + const err = await rejection(service.queryDataset(LEDGER_DATASET, { measures: ['row_count'], dimensions: ['meta_doc'] })); + expect(envelopeOf(err)).toEqual({ code: 'INVALID_FIELD', status: 400, member: 'meta_doc', param: 'dimensions', field: 'f_json', object: OBJECT }); + expect(err.message).toContain(`Dimension 'meta_doc' on cube 'ledger_ds' groups by field 'f_json'`); + expect(calls.raw).toEqual([]); + }); + + it('an ad-hoc query (no cube registered under the object name) is refused the same way', async () => { + const { service, calls } = makeService('native'); + const err = await rejection(service.query({ cube: OBJECT, measures: ['count'], dimensions: ['f_json'] })); + expect(envelopeOf(err)).toEqual({ code: 'INVALID_FIELD', status: 400, member: 'f_json', param: 'dimensions', field: 'f_json', object: OBJECT }); + expect(calls.raw).toEqual([]); + }); + + it('the dry-run door refuses before building the statement', async () => { + const { service } = makeService('native'); + const err = await rejection(service.generateSql({ cube: 'ledger_cube', measures: ['count'], dimensions: ['doc'] })); + expect(envelopeOf(err)).toMatchObject({ code: 'INVALID_FIELD', status: 400, member: 'doc' }); + }); +}); + +describe('what the door does not judge', () => { + it('CONTROL a text dimension is served — the native face runs its one statement', async () => { + const { service, calls } = makeService('native'); + const result = await service.query({ cube: 'ledger_cube', measures: ['count'], dimensions: ['title'] }); + expect(result.rows).toEqual([{ title: 'x', count: 2 }]); + expect(calls.raw).toHaveLength(1); + }); + + it('an unknown field keeps the existence gate\'s answer: that verdict comes first', async () => { + const { service } = makeService('native'); + const err = await rejection(service.query({ cube: OBJECT, measures: ['count'], dimensions: ['nope'] })); + expect({ code: err.code, status: err.status }).toEqual({ code: 'INVALID_FIELD', status: 400 }); + expect(err.message).toContain("which object 'ledger' does not have"); + }); + + it('a host that wires no sourceFieldMeta cannot name the type, so the door stands down', async () => { + const { service, calls } = makeService('native', { sourceFieldMeta: false }); + await service.query({ cube: 'ledger_cube', measures: ['count'], dimensions: ['doc'] }); + expect(calls.raw).toHaveLength(1); + }); + + it('GUARD the judged types are exactly the spec\'s STRUCTURED_JSON_TYPES, over every FieldType', async () => { + const { service } = makeService('native'); + for (const type of FieldType.options) { + const query: AnalyticsQuery = { cube: OBJECT, measures: ['count'], dimensions: [`f_${type}`] }; + const verdict = await service.generateSql(query).then( + () => null, + (e: Refusal) => ({ code: e.code, status: e.status, member: e.member }), + ); + expect(verdict, type).toEqual( + STRUCTURED_JSON_TYPES.has(type) ? { code: 'INVALID_FIELD', status: 400, member: `f_${type}` } : null, + ); + } + }); +}); From b1befe2a6bc384702adc7b0b89012d6bb9021467 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 14:31:58 +0000 Subject: [PATCH 2/3] fix(service-analytics): refuse a grouped dimension on a structured-JSON field at the analytics door A `dimensions` entry, or a bucketed `timeDimensions` entry, whose column is a declared structured-JSON field (the spec's STRUCTURED_JSON_TYPES, the class the engine's groupBy door reads) is refused INVALID_FIELD / 400 in `ensureCube`, naming the member the caller wrote, before either strategy builds anything. NativeSQLStrategy compiled GROUP BY by hand and never reached the engine's door: one group per serialized document on SQLite, 500 on PostgreSQL. The ObjectQL face reached the engine's door but was refused under `groupBy[0]`, a name the caller never wrote. Claude-Session: https://claude.ai/code/session_01XY5uCwTjZj7884yYtyur4H Co-authored-by: Claude --- .../src/analytics-service.ts | 46 ++++++ .../src/structured-json-dimension-door.ts | 134 ++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 packages/services/service-analytics/src/structured-json-dimension-door.ts diff --git a/packages/services/service-analytics/src/analytics-service.ts b/packages/services/service-analytics/src/analytics-service.ts index ce1c9d64876..f01a5b35885 100644 --- a/packages/services/service-analytics/src/analytics-service.ts +++ b/packages/services/service-analytics/src/analytics-service.ts @@ -83,6 +83,9 @@ import { evaluateAnalyticsQueryOverRows } from './preview-evaluator.js'; // member as the request spelled it (see `dataset-refusal.ts`'s header for why // that code and not `DATASET_INVALID`). import { invalidMemberError } from './dataset-refusal.js'; +// [#20807] A grouped dimension on a structured-JSON field is refused in +// `ensureCube`, ahead of both strategies, naming the member the caller wrote. +import { assertNoStructuredJsonDimension } from './structured-json-dimension-door.js'; // [#16206] The `sqlDialect` hook's DECLARED accept set, and the predicate that // says whether a host answered outside it. Both live next to the membership set // the compilers read, so the contract has one definition and this file states @@ -2373,6 +2376,12 @@ export class AnalyticsService implements IAnalyticsService { * where), so a query that gets several wrong is answered about one at a time, * naming a real mistake either way. * + * [#20807] One more door runs between the dimension gate and the `where` + * gate, on the same paths: {@link assertDimensionsGroupScalarColumns}, which + * refuses a grouped dimension whose column EXISTS but is declared + * structured JSON — a question about the column's type, not its existence, + * and so asked after it. + * * [#20356] "Registered" means registered in `scope`: the cube is read from it * and what this method mints is recorded in it — the call's own request * scope, on every door, so nothing minted here reaches the shared registry @@ -2401,6 +2410,8 @@ export class AnalyticsService implements IAnalyticsService { // spelling is in there — which is why the suggestion list is computed by // subtraction inside the gate rather than echoed verbatim. this.assertDimensionFields(query, cube, Object.keys(cube.dimensions)); + // [#20807] …and a grouped dimension's column must not be structured JSON. + this.assertDimensionsGroupScalarColumns(query, cube); // [#5669] …and the `where`'s, third and last of the three request keys that // carry a field name. Its members are read from the filter TREE, not from // `cube.dimensions` — which on this path was minted from this very query, @@ -2472,6 +2483,7 @@ export class AnalyticsService implements IAnalyticsService { // authored/compiled list IS the vocabulary a caller may name — and the one // the rejection suggests. this.assertDimensionFields(query, augmented, Object.keys(cube.dimensions)); + this.assertDimensionsGroupScalarColumns(query, augmented); // [#5669] The `where` gate resolves a filter member through dimensions AND // measures (that is what the strategies do for a filter member), so it is // handed the AUGMENTED cube — a caller filtering on a suffix-inferred @@ -2486,10 +2498,44 @@ export class AnalyticsService implements IAnalyticsService { // authored cube can declare a measure over a field the object dropped. this.assertMeasureFields(query, cube, Object.keys(cube.measures)); this.assertDimensionFields(query, cube, Object.keys(cube.dimensions)); + this.assertDimensionsGroupScalarColumns(query, cube); this.assertWhereFields(query, cube, Object.keys(cube.dimensions)); } } + /** + * [#20807] Refuse a GROUPED dimension whose column is a declared + * structured-JSON field — `INVALID_FIELD` / 400, naming the member the caller + * wrote — before either strategy builds anything. The rule, the + * measurements and the envelope are {@link assertNoStructuredJsonDimension}'s + * (`structured-json-dimension-door.ts`); this method supplies the two + * answers only the service has. + * + * - The column a member groups by is {@link resolveMemberSource}'s, the + * resolver the dimension source-field gate above reads, with the same + * `'dimension'` kind the strategies' `resolveDimensionSql` / + * `resolveFieldName(…, 'dimension')` resolve by. + * - Its declared type is {@link AnalyticsServiceConfig.sourceFieldMeta}'s. + * + * Runs after the dimension source-field gate on every `ensureCube` path, so a + * member naming a column the object does not have is answered as that first. + * Same stand-downs as that gate: no `sourceFieldMeta`, or a cube whose `sql` + * is not a bare object name. + */ + private assertDimensionsGroupScalarColumns(query: AnalyticsQuery, cube: Cube): void { + const fieldMeta = this.sourceFieldMeta; + if (!fieldMeta) return; + const object = typeof cube.sql === 'string' ? cube.sql.trim() : ''; + if (!object || !BARE_IDENTIFIER.test(object)) return; + assertNoStructuredJsonDimension( + query, + cube, + object, + (member) => resolveMemberSource(cube, member, 'dimension').source, + (o, field) => fieldMeta(o, field)?.type, + ); + } + /** * [#4437] Reject a measure whose SOURCE FIELD the backing object does not * have, BEFORE the strategy compiles it into SQL. diff --git a/packages/services/service-analytics/src/structured-json-dimension-door.ts b/packages/services/service-analytics/src/structured-json-dimension-door.ts new file mode 100644 index 00000000000..9eaa5e4debb --- /dev/null +++ b/packages/services/service-analytics/src/structured-json-dimension-door.ts @@ -0,0 +1,134 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20807] A GROUPED dimension on a STRUCTURED-JSON field — `json`, + * `composite`, `repeater`, `record`, `location`, `address`, `vector` — is + * refused `INVALID_FIELD` / 400 at the analytics door, naming the member the + * caller wrote, before either strategy builds anything. + * + * ## What ran before this door, measured on `origin/main` `793fb839` + * + * `POST /api/v1/analytics/query` over the service `AnalyticsServicePlugin` + * composes on a real engine, three rows with a different `meta` document each: + * + * | `dimensions` | SQLite | PostgreSQL 16 | + * |:--|:--|:--| + * | a `text` member (the control) | 200, one group per value | same | + * | a cube or dataset member over a `json` field | 200, one group per serialized document | **500 `DATABASE_ERROR`** ("could not identify an equality operator for type json") | + * + * `NativeSQLStrategy` compiled `GROUP BY meta` itself and ran it once through + * the raw-SQL bridge; `engine.aggregate` was asked 0 times, so the engine's own + * `groupBy` door (#20783, `packages/objectql`'s + * `group-by-structured-json-door.ts`) never saw the query. `ObjectQLStrategy` + * (a driver without raw SQL, or any bucketed time dimension) did reach that + * door, and was refused there under the engine's position (`groupBy[0]`), a + * name the caller never wrote. + * + * ## Where it stands + * + * In `AnalyticsService.ensureCube`, right after the dimension source-field + * gate, on every path out of it — so it runs ahead of strategy selection, for + * `query()` (the `/analytics/query` door, and every query a dataset selection + * runs through `DatasetExecutor`) and for `generateSql()` (the `/analytics/sql` + * dry run) alike. This is the first compile step that knows both halves of the + * verdict: the member the caller wrote (a `dimensions` entry, or a dataset + * dimension's name, which is the cube key it compiled to) and the declared type + * of the column it resolves to (`sourceFieldMeta`). One door ahead of both + * strategies gives one answer on every driver. + * + * ## What it judges + * + * - The members that GROUP: every `dimensions` entry, and every + * `timeDimensions` entry that carries a `granularity` (a bucket is a group + * key; no granularity makes a JSON document a date). A `timeDimensions` + * entry with no granularity only bounds a range and groups nothing. + * - The class is `@objectstack/spec/data`'s {@link STRUCTURED_JSON_TYPES}, + * the predicate the engine's door reads — never a list minted here. + * + * **Not judged** (the same "cannot answer, do not block" tiering as every + * sibling gate in `ensureCube`): a host that wires no `sourceFieldMeta`, a cube + * whose `sql` is not a bare object name, a member the resolver cannot pin to a + * bare base column (an expression `sql`, a dotted relation traversal: its + * column lives on a joined object that `sourceFieldMeta` does not answer for), + * and every other field type. + * + * ## The envelope + * + * `INVALID_FIELD` / 400 through {@link invalidMemberError} (ADR-0112): the + * verdict is about ONE MEMBER the request named, the family the three + * source-field gates and the engine's door already answer with + * `INVALID_FIELD`. `member` is the entry as the request spelled it; `field` is + * the column it groups by — it exists, and its declared type is the verdict — + * and `object` is the object that declares it. + * + * @see https://github.com/objectstack-ai/objectstack/issues/20807 + */ + +import type { Cube } from '@objectstack/spec/data'; +import { STRUCTURED_JSON_TYPES } from '@objectstack/spec/data'; +import type { AnalyticsQuery } from '@objectstack/spec/contracts'; +import { invalidMemberError } from './dataset-refusal.js'; + +/** One grouped member, tagged with the request key that carried it. */ +interface GroupedMember { + readonly member: string; + readonly param: 'dimensions' | 'timeDimensions'; +} + +/** + * The members of `query` that group the result, in request-key order: + * `dimensions` first, then each bucketed `timeDimensions` entry. + */ +function groupedMembers(query: AnalyticsQuery): GroupedMember[] { + return [ + ...(query.dimensions ?? []).map((member) => ({ member, param: 'dimensions' as const })), + ...(query.timeDimensions ?? []) + .filter((td) => !!td.granularity) + .map((td) => ({ member: td.dimension, param: 'timeDimensions' as const })), + ]; +} + +/** + * Refuse the first grouped member of `query` whose column is a declared + * structured-JSON field — `INVALID_FIELD` / 400. See the module header. + * + * @param object - The object `cube.sql` names (the caller has checked it is a + * bare object name). + * @param sourceOf - The bare base column a dimension member groups by, or + * `null` when this door cannot pin one (the dimension gate's resolver). + * @param declaredFieldType - The declared `FieldType` of a column on + * `object`, or `undefined` when nothing authoritative answers. + * + * The words put the verdict first, then that the query did not run, then the + * route, then the reason: a door that bounds a 4xx message keeps the front of + * it. + */ +export function assertNoStructuredJsonDimension( + query: AnalyticsQuery, + cube: Cube, + object: string, + sourceOf: (member: string) => string | null, + declaredFieldType: (object: string, field: string) => string | undefined, +): void { + for (const { member, param } of groupedMembers(query)) { + const field = sourceOf(member); + if (!field) continue; + const type = declaredFieldType(object, field); + if (typeof type !== 'string' || !STRUCTURED_JSON_TYPES.has(type)) continue; + + const kind = param === 'timeDimensions' ? 'Time dimension' : 'Dimension'; + const verb = param === 'timeDimensions' ? 'buckets' : 'groups by'; + const err = invalidMemberError( + `${kind} '${member}' on cube '${cube.name}' ${verb} field '${field}', which object '${object}' ` + + `declares as ${type} — a structured-JSON value, which analytics does not group by. ` + + 'The query was NOT run. Group by a field that stores one scalar value: store the part you ' + + 'group on in a field of its own and group by that field. A JSON document is no group key ' + + 'the SQL dialects share: one grouped each serialized document apart, another refused the ' + + 'statement.', + { member, param, cube: cube.name }, + ) as Error & { field?: string; object?: string }; + err.field = field; + err.object = object; + throw err; + } +} From 075a46340e2cfc9dd57804c26318dabbfe2d366e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 14:44:45 +0000 Subject: [PATCH 3/3] fix(service-analytics): judge a dotted dimension on the object its declared join names; changeset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dataset dimension over an included relationship (`account.hq`) reached NativeSQLStrategy unjudged: measured, SQLite 200 with one group per document and PostgreSQL 500, like a base-object one. The door now reads the column the way the strategy compiles it: a bare identifier on the cube's object, a dotted identifier path on the object the cube's declared join for that path names. A path with no declared join is a synthetic traversal and stays unjudged. Pins: the service door and the dataset door gain the joined case and its text control. Changeset: service-analytics minor, BREAKING, Clause-② no (narrowing), ADR-0087 not-required (no-migration-prescription). Claude-Session: https://claude.ai/code/session_01XY5uCwTjZj7884yYtyur4H Co-authored-by: Claude --- .../20807-analytics-json-dimension-refused.md | 19 ++++ ...lytics-dataset-json-dimension-door.test.ts | 59 +++++++++++-- .../dimension-structured-json-door.test.ts | 33 ++++++- .../src/analytics-service.ts | 15 ++-- .../src/structured-json-dimension-door.ts | 86 ++++++++++++++----- 5 files changed, 177 insertions(+), 35 deletions(-) create mode 100644 .changeset/20807-analytics-json-dimension-refused.md diff --git a/.changeset/20807-analytics-json-dimension-refused.md b/.changeset/20807-analytics-json-dimension-refused.md new file mode 100644 index 00000000000..7645f894e8a --- /dev/null +++ b/.changeset/20807-analytics-json-dimension-refused.md @@ -0,0 +1,19 @@ +--- +"@objectstack/service-analytics": minor +--- + +fix(service-analytics)!: a cube or dataset dimension on a structured-JSON field is refused with `INVALID_FIELD` / 400 at the analytics door, before any SQL is built + +Clause-②: no (narrowing) + + + +**BREAKING**: this narrows what the analytics query doors accept as a dimension. A cube dimension, or a dataset dimension, whose column is a declared field of the structured-JSON class (`json`, `composite`, `repeater`, `record`, `location`, `address`, `vector`) is refused before either strategy builds a statement, when it groups the result: a `dimensions` entry, or a `timeDimensions` entry with a `granularity`. The column is judged where it is declared: on the cube's object, or, for a dotted path such as a dataset dimension over `account.hq`, on the object the cube's declared join for that path names. It holds on `POST /api/v1/analytics/query`, on its dry run `POST /api/v1/analytics/sql`, and on `POST /api/v1/analytics/dataset/query`, on every driver. It ships as `minor` under the launch-window convention for accept-set narrowings. No export or published type changes. + +**What an author sees now.** `400 INVALID_FIELD`, naming the member as the request wrote it (the cube dimension, or the dataset dimension), the column it groups by, the object and the column's declared type, saying the query was not run, and naming the route: group by a field that stores one scalar value, storing the part of the document you group on in a field of its own. The thrown error carries `member`, `param` (`dimensions` or `timeDimensions`), `cube`, `field` and `object`. + +**Why a refusal.** A JSON document is no group key the SQL dialects share. Measured through `POST /api/v1/analytics/query` over three rows with a different document each, on the service `AnalyticsServicePlugin` composes over a real engine: SQLite answered 200 with one group per serialized document, and PostgreSQL 16 answered 500 `DATABASE_ERROR`. A dataset dimension over a joined object's `json` field answered the same two ways through `POST /api/v1/analytics/dataset/query`. The native-SQL strategy compiled the `GROUP BY` itself, so the engine's own refusal of a structured-JSON `groupBy` never saw the query; the engine-aggregate strategy did reach that refusal, but named the engine's `groupBy[0]` position rather than the member the caller wrote. The class is `@objectstack/spec/data`'s `STRUCTURED_JSON_TYPES`, the one the engine's refusal reads. No producer groups by such a field: no cube or dataset dimension in the example apps names one. + +**Who is affected.** A dashboard, report or caller that grouped an analytics query by a structured-JSON field on SQLite and read one group per serialized document as real groups. On PostgreSQL the same query was already a 500. + +**Unchanged.** A dimension on any other type; a `timeDimensions` entry with no `granularity`, which bounds a range and groups nothing; measures (this door judges only the members that group); a dotted dimension path the cube declares no join for, whose object is not a declaration; a member naming a column the object does not have, which keeps its existing `INVALID_FIELD` answer first; and a host that wires no `sourceFieldMeta`, where the column's type cannot be read. diff --git a/packages/rest/src/analytics-dataset-json-dimension-door.test.ts b/packages/rest/src/analytics-dataset-json-dimension-door.test.ts index bdb750f29a0..1f807a3fa4c 100644 --- a/packages/rest/src/analytics-dataset-json-dimension-door.test.ts +++ b/packages/rest/src/analytics-dataset-json-dimension-door.test.ts @@ -13,15 +13,19 @@ * the route this package serves: an INLINE dataset, compiled per request, * whose dimensions reach the same cube query through `DatasetExecutor`. * - * ## Measured on the base, through this door + * ## Measured without the refusal, through this door * - * Three rows, `title` x, x, y, and a different `meta` document per row; a - * dataset declaring `meta_doc` over the `json` field `meta`: + * On the base (the `meta_doc` row), and with the refusal ablated (the + * `acct_hq` row). Three rows, `title` x, x, y, and a different `meta` + * document per row; a dataset declaring `meta_doc` over the `json` field + * `meta`, and `acct_hq` over the `json` field `hq` of the object the + * `account` lookup references: * * | `selection.dimensions` | SQLite | PostgreSQL 16 | * |:--|:--|:--| * | `title_dim` (text, the control) | 200, `x` 2 · `y` 1 | same | * | `meta_doc` (json) | 200, one group per serialized document (3) | 500 | + * | `acct_hq` (`account.hq`, a json field of the `include`d object) | 200, one group per document (2) | 500 | * * ## The composition, and the dialect axis of THIS file * @@ -42,6 +46,17 @@ import { AnalyticsServicePlugin, type AnalyticsService } from '@objectstack/serv import { RestServer } from './rest-server'; const OBJECT = 'rest_dataset_json_dim_ledger'; +/** The object the ledger's `account` lookup references — joined through the dataset's `include`. */ +const ACCOUNT = 'rest_dataset_json_dim_account'; + +const ACCOUNT_OBJECT = { + name: ACCOUNT, + label: 'Dataset JSON dimension account', + fields: { + name: { name: 'name', type: 'text' as const }, + hq: { name: 'hq', type: 'json' as const }, + }, +}; const LEDGER = { name: OBJECT, @@ -49,13 +64,19 @@ const LEDGER = { fields: { title: { name: 'title', type: 'text' as const }, meta: { name: 'meta', type: 'json' as const }, + account: { name: 'account', type: 'lookup' as const, reference: ACCOUNT }, }, }; +const ACCOUNTS = [ + { id: 'a1', name: 'A', hq: { city: 'Paris' } }, + { id: 'a2', name: 'B', hq: { city: 'Rome' } }, +]; + const ROWS = [ - { id: 'r1', title: 'x', meta: { a: 1 } }, - { id: 'r2', title: 'x', meta: { a: 2 } }, - { id: 'r3', title: 'y', meta: { b: 1 } }, + { id: 'r1', title: 'x', meta: { a: 1 }, account: 'a1' }, + { id: 'r2', title: 'x', meta: { a: 2 }, account: 'a1' }, + { id: 'r3', title: 'y', meta: { b: 1 }, account: 'a2' }, ]; /** The inline dataset the request carries — as a Studio preview or a widget posts it. */ @@ -63,9 +84,12 @@ const DATASET = { name: 'json_dim_inline', label: 'JSON dimension inline', object: OBJECT, + include: ['account'], dimensions: [ { name: 'title_dim', field: 'title', type: 'string' }, { name: 'meta_doc', field: 'meta', type: 'string' }, + { name: 'acct_name', field: 'account.name', type: 'string' }, + { name: 'acct_hq', field: 'account.hq', type: 'string' }, ], measures: [{ name: 'row_count', aggregate: 'count' }], }; @@ -130,7 +154,7 @@ for (const cell of CELLS) { const dropTables = async () => { if (cell.id === 'sqlite') return; - await driver?.execute(`drop table if exists ${OBJECT}`).catch(() => {}); + for (const table of [OBJECT, ACCOUNT]) await driver?.execute(`drop table if exists ${table}`).catch(() => {}); }; beforeAll(async () => { @@ -139,8 +163,10 @@ for (const cell of CELLS) { engine = new ObjectQL({ logger: quiet } as any); engine.registerDriver(driver, true); await engine.init(); + engine.registry.registerObject(ACCOUNT_OBJECT as any); engine.registry.registerObject(LEDGER as any); await engine.syncSchemas(); + for (const row of ACCOUNTS) await engine.insert(ACCOUNT, { ...row } as any); for (const row of ROWS) await engine.insert(OBJECT, { ...row } as any); const realExecute = (engine as any).execute.bind(engine); @@ -200,6 +226,18 @@ for (const cell of CELLS) { expect(reads, 'no raw SQL and no engine aggregate for the object').toEqual(before); }); + it('a dataset dimension over an included relationship\'s json field answers the same 400, naming the joined object — no statement reaches the engine', async () => { + const before = { ...reads }; + const res = await query({ measures: ['row_count'], dimensions: ['acct_hq'] }); + expect(res.status, JSON.stringify(res.body)).toBe(400); + expect(res.body.code).toBe('INVALID_FIELD'); + expect(String(res.body.message)).toContain( + `Dimension 'acct_hq' on cube '${DATASET.name}' groups by field 'account.hq', whose column 'hq' the joined object '${ACCOUNT}' declares as json`, + ); + expect(String(res.body.message)).toContain(ROUTE); + expect(reads, 'no raw SQL and no engine aggregate for the object').toEqual(before); + }); + it('CONTROL a text dataset dimension is served unchanged: one group per value, counted', async () => { const before = { ...reads }; const res = await query({ measures: ['row_count'], dimensions: ['title_dim'] }); @@ -209,6 +247,13 @@ for (const cell of CELLS) { .sort(([a], [b]) => a.localeCompare(b)); expect(groups).toEqual([['x', 2], ['y', 1]]); expect(reads.rawSql - before.rawSql, 'the native strategy answered').toBeGreaterThanOrEqual(1); + + const joined = await query({ measures: ['row_count'], dimensions: ['acct_name'] }); + expect(joined.status, JSON.stringify(joined.body)).toBe(200); + const joinedGroups = (joined.body.rows as Array<{ acct_name: string; row_count: number | string }>) + .map((r) => [r.acct_name, Number(r.row_count)] as const) + .sort(([a], [b]) => a.localeCompare(b)); + expect(joinedGroups).toEqual([['A', 2], ['B', 1]]); }); }, ); diff --git a/packages/services/service-analytics/src/__tests__/dimension-structured-json-door.test.ts b/packages/services/service-analytics/src/__tests__/dimension-structured-json-door.test.ts index fd0c7397b1f..a92940cee0a 100644 --- a/packages/services/service-analytics/src/__tests__/dimension-structured-json-door.test.ts +++ b/packages/services/service-analytics/src/__tests__/dimension-structured-json-door.test.ts @@ -42,12 +42,20 @@ const silentLogger = { const OBJECT = 'ledger'; -/** One field per `FieldType`, named `f_`, plus the text control `title`. */ +/** One field per `FieldType`, named `f_`, plus the text control `title` and a lookup to `account`. */ const FIELDS: Record = { title: { type: 'text' }, + account: { type: 'lookup' }, ...Object.fromEntries(FieldType.options.map((type) => [`f_${type}`, { type }])), }; +/** The joined object: a text column and a json one. */ +const JOINED = 'account'; +const JOINED_FIELDS: Record = { + name: { type: 'text' }, + hq: { type: 'json' }, +}; + /** An authored cube: a text dimension, a json one filed under a key that is not its column, and a time one. */ const LEDGER_CUBE: Cube = { name: 'ledger_cube', @@ -69,9 +77,12 @@ const LEDGER_DATASET = { name: 'ledger_ds', label: 'Ledger dataset', object: OBJECT, + include: ['account'], dimensions: [ { name: 'title_dim', field: 'title', type: 'string' }, { name: 'meta_doc', field: 'f_json', type: 'string' }, + { name: 'acct_name', field: 'account.name', type: 'string' }, + { name: 'acct_hq', field: 'account.hq', type: 'string' }, ], measures: [{ name: 'row_count', aggregate: 'count' }], } as unknown as Dataset; @@ -106,7 +117,7 @@ function makeService(face: Face, opts: { sourceFieldMeta?: boolean } = {}) { getObjectFieldNames: (n: string) => (n === OBJECT ? Object.keys(FIELDS) : undefined), ...(opts.sourceFieldMeta === false ? {} - : { sourceFieldMeta: (o: string, f: string) => (o === OBJECT ? FIELDS[f] : undefined) }), + : { sourceFieldMeta: (o: string, f: string) => (o === OBJECT ? FIELDS[f] : o === JOINED ? JOINED_FIELDS[f] : undefined) }), }); return { service, calls }; } @@ -182,6 +193,17 @@ describe('a dimension on a structured-JSON field is refused at the analytics doo expect(calls.raw).toEqual([]); }); + it('a dataset dimension over an included relationship is judged on the JOINED object — naming the dataset dimension', async () => { + const { service, calls } = makeService('native'); + const err = await rejection(service.queryDataset(LEDGER_DATASET, { measures: ['row_count'], dimensions: ['acct_hq'] })); + expect(envelopeOf(err)).toEqual({ code: 'INVALID_FIELD', status: 400, member: 'acct_hq', param: 'dimensions', field: 'account.hq', object: JOINED }); + expect(err.message).toContain(`Dimension 'acct_hq' on cube 'ledger_ds' groups by field 'account.hq', whose column 'hq' the joined object '${JOINED}' declares as json`); + expect(calls.raw).toEqual([]); + // CONTROL the joined text column is served. + await service.queryDataset(LEDGER_DATASET, { measures: ['row_count'], dimensions: ['acct_name'] }); + expect(calls.raw).toHaveLength(1); + }); + it('an ad-hoc query (no cube registered under the object name) is refused the same way', async () => { const { service, calls } = makeService('native'); const err = await rejection(service.query({ cube: OBJECT, measures: ['count'], dimensions: ['f_json'] })); @@ -211,6 +233,13 @@ describe('what the door does not judge', () => { expect(err.message).toContain("which object 'ledger' does not have"); }); + it('a dotted path the cube declares no join for is a synthetic traversal: its object is not a declaration, so it is not judged', async () => { + const { service, calls } = makeService('native'); + await service.generateSql({ cube: 'ledger_cube', measures: ['count'], dimensions: ['account.hq'] }); + await service.query({ cube: 'ledger_cube', measures: ['count'], dimensions: ['account.hq'] }); + expect(calls.raw).toHaveLength(1); + }); + it('a host that wires no sourceFieldMeta cannot name the type, so the door stands down', async () => { const { service, calls } = makeService('native', { sourceFieldMeta: false }); await service.query({ cube: 'ledger_cube', measures: ['count'], dimensions: ['doc'] }); diff --git a/packages/services/service-analytics/src/analytics-service.ts b/packages/services/service-analytics/src/analytics-service.ts index f01a5b35885..bcbc71ca3bc 100644 --- a/packages/services/service-analytics/src/analytics-service.ts +++ b/packages/services/service-analytics/src/analytics-service.ts @@ -2511,10 +2511,11 @@ export class AnalyticsService implements IAnalyticsService { * (`structured-json-dimension-door.ts`); this method supplies the two * answers only the service has. * - * - The column a member groups by is {@link resolveMemberSource}'s, the - * resolver the dimension source-field gate above reads, with the same - * `'dimension'` kind the strategies' `resolveDimensionSql` / - * `resolveFieldName(…, 'dimension')` resolve by. + * - The dimension `sql` a member resolves to is {@link declaredMemberEntry}'s + * (`cube.dimensions` only, the `'dimension'` kind the strategies' + * `resolveDimensionSql` / `resolveFieldName(…, 'dimension')` resolve by), + * and the member itself when the cube declares none — the column the + * strategies group by in that case. * - Its declared type is {@link AnalyticsServiceConfig.sourceFieldMeta}'s. * * Runs after the dimension source-field gate on every `ensureCube` path, so a @@ -2531,7 +2532,11 @@ export class AnalyticsService implements IAnalyticsService { query, cube, object, - (member) => resolveMemberSource(cube, member, 'dimension').source, + (member) => { + const entry = declaredMemberEntry(cube, member, 'dimension'); + if (!entry) return member; + return typeof entry.sql === 'string' ? entry.sql : ''; + }, (o, field) => fieldMeta(o, field)?.type, ); } diff --git a/packages/services/service-analytics/src/structured-json-dimension-door.ts b/packages/services/service-analytics/src/structured-json-dimension-door.ts index 9eaa5e4debb..47ce2b2acc1 100644 --- a/packages/services/service-analytics/src/structured-json-dimension-door.ts +++ b/packages/services/service-analytics/src/structured-json-dimension-door.ts @@ -45,12 +45,21 @@ * - The class is `@objectstack/spec/data`'s {@link STRUCTURED_JSON_TYPES}, * the predicate the engine's door reads — never a list minted here. * + * - The column is read the way `NativeSQLStrategy` compiles it: the member's + * dimension `sql` (the member itself when the cube declares none). A bare + * identifier is a column of the cube's object. A dotted identifier path + * (`account.hq`, a dataset dimension over an `include`d relationship) is the + * last segment, on the object the cube's own `joins` entry for that path + * names — the alias the dataset compiler registers and the strategy joins. + * Measured on the base the same way as the table above: a dataset dimension + * over `account.hq` (a `json` field of the joined object) answered one group + * per document on SQLite and 500 on PostgreSQL, like a base-object one. + * * **Not judged** (the same "cannot answer, do not block" tiering as every * sibling gate in `ensureCube`): a host that wires no `sourceFieldMeta`, a cube - * whose `sql` is not a bare object name, a member the resolver cannot pin to a - * bare base column (an expression `sql`, a dotted relation traversal: its - * column lives on a joined object that `sourceFieldMeta` does not answer for), - * and every other field type. + * whose `sql` is not a bare object name, an expression `sql`, a dotted path + * the cube declares no join for (a synthetic traversal: which object it lands + * on is the strategy's guess, not a declaration), and every other field type. * * ## The envelope * @@ -58,8 +67,9 @@ * verdict is about ONE MEMBER the request named, the family the three * source-field gates and the engine's door already answer with * `INVALID_FIELD`. `member` is the entry as the request spelled it; `field` is - * the column it groups by — it exists, and its declared type is the verdict — - * and `object` is the object that declares it. + * the column it groups by, spelled as the dimension's `sql` spells it + * (`account.hq` for a joined one) — it exists, and its declared type is the + * verdict — and `object` is the object that declares it. * * @see https://github.com/objectstack-ai/objectstack/issues/20807 */ @@ -75,6 +85,21 @@ interface GroupedMember { readonly param: 'dimensions' | 'timeDimensions'; } +/** The column a grouped member reads: where it is declared, and how the dimension spells it. */ +interface DimensionColumn { + /** The object that declares the column. */ + readonly object: string; + /** The column's name on that object. */ + readonly column: string; + /** The dimension's `sql` as written: the column, or the dotted path to it. */ + readonly path: string; +} + +/** A bare identifier: one column. */ +const BARE_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/; +/** A dotted identifier path: relationship hops, then one column (`NativeSQLStrategy`'s own test). */ +const IDENTIFIER_PATH = /^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)+$/; + /** * The members of `query` that group the result, in request-key order: * `dimensions` first, then each bucketed `timeDimensions` entry. @@ -88,16 +113,32 @@ function groupedMembers(query: AnalyticsQuery): GroupedMember[] { ]; } +/** + * The column a dimension `sql` reads, or `null` when this door cannot pin one. + * A dotted path's object is the cube's DECLARED join at that path — the alias + * is the path with its dots as `__`, as the dataset compiler registers it and + * the strategy joins it — and a path with no declared join is not judged. + */ +function columnOf(cube: Cube, baseObject: string, sql: string): DimensionColumn | null { + const path = sql.trim(); + if (BARE_IDENTIFIER.test(path)) return { object: baseObject, column: path, path }; + if (!IDENTIFIER_PATH.test(path)) return null; + const segments = path.split('.'); + const column = segments.pop() as string; + const joined = (cube.joins as Record | undefined)?.[segments.join('__')]?.name; + return typeof joined === 'string' && joined !== '' ? { object: joined, column, path } : null; +} + /** * Refuse the first grouped member of `query` whose column is a declared * structured-JSON field — `INVALID_FIELD` / 400. See the module header. * - * @param object - The object `cube.sql` names (the caller has checked it is a - * bare object name). - * @param sourceOf - The bare base column a dimension member groups by, or - * `null` when this door cannot pin one (the dimension gate's resolver). - * @param declaredFieldType - The declared `FieldType` of a column on - * `object`, or `undefined` when nothing authoritative answers. + * @param baseObject - The object `cube.sql` names (the caller has checked it + * is a bare object name). + * @param sqlOf - The dimension `sql` a member resolves to, or the member itself + * when the cube declares none — the strategies' own lookup. + * @param declaredFieldType - The declared `FieldType` of a column on an + * object, or `undefined` when nothing authoritative answers. * * The words put the verdict first, then that the query did not run, then the * route, then the reason: a door that bounds a 4xx message keeps the front of @@ -106,29 +147,32 @@ function groupedMembers(query: AnalyticsQuery): GroupedMember[] { export function assertNoStructuredJsonDimension( query: AnalyticsQuery, cube: Cube, - object: string, - sourceOf: (member: string) => string | null, + baseObject: string, + sqlOf: (member: string) => string, declaredFieldType: (object: string, field: string) => string | undefined, ): void { for (const { member, param } of groupedMembers(query)) { - const field = sourceOf(member); - if (!field) continue; - const type = declaredFieldType(object, field); + const target = columnOf(cube, baseObject, sqlOf(member)); + if (!target) continue; + const type = declaredFieldType(target.object, target.column); if (typeof type !== 'string' || !STRUCTURED_JSON_TYPES.has(type)) continue; const kind = param === 'timeDimensions' ? 'Time dimension' : 'Dimension'; const verb = param === 'timeDimensions' ? 'buckets' : 'groups by'; + const declarer = target.path === target.column + ? `which object '${target.object}' declares` + : `whose column '${target.column}' the joined object '${target.object}' declares`; const err = invalidMemberError( - `${kind} '${member}' on cube '${cube.name}' ${verb} field '${field}', which object '${object}' ` - + `declares as ${type} — a structured-JSON value, which analytics does not group by. ` + `${kind} '${member}' on cube '${cube.name}' ${verb} field '${target.path}', ${declarer} ` + + `as ${type} — a structured-JSON value, which analytics does not group by. ` + 'The query was NOT run. Group by a field that stores one scalar value: store the part you ' + 'group on in a field of its own and group by that field. A JSON document is no group key ' + 'the SQL dialects share: one grouped each serialized document apart, another refused the ' + 'statement.', { member, param, cube: cube.name }, ) as Error & { field?: string; object?: string }; - err.field = field; - err.object = object; + err.field = target.path; + err.object = target.object; throw err; } }