diff --git a/bun.lock b/bun.lock index 612a5f6..db92323 100644 --- a/bun.lock +++ b/bun.lock @@ -16,6 +16,7 @@ "@types/node": "^22.10.2", "effect": "4.0.0", "expect-type": "^1.3.0", + "fast-check": "^4.9.0", "tsdown": "^0.22.14", "typescript": "^7.0.2", "vitest": "^4.1.9", @@ -281,6 +282,8 @@ "expect-type": ["expect-type@1.4.0", "", {}, "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA=="], + "fast-check": ["fast-check@4.10.2", "", { "dependencies": { "pure-rand": "^8.0.0" } }, "sha512-iK2f+YrcmoeGqk6fA0ea2bptcu/itMIm4NfEozq6N25+aG6h7s5HZbB/k1aV7b5w5sFLMCbbtRUsTVR+BgC3xw=="], + "fast-string-truncated-width": ["fast-string-truncated-width@3.0.3", "", {}, "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g=="], "fast-string-width": ["fast-string-width@3.0.2", "", { "dependencies": { "fast-string-truncated-width": "^3.0.2" } }, "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg=="], @@ -347,6 +350,8 @@ "postcss": ["postcss@8.5.28", "", { "dependencies": { "nanoid": "^3.3.18", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A=="], + "pure-rand": ["pure-rand@8.4.2", "", {}, "sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng=="], + "quansync": ["quansync@1.0.0", "", {}, "sha512-5xZacEEufv3HSTPQuchrvV6soaiACMFnq1H8wkVioctoH3TRha9Sz66lOxRwPK/qZj7HPiSveih9yAyh98gvqA=="], "redis": ["redis@6.3.0", "", { "dependencies": { "@redis/bloom": "6.3.0", "@redis/client": "6.3.0", "@redis/json": "6.3.0", "@redis/search": "6.3.0", "@redis/time-series": "6.3.0" } }, "sha512-XFQbPie1lGpKeUZ8ySYY43yLnQy/iIQtHNvIr/1XdaABpb96rUSdm3Mfl+98FEVbv9ZJIzINMno5s0Nh8E5LEw=="], diff --git a/docs/testing.md b/docs/testing.md index bc3a5dd..eb74994 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -93,6 +93,42 @@ or exempted with a reason. Tests preserve documented behavior: arithmetic chains follow SQL precedence, not call order, and `windowFunnel` with `strict_order` rejects intervening events. +## Fuzzing + +`tests/fuzz/` generates random inputs with fast-check and checks them against oracles that do +not trust the builder: + +- `literals.fuzz.test.ts`: a random value of each column type reaches the database twice. One + copy goes through the database's own JSON parser as the reference, the other through the + builder, both as a comparison literal and as a param in the SELECT list. The two must agree. +- `queries.fuzz.test.ts`: random query plans (joins, nested conditions, grouping, having, + ordering, paging, subqueries) run on the core fixture and must match `model.ts`, a JS + evaluator with SQL three-valued logic. A plan that reports `single-tenant` must have read + one org's rows only. +- `compile.fuzz.test.ts`: needs no database. Valid plans always compile, compile + deterministically, leak no `undefined`, `NaN` or `__PARAM_`, and number Postgres + placeholders `$1..$n`. Any JS value placed where a value goes is compiled or refused with a + `QueryBuilderError`, never thrown and never a defect. + +`bun run test` runs a small budget (a few seconds). Use `bun run fuzz` for longer runs: + +```sh +FUZZ_SCALE=50 bun run fuzz # 50× every property's default run count +FUZZ_RUNS=10000 bun run fuzz -t "plans match the model" +FUZZ_VERBOSE=1 bun run fuzz # print checked / skipped counts per property +``` + +A failure prints the shrunk input, the SQL it compiled to, the database's reply, and a +`FUZZ_SEED=… FUZZ_PATH=…` command that replays it. Pin the input in +`tests/fuzz/regressions.ts`, which is replayed before any random input. + +Bugs found but not yet fixed live in `tests/fuzz/known-issues.ts`. Each entry recognises +the inputs that reach its bug and gives a reason, so the suite stays green while the rest of +the input space keeps being fuzzed. After fixing a bug, delete its entry and run with +`FUZZ_STRICT=1`, which ignores every entry, to confirm nothing else matched it. Differences +between the databases that are not bugs are encoded in the model instead, each with the +difference it represents. + ## Version and CI policy Every push to `main` and every pull request runs the complete release check against diff --git a/package.json b/package.json index 53e7097..12547cc 100644 --- a/package.json +++ b/package.json @@ -108,6 +108,7 @@ "test": "vitest run && node scripts/check-doc-citations.mjs && node scripts/check-exports-documented.mjs && node scripts/check-doc-examples.mjs", "prepublishOnly": "bun run test:release", "test:clickhouse": "bun run build && bun scripts/test-clickhouse.ts", + "fuzz": "bun run build && vitest run tests/fuzz", "test:package": "bun run build && bun scripts/check-package.ts", "test:release": "bun scripts/test-clickhouse.ts --release", "changeset": "changeset", @@ -126,6 +127,7 @@ "@types/node": "^22.10.2", "effect": "4.0.0", "expect-type": "^1.3.0", + "fast-check": "^4.9.0", "tsdown": "^0.22.14", "typescript": "^7.0.2", "vitest": "^4.1.9" diff --git a/tests/core-cases.ts b/tests/core-cases.ts index 595f296..3698368 100644 --- a/tests/core-cases.ts +++ b/tests/core-cases.ts @@ -88,7 +88,7 @@ const fixtureColumns = { } satisfies Record type Value = string | number | null -const fixtureRows: Record>> = { +export const fixtureRows: Record>> = { orders: [ ["org_1", 1, "acme", 10, "paid", "rush", "2026-01-01 00:00:00.000"], ["org_1", 2, "acme", 20, "paid", null, "2026-01-01 00:10:00.000"], diff --git a/tests/fuzz/build.ts b/tests/fuzz/build.ts new file mode 100644 index 0000000..c0e4ebd --- /dev/null +++ b/tests/fuzz/build.ts @@ -0,0 +1,201 @@ +// A plan as builder calls, for either dialect. +// +// The builder is driven through its public, typed surface exactly as an +// application would; `any` here only erases the static types a random plan +// cannot have. Params are named up front from the plan's structure so that +// compiling the same plan twice resolves the same names to the same values. +import { DateTime, Effect, Exit } from "effect" +import * as CH from "@maple-dev/effect-orm/clickhouse" +import * as PG from "@maple-dev/effect-orm/postgres" +import type { CoreContext } from "../core-cases" +import type { Agg, Cond, Expr, Operand, Plan } from "./plan" + +const pad = (n: number, width = 2) => String(n).padStart(width, "0") +/** `YYYY-MM-DD hh:mm:ss.sss` in UTC: the string form a DateTime comparison accepts. */ +export const timeString = (ms: number): string => { + const d = new Date(ms) + return `${pad(d.getUTCFullYear(), 4)}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}.${pad(d.getUTCMilliseconds(), 3)}` +} + +/** Every param operand in the plan, in a fixed walk order, with the name it compiles as. */ +const nameParams = (plan: Plan): { readonly names: Map; readonly params: Record } => { + const names = new Map() + const params: Record = { orgId: plan.orgId } + const operand = (op: Operand) => { + if (op.k === "expr") return expr(op.e) + if (op.k !== "param") return + const name = `p${names.size}` + names.set(op, name) + params[name] = op.kind === "dateTime" ? DateTime.makeUnsafe(op.v as number) : op.v + } + const expr = (e: Expr): void => { + if (e.k === "arith") { + expr(e.l) + if (typeof e.r !== "number") expr(e.r) + } else if (e.k === "coalesce") { + expr(e.e) + expr(e.fallback) + } + } + const cond = (c: Cond): void => { + switch (c.k) { + case "cmp": + expr(c.l) + return operand(c.r) + case "like": + case "in": + return expr(c.k === "like" ? c.l : c.e) + case "null": + return expr(c.e) + case "between": + expr(c.e) + operand(c.lo) + return operand(c.hi) + case "and": + case "or": + return c.cs.forEach(cond) + case "not": + return cond(c.c) + } + } + const agg = (a: Agg) => (a.k === "countIf" ? cond(a.c) : a.k === "count" ? undefined : expr(a.e)) + plan.where.forEach(cond) + if (plan.shape.k === "rows") plan.shape.select.forEach(([, e]) => expr(e)) + else { + plan.shape.keys.forEach(([, e]) => expr(e)) + plan.shape.aggs.forEach(([, a]) => agg(a)) + plan.shape.having.forEach((h) => agg(h.agg)) + } + return { names, params } +} + +export interface Built { + readonly query: CH.CHQuery + readonly params: Record +} + +export const buildPlan = (ctx: CoreContext, plan: Plan): Built => { + const { names, params } = nameParams(plan) + + const expr = ($: any, e: Expr): any => { + switch (e.k) { + case "col": + return $[e.col] + case "jcol": + return $.c[e.col] + case "num": + case "str": + return CH.lit(e.v as never) + case "arith": + return expr($, e.l)[e.op](typeof e.r === "number" ? e.r : expr($, e.r)) + case "coalesce": + return ctx.fn.coalesce(expr($, e.e), expr($, e.fallback)) + } + } + const operand = ($: any, op: Operand): any => { + switch (op.k) { + case "value": + return op.v + case "time": + return op.form === "DateTime" ? DateTime.makeUnsafe(op.ms) : op.form === "Date" ? new Date(op.ms) : timeString(op.ms) + case "param": + return (CH.param as any)[op.kind](names.get(op)!) + case "expr": + return expr($, op.e) + } + } + const cond = ($: any, c: Cond): any => { + switch (c.k) { + case "cmp": + return expr($, c.l)[c.op](operand($, c.r)) + case "like": + return expr($, c.l)[c.op](c.pattern) + case "null": + return expr($, c.e)[c.op]() + case "between": + return expr($, c.e)[c.neg ? "notBetween" : "between"](operand($, c.lo), operand($, c.hi)) + case "in": + return expr($, c.e)[c.neg ? "notIn" : "in_"](...c.values) + case "and": + case "or": { + const parts = c.cs.map((x) => cond($, x)) + if (c.style === "fn") return (c.k === "and" ? CH.and : CH.or)(...parts) + return parts.slice(1).reduce((acc, part) => acc[c.k](part), parts[0]) + } + case "not": + return CH.not(cond($, c.c)) + case "const": + return CH.rawCond(c.v ? "1 = 1" : "1 = 0") + case "tenant": + return (c.side === "orders" ? $ : $.c).OrgId.eq(CH.param.string("orgId")) + } + } + const agg = ($: any, a: Agg): any => { + switch (a.k) { + case "count": + return ctx.fn.count() + case "countIf": + return ctx.fn.countIf(cond($, a.c)) + case "sum": + case "min": + case "max": + return ctx.fn[a.k](expr($, a.e)) + } + } + + let q: any = ctx.from(ctx.orders) + if (plan.join !== undefined) { + q = q[plan.join === "inner" ? "innerJoin" : "leftJoin"](ctx.customers, "c", (o: any, c: any) => + o.Customer.eq(c.Name).and(o.OrgId.eq(c.OrgId)), + ) + } + const shape = plan.shape + if (shape.k === "rows") { + q = q.select(($: any) => Object.fromEntries(shape.select.map(([alias, e]) => [alias, expr($, e)]))) + if (shape.distinct) q = q.distinct() + } else { + q = q.select(($: any) => ({ + ...Object.fromEntries(shape.keys.map(([alias, e]) => [alias, expr($, e)])), + ...Object.fromEntries(shape.aggs.map(([alias, a]) => [alias, agg($, a)])), + })) + if (shape.keys.length > 0) q = q.groupBy(...shape.keys.map(([alias]) => alias)) + if (shape.having.length > 0) q = q.having(($: any) => shape.having.map((h) => agg($, h.agg)[h.op](h.v))) + } + if (plan.where.length > 0) q = q.where(($: any) => plan.where.map((c) => cond($, c))) + if (shape.orderBy.length > 0) q = q.orderBy(...shape.orderBy.map(([alias, dir]) => [alias, dir])) + if (shape.limit !== undefined) q = q.limit(shape.limit) + if (shape.offset !== undefined) q = q.offset(shape.offset) + if (plan.crossTenant) q = q.crossTenant() + if (plan.wrap === "all") q = CH.fromQuery(q, "w").select() + if (plan.wrap === "names") { + const names = shape.k === "rows" ? shape.select.map(([alias]) => alias) : [...shape.keys, ...shape.aggs].map(([alias]) => alias) + q = CH.fromQuery(q, "w").select(...names) + } + return { query: q, params } +} + +export type Compiled = + | { readonly _tag: "Compiled"; readonly compiled: CH.CompiledQuery } + | { readonly _tag: "Refused"; readonly error: string } + | { readonly _tag: "Defect"; readonly error: string } + | { readonly _tag: "Threw"; readonly error: string } + +/** + * Compile through the Effect API and classify the outcome. A typed + * `QueryBuilderError` is `Refused`; a die is a `Defect`; an exception that + * escapes `compile` — or the builder calls before it — is `Threw`. + */ +export const compilePlan = (ctx: CoreContext, plan: Plan): Compiled => { + try { + const { query, params } = buildPlan(ctx, plan) + const compile = ctx.dialect === "clickhouse" ? CH.compile : PG.compile + const exit = Effect.runSyncExit((compile as any)(query, params) as Effect.Effect, unknown>) + if (Exit.isSuccess(exit)) return { _tag: "Compiled", compiled: exit.value } + const failure = exit.cause.reasons.find((reason) => reason._tag === "Fail") + return failure !== undefined + ? { _tag: "Refused", error: String((failure as any).error?.message ?? failure) } + : { _tag: "Defect", error: String(exit.cause) } + } catch (error) { + return { _tag: "Threw", error: error instanceof Error ? `${error.name}: ${error.message}\n${error.stack}` : String(error) } + } +} diff --git a/tests/fuzz/compile.fuzz.test.ts b/tests/fuzz/compile.fuzz.test.ts new file mode 100644 index 0000000..a033c79 --- /dev/null +++ b/tests/fuzz/compile.fuzz.test.ts @@ -0,0 +1,141 @@ +// Compile-time properties: no database, so these run thousands of plans a second. +// +// plans compile cleanly a well-typed plan is never refused, and nothing throws +// deterministic compiling the same plan twice gives the same SQL and params +// no leaked JS no `undefined`, `[object …]`, `NaN` or `__PARAM_` in the SQL +// placeholders line up Postgres `$n` are exactly 1..parameters.length +// tenant scope `.crossTenant()` is always cross-tenant; a top-level tenant +// predicate on an unjoined query is always single-tenant +// hostile values any JS value where a value goes is compiled or refused with a +// typed QueryBuilderError — never a throw, never a defect +import { describe, expect, it } from "vitest" +import fc from "fast-check" +import { Effect, Exit } from "effect" +import * as CH from "@maple-dev/effect-orm/clickhouse" +import * as PG from "@maple-dev/effect-orm/postgres" +import type { CoreContext } from "../core-cases" +import { scopeContexts as contexts } from "./contexts" +import { compilePlan } from "./build" +import { config, FuzzFailure, property, skipKnown } from "./harness" +import { knownHostileIssue, knownPlanIssue } from "./known-issues" +import { nastyOptions, ORDER_COLUMNS, plan, type OrderColumn, type Plan } from "./plan" + +const stringsIn = (value: unknown): Array => + typeof value === "string" ? [value] : Array.isArray(value) ? value.flatMap(stringsIn) : value !== null && typeof value === "object" ? Object.values(value).flatMap(stringsIn) : [] + +const serialize = (value: unknown) => JSON.stringify(value, (_, v) => (typeof v === "bigint" ? `${v}n` : v instanceof Date ? `Date(${v.getTime()})` : v)) + +const compiles = (ctx: CoreContext, input: Plan) => { + if (!config.strict) { + const known = knownPlanIssue({ target: ctx.dialect, plan: input }) + if (known !== undefined) skipKnown(known) + } + const first = compilePlan(ctx, input) + if (first._tag !== "Compiled") throw new FuzzFailure(`a valid plan was not compiled (${first._tag})`, { error: first.error }) + const { compiled } = first + const context = { sql: compiled.sql, parameters: serialize(compiled.parameters), tenantScope: compiled.tenantScope } + const fail = (message: string): never => { + throw new FuzzFailure(message, context) + } + + const second = compilePlan(ctx, input) + if (second._tag !== "Compiled" || second.compiled.sql !== compiled.sql || serialize(second.compiled.parameters) !== serialize(compiled.parameters)) { + fail("compiling the same plan twice gave different output") + } + + const strings = stringsIn(input) + for (const token of ["undefined", "[object ", "NaN", "Infinity", "__PARAM_"]) { + if (compiled.sql.includes(token) && !strings.some((s) => s.includes(token))) fail(`the SQL contains ${JSON.stringify(token)}`) + } + + if (ctx.dialect === "postgres" && !strings.some((s) => s.includes("$"))) { + const used = new Set([...compiled.sql.matchAll(/\$(\d+)/g)].map((m) => Number(m[1]))) + const expected = Array.from({ length: compiled.parameters.length }, (_, i) => i + 1) + if (used.size !== expected.length || !expected.every((n) => used.has(n))) { + fail(`placeholders ${[...used].sort((a, b) => a - b).join(",")} do not match ${compiled.parameters.length} parameters`) + } + } + + if (input.crossTenant && compiled.tenantScope !== "cross-tenant") fail(".crossTenant() did not make the query cross-tenant") + const pinned = input.where.some((c) => c.k === "tenant" && c.side === "orders") + if (pinned && input.join === undefined && !input.crossTenant && compiled.tenantScope !== "single-tenant") { + fail("a top-level tenant predicate on an unjoined query did not make it single-tenant") + } +} + +describe("compile properties", () => { + for (const ctx of Object.values(contexts)) { + const name = `plans compile cleanly (${ctx.dialect})` + it(name, () => property(name, plan(nastyOptions(ctx.dialect)), (input) => compiles(ctx, input), { runs: 300 }), 120_000) + } +}) + +// Hostile values + +/** Values no column type accepts, plus near-misses that some do. */ +const hostile: fc.Arbitrary = fc.oneof( + fc.anything({ withBigInt: true, withDate: true, withMap: true, withSet: true, withNullPrototype: true, withTypedArray: true, withSparseArray: true }), + fc.bigInt(), + fc.constantFrom(Number.NaN, Infinity, -Infinity, -0, 2 ** 53, 1e308, 5e-324, new Date(Number.NaN), Symbol("s"), () => 1, undefined, null, "", "2026-13-45 99:99:99", { _brand: "Expr" }, { _brand: "Expr", toFragment: 1 }), +) + +type HostileSite = "eq" | "neq" | "gt" | "in_" | "notIn" | "between" | "like" | "param.int" | "param.float" | "param.string" | "param.bool" | "param.dateTime" | "param.of" | "select" +export interface HostileInput { + readonly dialect: "clickhouse" | "postgres" + readonly column: OrderColumn + readonly site: HostileSite + readonly value: unknown +} + +const hostileQuery = (ctx: CoreContext, input: HostileInput) => { + const { site, value, column } = input + const params: Record = { orgId: "org_1", h: value } + let q: any = ctx.from(ctx.orders) + if (site === "select") return { query: q.select(() => ({ id: CH.lit(1), bad: value })), params } + q = q.select(($: any) => ({ id: $.Id })) + if (site.startsWith("param.")) { + const kind = site.slice("param.".length) + const marker = kind === "of" ? CH.param.of((ctx.orders as any).columns[column], "h") : (CH.param as any)[kind]("h") + return { query: q.where(($: any) => [$[column].eq(marker)]), params } + } + return { + query: q.where(($: any) => [site === "in_" || site === "notIn" ? $[column][site](value, value) : site === "between" ? $[column].between(value, value) : $[column][site](value)]), + params, + } +} + +const hostileInput = (dialect: "clickhouse" | "postgres"): fc.Arbitrary => + fc.record({ + dialect: fc.constant(dialect), + column: fc.constantFrom(...(Object.keys(ORDER_COLUMNS) as OrderColumn[])), + site: fc.constantFrom("eq", "neq", "gt", "in_", "notIn", "between", "like", "param.int", "param.float", "param.string", "param.bool", "param.dateTime", "param.of", "select"), + value: hostile, + }) + +const neverThrows = (ctx: CoreContext, input: HostileInput) => { + if (!config.strict) { + const known = knownHostileIssue(input) + if (known !== undefined) skipKnown(known) + } + let exit: Exit.Exit + try { + const { query, params } = hostileQuery(ctx, input) + exit = Effect.runSyncExit(((ctx.dialect === "clickhouse" ? CH.compile : PG.compile) as any)(query, params)) + } catch (error) { + throw new FuzzFailure("the builder threw instead of failing with a QueryBuilderError", { + error: error instanceof Error ? `${error.name}: ${error.message}\n${error.stack?.split("\n").slice(1, 6).join("\n")}` : String(error), + }) + } + if (Exit.isFailure(exit)) { + const typed = exit.cause.reasons.every((reason) => reason._tag === "Fail" && String((reason as any).error?._tag).endsWith("QueryBuilderError")) + if (!typed) throw new FuzzFailure("compile died or failed with something other than a QueryBuilderError", { cause: String(exit.cause) }) + } + expect(true).toBe(true) +} + +describe("hostile values", () => { + for (const ctx of Object.values(contexts)) { + const name = `hostile values are refused, never thrown (${ctx.dialect})` + it(name, () => property(name, hostileInput(ctx.dialect), (input) => neverThrows(ctx, input), { runs: 500 }), 120_000) + } +}) diff --git a/tests/fuzz/contexts.ts b/tests/fuzz/contexts.ts new file mode 100644 index 0000000..870eb20 --- /dev/null +++ b/tests/fuzz/contexts.ts @@ -0,0 +1,20 @@ +// Contexts for compiling plans. +// +// `contexts` are the core suite's: the fixture tables are CTEs over literal +// rows, so the same plan runs on every database. Those CTEs are bare SQL the +// builder cannot see into, and a CTE named `orders` shadows the `orders` table, +// so a query compiled against them never reports a meaningful tenant scope. +// +// `scopeContexts` compile the same plan against the plain tables, with no CTEs: +// the tenant scope the builder reports there is the one a real application +// gets, and the tenant oracle checks it against the rows the model says the +// query actually read. +import * as CH from "@maple-dev/effect-orm/clickhouse" +import { contexts as core, type CoreContext, type DialectName } from "../core-cases" + +export const contexts: Record = core + +export const scopeContexts: Record = { + clickhouse: { ...core.clickhouse, from: (table) => CH.from(table) as any }, + postgres: { ...core.postgres, from: (table) => CH.from(table) as any }, +} diff --git a/tests/fuzz/db.ts b/tests/fuzz/db.ts new file mode 100644 index 0000000..b1d7dc4 --- /dev/null +++ b/tests/fuzz/db.ts @@ -0,0 +1,97 @@ +// Run a compiled query on a real database and say what happened, without throwing. +// +// Oracles need to tell four outcomes apart — the builder refused, the database +// refused, the rows did not decode, or rows came back — so each is a tagged +// value rather than an exception the predicate would have to pick apart. +import { PGlite } from "@electric-sql/pglite" +import { Cause, Effect, Exit } from "effect" +import type * as CH from "@maple-dev/effect-orm/clickhouse" + +export type Outcome = + | { readonly _tag: "Rows"; readonly rows: ReadonlyArray; readonly wire: ReadonlyArray } + | { readonly _tag: "DatabaseError"; readonly message: string; readonly wire?: undefined } + | { readonly _tag: "DecodeError"; readonly message: string; readonly wire: ReadonlyArray } + +const decode = async (compiled: CH.CompiledQuery, wire: ReadonlyArray): Promise => { + const exit = await Effect.runPromiseExit(compiled.decodeRows(wire as ReadonlyArray>)) + return Exit.isSuccess(exit) + ? { _tag: "Rows", rows: exit.value, wire } + : { _tag: "DecodeError", message: Cause.pretty(exit.cause), wire } +} + +// Postgres + +let pglite: PGlite | undefined +/** One in-process Postgres per test file; PGlite takes the time zone from the host otherwise. */ +export const postgres = (): PGlite => (pglite ??= new PGlite({ postgresqlconf: "timezone = 'UTC'" })) +export const closePostgres = async (): Promise => { + await pglite?.close() + pglite = undefined +} + +export const runPostgres = async (compiled: CH.CompiledQuery): Promise => { + let wire: ReadonlyArray + try { + wire = (await postgres().query>(compiled.sql, [...compiled.parameters])).rows + } catch (error) { + return { _tag: "DatabaseError", message: error instanceof Error ? error.message : String(error) } + } + return decode(compiled, wire) +} + +// ClickHouse + +export const clickhouseEndpoint = process.env.EFFECT_ORM_CLICKHOUSE_URL +const user = process.env.EFFECT_ORM_CLICKHOUSE_USER ?? "default" +const password = process.env.EFFECT_ORM_CLICKHOUSE_PASSWORD ?? "" + +/** POST raw SQL; the text reply, or the server's error. */ +export const clickhouseText = async ( + sql: string, + settings: Record = {}, +): Promise<{ readonly ok: boolean; readonly text: string }> => { + const url = new URL(clickhouseEndpoint!) + for (const [key, value] of Object.entries({ default_format: "JSONEachRow", session_timezone: "UTC", ...settings })) { + url.searchParams.set(key, value) + } + // A soak sends tens of thousands of requests; a dropped socket is the harness's problem, not a finding. + for (let attempt = 1; ; attempt++) { + try { + const response = await fetch(url, { + method: "POST", + headers: { "X-ClickHouse-User": user, "X-ClickHouse-Key": password }, + body: sql, + signal: AbortSignal.timeout(10_000), + }) + return { ok: response.ok, text: await response.text() } + } catch (error) { + const code = (error as { cause?: { code?: string } }).cause?.code ?? String(error) + if (attempt >= 5) throw new Error(`ClickHouse unreachable after ${attempt} attempts (${code}) — infrastructure, not a finding`) + await new Promise((resolve) => setTimeout(resolve, 100 * 2 ** attempt)) + } + } +} + +export const runClickHouse = async ( + compiled: CH.CompiledQuery, + settings: Record = {}, +): Promise => { + const { ok, text } = await clickhouseText(compiled.sql, settings) + if (!ok) return { _tag: "DatabaseError", message: text.trim() } + const wire = text + .split("\n") + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as unknown) + return decode(compiled, wire) +} + +/** What each fuzzed query runs on: Postgres always, ClickHouse when an endpoint is set. */ +export type Target = "postgres" | "clickhouse" | "clickhouse-join-nulls" + +export const run = (target: Target, compiled: CH.CompiledQuery): Promise => + target === "postgres" + ? runPostgres(compiled) + : runClickHouse(compiled, { + join_use_nulls: target === "clickhouse-join-nulls" ? "1" : "0", + output_format_json_quote_64bit_integers: target === "clickhouse-join-nulls" ? "1" : "0", + }) diff --git a/tests/fuzz/harness.ts b/tests/fuzz/harness.ts new file mode 100644 index 0000000..586f2d9 --- /dev/null +++ b/tests/fuzz/harness.ts @@ -0,0 +1,159 @@ +// The fuzz harness: one `property` wrapper every fuzz suite runs through. +// +// It exists so a failure is cheap to act on. fast-check already shrinks the input; +// this adds the rest of the loop: +// +// - budget and seed come from the environment, so the same test is a 50-run +// smoke check in `bun run test` and a 50 000-run soak under `bun run fuzz`; +// - a failure prints the shrunk input, the SQL it compiled to, and the exact +// command that replays it (`FUZZ_SEED`, `FUZZ_PATH`); +// - inputs pinned in `regressions.ts` are replayed before any random one, so +// a counterexample stays a test after the bug is fixed; +// - `known` issues are skipped by name (see `known-issues.ts`), and +// `FUZZ_STRICT=1` turns them back on to check whether one is fixed. +import fc from "fast-check" +import { pinnedFor } from "./regressions" + +const int = (name: string): number | undefined => { + const raw = process.env[name] + return raw === undefined || raw === "" ? undefined : Number.parseInt(raw, 10) +} + +export const config = { + /** Multiplies every property's own default run count. */ + scale: Number.parseFloat(process.env.FUZZ_SCALE ?? "1"), + runs: int("FUZZ_RUNS"), + seed: int("FUZZ_SEED"), + path: process.env.FUZZ_PATH || undefined, + strict: process.env.FUZZ_STRICT === "1", + verbose: process.env.FUZZ_VERBOSE === "1", +} + +/** + * What a predicate attaches to its failure: the compiled SQL and params, or + * anything else that makes the counterexample readable without a debugger. + */ +export class FuzzFailure extends Error { + constructor( + message: string, + readonly context: Record, + ) { + super(message) + } +} + +/** Run `body`, and if it throws, attach `context` to the error. */ +export const withContext = async (context: Record, body: () => A | Promise): Promise => { + try { + return await body() + } catch (error) { + if (error instanceof FuzzFailure) throw error + const message = error instanceof Error ? error.message : String(error) + throw new FuzzFailure(message, context) + } +} + +/** Thrown to discard an input, like `fc.pre`, but with a reason the stats can count. */ +export class Skip extends Error { + constructor( + readonly reason: string, + readonly known: boolean, + ) { + super(`skipped: ${reason}`) + } +} +/** Discard an input the oracle cannot judge (a generator or harness limit). */ +export const skip = (reason: string): never => { + throw new Skip(reason, false) +} +/** Discard an input that reaches an open bug in `known-issues.ts`. */ +export const skipKnown = (issue: string): never => { + throw new Skip(issue, true) +} + +/** Straight to stdout: this suite's vitest setup swallows `console.*` from passing tests. */ +const report = (line: string) => process.stdout.write(`${line}\n`) + +const describeContext = (context: Record): string => + Object.entries(context) + .map(([key, value]) => `--- ${key}\n${typeof value === "string" ? value : fc.stringify(value)}`) + .join("\n") + +export interface PropertyOptions { + /** Runs at FUZZ_SCALE=1 when FUZZ_RUNS is unset. */ + readonly runs: number + readonly timeoutMs?: number +} + +/** + * Check `predicate` over `arb`, throwing a report a person can act on. + * + * Inputs the predicate `skip`s are discarded rather than counted; if most are, + * the generator is wrong and the run says so instead of passing on nothing. + */ +export const property = async ( + name: string, + arb: fc.Arbitrary, + predicate: (input: T) => void | Promise, + options: PropertyOptions, +): Promise => { + const skips = new Map() + const knownSkips = new Set() + let checked = 0 + const details = await fc.check( + fc.asyncProperty(arb, async (input) => { + try { + await predicate(input) + checked++ + } catch (error) { + if (error instanceof Skip) { + skips.set(error.reason, (skips.get(error.reason) ?? 0) + 1) + if (error.known) knownSkips.add(error.reason) + fc.pre(false) + } + throw error + } + }), + { + numRuns: config.runs ?? Math.max(1, Math.round(options.runs * config.scale)), + ...(config.seed !== undefined ? { seed: config.seed } : {}), + ...(config.path !== undefined ? { path: config.path } : {}), + examples: pinnedFor(name).map((input) => [input as T]), + interruptAfterTimeLimit: options.timeoutMs, + maxSkipsPerRun: 1000, + endOnFailure: false, + }, + ) + if (config.verbose || details.failed) { + const skipped = [...skips].map(([reason, count]) => `${count}× ${reason}`).join(", ") + report(`[fuzz] ${name}: ${checked} checked${skipped ? `, skipped ${skipped}` : ""}`) + } + // fast-check fails a run whose every input was discarded; say why instead. + if (checked === 0 && details.numSkips > 0 && details.counterexample === null) { + // A property an open bug covers entirely is the backlog doing its job, not a broken generator. + if (knownSkips.size > 0) { + report(`[fuzz] ${name}: every input hits a known issue (${[...knownSkips].join(", ")})`) + return + } + throw new Error(`[fuzz] ${name}: every input was skipped (${[...skips.keys()].join(", ")})`) + } + if (!details.failed) return + + const error = details.errorInstance + const counterexample = details.counterexample?.[0] + // vitest reads `-t` as a regex, and names like `plans match the model (postgres, …)` have metacharacters. + const pattern = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + const replay = `FUZZ_SEED=${details.seed} FUZZ_PATH=${details.counterexamplePath} bun run fuzz -t '${pattern}'` + throw new Error( + [ + `[fuzz] ${name} failed after ${details.numRuns} runs (${details.numShrinks} shrinks)`, + `replay: ${replay}`, + "--- input (shrunk)", + fc.stringify(counterexample), + "--- error", + error instanceof Error ? error.message : String(error), + error instanceof FuzzFailure ? describeContext(error.context) : "", + `--- pin it: add the input above to tests/fuzz/regressions.ts under a key matching ${JSON.stringify(name)}`, + ].join("\n"), + ) +} diff --git a/tests/fuzz/known-issues.ts b/tests/fuzz/known-issues.ts new file mode 100644 index 0000000..8d787e9 --- /dev/null +++ b/tests/fuzz/known-issues.ts @@ -0,0 +1,192 @@ +// Bugs the fuzzer has found and nobody has fixed yet. +// +// Each entry recognises the inputs that reach one bug, so the suite stays green +// while it is open and the rest of the input space keeps being fuzzed. This file +// is the backlog: fix the bug, delete its entry, and run with `FUZZ_STRICT=1` +// to confirm nothing else matched it. An entry needs a reason a person can act on. +import type { LiteralType } from "./values" + +export interface KnownIssue { + readonly id: string + readonly reason: string + readonly matches: (input: A) => boolean +} + +export type LiteralPath = "compare" | "select" +export interface LiteralInput { + readonly literal: LiteralType + readonly value: unknown + readonly path: LiteralPath +} + +const isFloat64 = (literal: LiteralType) => literal.dialect === "clickhouse" && literal.type.sql === "Float64" + +const literalIssues: ReadonlyArray> = [ + { + id: "ch-float-subnormal-syntax", + reason: "A subnormal float (`5e-324`) written as String(value) is a ClickHouse SYNTAX_ERROR.", + matches: ({ literal, value }) => + isFloat64(literal) && typeof value === "number" && value !== 0 && Math.abs(value) < 2.2250738585072014e-308, + }, + { + id: "ch-array-literal-untyped", + reason: + "An array literal is written without a cast, so ClickHouse infers its element type from the values: " + + "`[0, 4294967296]` is Array(UInt64), and comparing it with an Array(Nullable(Int64)) column is " + + "ILLEGAL_TYPE_OF_ARGUMENT. CAST(… AS ) would fix it.", + matches: ({ literal, value, path }) => + literal.dialect === "clickhouse" && + path === "compare" && + literal.type.sql === "Array(Nullable(Int64))" && + Array.isArray(value) && + value.every((n) => n === null || n >= 0) && + value.some((n) => n !== null && n >= 2 ** 32), + }, + { + id: "pg-untyped-select-param", + reason: + "A param in a Postgres SELECT list is bound untyped (`$1 AS \"p\"`), so the server infers `text` and a " + + "boolean comes back as `'false'`, an array as `'{…}'`, neither of which the column's codec decodes. " + + "Writing `$1::` would fix it.", + matches: ({ literal, path }) => + literal.dialect === "postgres" && path === "select" && (literal.type.sql === "boolean" || literal.type._tag === "Array"), + }, +] + +/** The id of the open issue this input would hit, if any. */ +export const knownLiteralIssue = (input: LiteralInput): string | undefined => + literalIssues.find((issue) => issue.matches(input))?.id + +// Hostile values (compile.fuzz.test.ts) + +import type { HostileInput } from "./compile.fuzz.test" + +const containsBigInt = (value: unknown, seen = new Set()): boolean => { + if (typeof value === "bigint") return true + if (value === null || typeof value !== "object" || seen.has(value)) return false + seen.add(value) + try { + return Reflect.ownKeys(value).some((key) => containsBigInt((value as any)[key], seen)) + } catch { + return false + } +} + +const hostileIssues: ReadonlyArray> = [ + { + id: "describe-bigint", + reason: + "literal.ts `describe()` formats an unexpected value with JSON.stringify, which throws on a bigint, so " + + "`$.OrgId.eq(0n)` dies with `TypeError: Do not know how to serialize a BigInt` instead of an InvalidLiteral error.", + matches: ({ value }) => containsBigInt(value), + }, + { + id: "like-non-string-pattern", + reason: + "`like`/`notLike`/`ilike` call `pattern.toFragment()` on anything that is not a string, so a number, " + + "object or Map pattern dies with `TypeError: pattern.toFragment is not a function`.", + matches: ({ site, value }) => site === "like" && typeof value !== "string", + }, + { + id: "select-non-expression", + reason: + "A select callback returning a non-expression (`{ bad: 1 }`, `{ bad: undefined }`) dies with " + + "`TypeError: selectExprs[alias].toFragment is not a function` instead of a QueryBuilderError naming the alias.", + matches: ({ site, value }) => + site === "select" && !(value !== null && typeof value === "object" && typeof (value as any).toFragment === "function"), + }, + { + id: "pg-timestamp-invalid-string-dies", + reason: + "The Postgres timestamp literal codec formats an unparseable string or invalid Date with toISOString(), which " + + "throws `RangeError: Invalid time value` inside the transform: compile dies instead of failing InvalidLiteral.", + matches: ({ dialect, site, column, value }) => + dialect === "postgres" && + (site === "param.dateTime" || (column === "Created" && (site === "param.of" || !site.startsWith("param.")))) && + (typeof value === "string" || value instanceof Date) && + Number.isNaN(new Date(value as string).getTime()), + }, +] + +export const knownHostileIssue = (input: HostileInput): string | undefined => + hostileIssues.find((issue) => issue.matches(input))?.id + +// Query plans (queries.fuzz.test.ts) + +import type { Target } from "./db" +import type { Plan } from "./plan" + +export interface PlanInput { + readonly target: Target + readonly plan: Plan +} + +/** Every object in a plan, depth-first: plans are plain data, so this reaches every expression. */ +const nodes = function* (value: unknown): Generator> { + if (Array.isArray(value)) for (const item of value) yield* nodes(item) + else if (value !== null && typeof value === "object") { + yield value as Record + for (const item of Object.values(value)) yield* nodes(item) + } +} +const precedence = (op: unknown) => (op === "add" || op === "sub" ? 1 : 2) +const isArith = (e: unknown): e is { op: string; l: unknown; r: unknown } => (e as any)?.k === "arith" + +/** The conjuncts a WHERE list ANDs together, flattening nested ANDs. */ +const conjuncts = (conds: ReadonlyArray): Array => + conds.flatMap((c: any) => (c?.k === "and" ? conjuncts(c.cs) : [c])) +const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b) + +const planIssues: ReadonlyArray> = [ + { + id: "clickhouse-server-and-absorption", + reason: + "A ClickHouse 25.8 server bug, not the builder: `WHERE a AND ((b AND a) OR a)` fails with " + + "`Number of arguments for function \"and\" should be at least 2` — the common-expression rewrite leaves a " + + "one-argument and(). `SETTINGS optimize_extract_common_expressions = 0` avoids it.", + matches: ({ target, plan }) => { + if (target === "postgres") return false + const top = conjuncts(plan.where) + return [...nodes(plan)].some( + (node) => + node.k === "or" && + top.some( + (w) => + (node.cs as Array).some((d) => same(d, w)) && + (node.cs as Array).some((d) => d?.k === "and" && conjuncts(d.cs).some((x) => same(x, w))), + ), + ) + }, + }, + { + id: "pg-fraction-bound-to-integer", + reason: + "On Postgres a comparison value is bound as a parameter, and the server types it from the integer side: " + + "`\"Amount\" > $1` with 9.75 fails `invalid input syntax for type bigint`. ClickHouse inlines the literal " + + "and compares correctly. The builder accepts the value (int8's codec is any finite number).", + matches: ({ target, plan }) => + target === "postgres" && + [...nodes(plan)].some((node) => node.k === "value" && typeof node.v === "number" && !Number.isInteger(node.v)), + }, + { + id: "arith-argument-not-parenthesised", + reason: + "A compound expression passed as an arithmetic argument loses its boundary: `$.a.sub($.b.add(1))` compiles to " + + "`a - b + 1`, `$.a.div($.b.mul(2))` to `a / b * 2`. docs/expressions.md documents that *chains* follow SQL " + + "precedence (the model applies that), but an argument is not a chain, and no call order expresses " + + "`a - (b + 1)`. Parenthesising a compound right operand would fix it without changing chained SQL.", + matches: ({ plan }) => + [...nodes(plan)].some( + (node) => isArith(node) && isArith(node.r) && precedence(node.r.op) <= precedence(node.op), + ), + }, + { + id: "fromquery-select-all", + reason: + "`CH.fromQuery(q, alias).select()` is typed as selecting every column, but a subquery source has no column " + + "list to expand, so compile dies with QueryBuilderDefect `select() is required`. `select(...names)` works.", + matches: ({ plan }) => plan.wrap === "all", + }, +] + +export const knownPlanIssue = (input: PlanInput): string | undefined => planIssues.find((issue) => issue.matches(input))?.id diff --git a/tests/fuzz/literals.fuzz.test.ts b/tests/fuzz/literals.fuzz.test.ts new file mode 100644 index 0000000..74ece2a --- /dev/null +++ b/tests/fuzz/literals.fuzz.test.ts @@ -0,0 +1,64 @@ +// Literal round-trip: any value of a column's type survives being written into SQL. +// +// For each column type, a random value reaches the database as JSON through +// the database's own parser — the reference row `v` — and, independently, +// through the builder along two paths: +// +// compare `WHERE v = ` the literal a comparison writes +// select `SELECT AS p` a param in the SELECT list +// +// Each path must compile, run, and decode back to the value. They are separate +// properties so a bug on one cannot hide the other. +// +// What it catches: escaping (quotes, backslashes, placeholders inside values), +// numbers printed in a form the database reads differently, timestamps at the +// edges of a type's range, and nested arrays and maps. +import { afterAll, describe, expect, it } from "vitest" +import { Effect, Exit } from "effect" +import * as CH from "@maple-dev/effect-orm/clickhouse" +import * as PG from "@maple-dev/effect-orm/postgres" +import { clickhouseEndpoint, closePostgres, run } from "./db" +import { config, FuzzFailure, property, skip, skipKnown, withContext } from "./harness" +import { knownLiteralIssue, type LiteralPath } from "./known-issues" +import { literalTypes, normalize, referenceJson, wellFormed, type LiteralType } from "./values" + +afterAll(closePostgres) + +const build = (literal: LiteralType, value: unknown, path: LiteralPath) => { + const M = literal.dialect === "clickhouse" ? CH : PG + const src = (M.table as typeof CH.table)("src", { external: true, columns: { v: literal.type } }) + const base = (M.from(src) as any).withCTE("src", literal.source(referenceJson(literal.wire(value)), value)) + if (path === "select") return base.select(($: any) => ({ v: $.v, p: M.param.of(literal.type, "p") })) + return base.select(($: any) => ({ v: $.v })).where(($: any) => [value === null ? $.v.isNull() : $.v.eq(value)]) +} + +const roundTrip = async (literal: LiteralType, value: unknown, path: LiteralPath): Promise => { + // The reference row goes through JSON, which cannot carry a lone surrogate the way SQL text does. + if (!wellFormed(value)) skip("lone surrogate") + if (!config.strict) { + const known = knownLiteralIssue({ literal, value, path }) + if (known !== undefined) skipKnown(known) + } + const M = literal.dialect === "clickhouse" ? CH : PG + const exit = Effect.runSyncExit((M.compile as any)(build(literal, value, path), { p: value })) + if (Exit.isFailure(exit)) throw new FuzzFailure("the builder refused a valid value", { value, cause: String(exit.cause) }) + const compiled = exit.value as CH.CompiledQuery + const outcome = await run(literal.dialect, compiled) + await withContext({ sql: compiled.sql, parameters: compiled.parameters, outcome }, () => { + expect(outcome._tag).toBe("Rows") + if (outcome._tag !== "Rows") return + const expected = path === "select" ? { v: value, p: value } : { v: value } + expect(normalize(outcome.rows)).toEqual([normalize(expected)]) + }) +} + +describe("literal round-trip", () => { + for (const literal of literalTypes) { + const live = literal.dialect === "postgres" || clickhouseEndpoint !== undefined + for (const path of ["compare", "select"] as const) { + if (path === "compare" && !literal.comparable) continue + const name = `${literal.id} ${path}` + it.skipIf(!live)(name, () => property(name, literal.value, (value) => roundTrip(literal, value, path), { runs: 40 }), 120_000) + } + } +}) diff --git a/tests/fuzz/model.ts b/tests/fuzz/model.ts new file mode 100644 index 0000000..2d11825 --- /dev/null +++ b/tests/fuzz/model.ts @@ -0,0 +1,394 @@ +// The reference model: what a plan's rows should be, computed in JS over the fixture. +// +// This is the oracle that does not trust the builder. It evaluates a `Plan` +// directly — SQL three-valued logic, the join, grouping, ordering, paging — so a +// query that runs cleanly but answers wrong (a lost parenthesis, a negation +// applied to the wrong side, NULL handled like a value) shows up as a row diff. +// +// Where the databases genuinely differ, the model says so per target rather +// than papering over it; each such rule names the difference it encodes. +// Inputs the model cannot judge raise `skip` with a reason. +import { fixtureRows } from "../core-cases" +import type { Target } from "./db" +import { skip } from "./harness" +import type { Agg, AggShape, ArithOp, Cond, Expr, Operand, Plan, RowsShape } from "./plan" + +type V = number | string | null +type Bool = boolean | null + +interface Order { + readonly OrgId: string + readonly Id: number + readonly Customer: string + readonly Amount: number + readonly Status: string + readonly Note: string | null + /** Epoch milliseconds. */ + readonly Created: number +} +interface Customer { + readonly OrgId: string + readonly Name: string + readonly Tier: string +} + +const orders: ReadonlyArray = fixtureRows.orders.map(([OrgId, Id, Customer, Amount, Status, Note, Created]) => ({ + OrgId: OrgId as string, + Id: Id as number, + Customer: Customer as string, + Amount: Amount as number, + Status: Status as string, + Note: Note as string | null, + Created: Date.parse(`${(Created as string).replace(" ", "T")}Z`), +})) +const customers: ReadonlyArray = fixtureRows.customers.map(([OrgId, Name, Tier]) => ({ + OrgId: OrgId as string, + Name: Name as string, + Tier: Tier as string, +})) + +interface Row { + readonly o: Order + /** The joined customer; `undefined` for an unmatched LEFT JOIN row. */ + readonly c?: Customer +} + +interface Env { + readonly target: Target + readonly plan: Plan + readonly joined: boolean +} + +const isTime = (e: Expr) => e.k === "col" && e.col === "Created" + +/** + * An integer expression's ClickHouse signedness. ClickHouse keeps unsigned + * types through `+` and `*` (`Id` is UInt32, a non-negative literal UInt8) and + * computes `%` across a signed and an unsigned operand in ways no other + * database does (`-7 % toUInt32(3)` is 0), so the model declines those. + */ +type ChKind = "unsigned" | "signed" | "float" +const chKind = (e: Expr | number): ChKind => { + if (typeof e === "number") return !Number.isInteger(e) ? "float" : e >= 0 ? "unsigned" : "signed" + switch (e.k) { + case "col": + return e.col === "Id" ? "unsigned" : "signed" + case "num": + return chKind(e.v) + case "arith": { + const l = chKind(e.l) + const r = chKind(e.r) + if (e.op === "div" || l === "float" || r === "float") return "float" + if (e.op === "sub") return "signed" + return l === "unsigned" && r === "unsigned" ? "unsigned" : "signed" + } + default: + return "signed" + } +} + +// Expressions + +/** + * Arithmetic as the SQL text parses it. The builder writes `${lhs} ${op} ${rhs}` + * without parentheses — documented in docs/expressions.md ("Arithmetic does not + * parenthesise"): a chain follows SQL precedence, not call order. So the model + * flattens a tree to the token sequence the SQL contains and re-parses it with + * `*` `/` `%` binding tighter than `+` `-`, both left-associative. + */ +const parsed = new WeakSet() +const flatten = (e: Expr): Array => + e.k === "arith" ? [...flatten(e.l), e.op, ...(typeof e.r === "number" ? [{ k: "num", v: e.r } as Expr] : flatten(e.r))] : [e] +const asParsed = (e: Expr & { k: "arith" }): Expr => { + const tokens = flatten(e) + const node = (op: ArithOp, l: Expr, r: Expr): Expr => { + const made: Expr = { k: "arith", op, l, r } + parsed.add(made) + return made + } + const terms: Array = [] + const ops: Array = [] + let current = tokens[0] as Expr + for (let i = 1; i < tokens.length; i += 2) { + const op = tokens[i] as ArithOp + const operand = tokens[i + 1] as Expr + if (op === "mul" || op === "div" || op === "mod") current = node(op, current, operand) + else { + terms.push(current) + ops.push(op) + current = operand + } + } + terms.push(current) + return ops.reduce((acc, op, j) => node(op, acc, terms[j + 1]!), terms[0]!) +} + +const evalExpr = (env: Env, row: Row, e: Expr): V => { + switch (e.k) { + case "col": + return row.o[e.col] + case "jcol": + if (row.c !== undefined) return row.c[e.col] + // ClickHouse with join_use_nulls=0 fills a missing row with column defaults. + return env.target === "clickhouse" ? "" : null + case "num": + case "str": + return e.v + case "coalesce": + return evalExpr(env, row, e.e) ?? evalExpr(env, row, e.fallback) + case "arith": { + if (!parsed.has(e)) return evalExpr(env, row, asParsed(e)) + const l = evalExpr(env, row, e.l) as number | null + const r = typeof e.r === "number" ? e.r : (evalExpr(env, row, e.r) as number | null) + if (l === null || r === null) return null + switch (e.op) { + case "add": + return l + r + case "sub": + return l - r + case "mul": + return l * r + case "div": + if (r === 0) skip("division by zero") + // `/` is float division on ClickHouse, integer division of two Postgres integers. + return env.target === "postgres" ? Math.trunc(l / r) : l / r + case "mod": + if (r === 0) skip("division by zero") + if (env.target !== "postgres") { + const kinds = new Set([chKind(e.l), chKind(e.r)]) + if (kinds.has("unsigned") && kinds.has("signed")) skip("ClickHouse % across signed and unsigned") + } + // Both databases truncate towards zero, like JS `%`. + return l % r + } + } + } +} + +/** Strings compare by UTF-8 bytes on both targets (ClickHouse always, Postgres under the C collation). */ +const encoder = new TextEncoder() +const compareStrings = (a: string, b: string): number => { + const x = encoder.encode(a) + const y = encoder.encode(b) + for (let i = 0; i < Math.min(x.length, y.length); i++) if (x[i] !== y[i]) return x[i]! - y[i]! + return x.length - y.length +} +const compare = (a: number | string, b: number | string): number => + typeof a === "number" && typeof b === "number" ? a - b : compareStrings(String(a), String(b)) + +const evalOperand = (env: Env, row: Row, op: Operand): V => { + switch (op.k) { + case "value": + return op.v + case "time": + return op.ms + case "param": + return op.v + case "expr": + return evalExpr(env, row, op.e) + } +} + +// Conditions (three-valued) + +const not3 = (b: Bool): Bool => (b === null ? null : !b) +const and3 = (bs: ReadonlyArray): Bool => (bs.includes(false) ? false : bs.includes(null) ? null : true) +const or3 = (bs: ReadonlyArray): Bool => (bs.includes(true) ? true : bs.includes(null) ? null : false) + +const likeRegex = (pattern: string, caseInsensitive: boolean): RegExp => { + let source = "" + for (let i = 0; i < pattern.length; i++) { + const ch = pattern[i]! + if (ch === "\\") { + const next = pattern[i + 1] + if (next === undefined) skip("LIKE pattern ends in an escape") + source += next!.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + i++ + } else if (ch === "%") source += "[\\s\\S]*" + else if (ch === "_") source += "[\\s\\S]" + else source += ch.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&") + } + return new RegExp(`^${source}$`, caseInsensitive ? "iu" : "u") +} + +const evalCond = (env: Env, row: Row, c: Cond): Bool => { + switch (c.k) { + case "const": + return c.v + case "tenant": { + const org = c.side === "orders" ? row.o.OrgId : row.c?.OrgId ?? (env.target === "clickhouse" ? "" : null) + return org === null ? null : org === env.plan.orgId + } + case "not": + return not3(evalCond(env, row, c.c)) + case "and": + return and3(c.cs.map((x) => evalCond(env, row, x))) + case "or": + return or3(c.cs.map((x) => evalCond(env, row, x))) + case "null": { + const isNull = evalExpr(env, row, c.e) === null + return c.op === "isNull" ? isNull : !isNull + } + case "cmp": { + const l = evalExpr(env, row, c.l) + const r = evalOperand(env, row, c.r) + if (l === null || r === null) return null + const d = compare(l, r) + switch (c.op) { + case "eq": + return d === 0 + case "neq": + return d !== 0 + case "gt": + return d > 0 + case "gte": + return d >= 0 + case "lt": + return d < 0 + case "lte": + return d <= 0 + } + } + case "like": { + const l = evalExpr(env, row, c.l) as string | null + if (l === null) return null + // Case folding beyond ASCII and `_` over multi-byte text differ between the two; not modelled. + if (/[^\x00-\x7f]/.test(l)) skip("LIKE over non-ASCII text") + const matched = likeRegex(c.pattern, c.op === "ilike").test(l) + return c.op === "notLike" ? !matched : matched + } + case "between": { + const e = evalExpr(env, row, c.e) + const lo = evalOperand(env, row, c.lo) + const hi = evalOperand(env, row, c.hi) + const inside = and3([e === null || lo === null ? null : compare(e, lo) >= 0, e === null || hi === null ? null : compare(e, hi) <= 0]) + return c.neg ? not3(inside) : inside + } + case "in": { + // An empty list is written `1 = 0` / `1 = 1`: constant, whatever the left side is. + if (c.values.length === 0) return c.neg + const e = evalExpr(env, row, c.e) + if (e === null) return null + const found = c.values.some((v) => compare(e, v) === 0) + return c.neg ? !found : found + } + } +} + +// Queries + +const sourceRows = (env: Env): ReadonlyArray => + orders.flatMap((o): Array => { + if (env.plan.join === undefined) return [{ o }] + const matches = customers.filter((c) => c.OrgId === o.OrgId && c.Name === o.Customer) + if (matches.length > 0) return matches.map((c) => ({ o, c })) + return env.plan.join === "left" ? [{ o }] : [] + }) + +type OutRow = Record + +const nullsFirst = (target: Target, dir: "asc" | "desc") => + // Postgres: NULLS LAST ascending, NULLS FIRST descending. ClickHouse: NULLS LAST either way. + target === "postgres" && dir === "desc" + +const sortRows = (env: Env, rows: Array, orderBy: ReadonlyArray) => + rows.sort((a, b) => { + for (const [alias, dir] of orderBy) { + const x = a[alias] ?? null + const y = b[alias] ?? null + if (x === null && y === null) continue + if (x === null || y === null) { + const first = nullsFirst(env.target, dir) + return (x === null) === first ? -1 : 1 + } + const d = compare(x, y) + if (d !== 0) return dir === "asc" ? d : -d + } + return 0 + }) + +const page = (rows: ReadonlyArray, shape: RowsShape | AggShape): ReadonlyArray => { + const start = shape.offset ?? 0 + return rows.slice(start, shape.limit === undefined ? undefined : start + shape.limit) +} + +const outValue = (e: Expr, v: V): V => (v !== null && isTime(e) ? `@${new Date(v as number).toISOString()}` : v) + +const evalAgg = (env: Env, rows: ReadonlyArray, agg: Agg, global: boolean): V => { + switch (agg.k) { + case "count": + return rows.length + case "countIf": + return rows.filter((row) => evalCond(env, row, agg.c) === true).length + default: { + const values = rows.map((row) => evalExpr(env, row, agg.e)).filter((v): v is number => v !== null) + // Postgres answers an aggregate over no rows with NULL. ClickHouse answers with a default that its + // optimizer moves around (`min(-1 + x)` is rewritten `-1 + min(x)`, so it is -1, not 0): not modelled. + if (values.length === 0) return global && env.target !== "postgres" ? skip("ClickHouse aggregate over no rows") : null + if (agg.k === "sum") return values.reduce((a, b) => a + b, 0) + return agg.k === "min" ? Math.min(...values) : Math.max(...values) + } + } +} + +export interface ModelResult { + readonly rows: ReadonlyArray + /** Whether the plan fixes row order, so rows compare as a list rather than a multiset. */ + readonly ordered: boolean + /** The orgs of every source row that reached the result: the tenant oracle's evidence. */ + readonly orgs: ReadonlySet +} + +/** Every arithmetic node in the plan: plans are plain data, so a walk reaches all of them. */ +const ariths = (value: unknown): Array => { + if (Array.isArray(value)) return value.flatMap(ariths) + if (value === null || typeof value !== "object") return [] + const own = (value as Expr).k === "arith" ? [value as Expr] : [] + return [...own, ...Object.values(value).flatMap(ariths)] +} + +export const evaluate = (plan: Plan, target: Target): ModelResult => { + const env: Env = { target, plan, joined: plan.join !== undefined } + // A database may evaluate any expression on any row — or fold a constant one at plan time — before + // filtering decides it was not needed. So a division by zero anywhere is a division by zero. + const all = ariths(plan) + for (const row of sourceRows(env)) for (const e of all) evalExpr(env, row, e) + const filtered = sourceRows(env).filter((row) => and3(plan.where.map((c) => evalCond(env, row, c))) === true) + const orgs = new Set(filtered.flatMap((row) => [row.o.OrgId, ...(row.c ? [row.c.OrgId] : [])])) + const shape = plan.shape + + let rows: Array + if (shape.k === "rows") { + rows = filtered.map((row) => Object.fromEntries(shape.select.map(([alias, e]) => [alias, outValue(e, evalExpr(env, row, e))]))) + if (shape.distinct) { + const seen = new Set() + rows = rows.filter((row) => { + const key = JSON.stringify(row) + return seen.has(key) ? false : (seen.add(key), true) + }) + } + } else { + const groups = new Map>() + if (shape.keys.length === 0) groups.set("", [...filtered]) + for (const row of shape.keys.length === 0 ? [] : filtered) { + const key = JSON.stringify(shape.keys.map(([, e]) => evalExpr(env, row, e))) + groups.set(key, [...(groups.get(key) ?? []), row]) + } + rows = [...groups.values()] + .filter((group) => + shape.having.every((h) => { + const v = evalAgg(env, group, h.agg, false) + return v !== null && evalCond(env, group[0]!, { k: "cmp", op: h.op, l: { k: "num", v: v as number }, r: { k: "value", v: h.v } }) === true + }), + ) + .map((group) => ({ + ...Object.fromEntries(shape.keys.map(([alias, e]) => [alias, outValue(e, evalExpr(env, group[0]!, e))])), + ...Object.fromEntries(shape.aggs.map(([alias, agg]) => [alias, evalAgg(env, group, agg, shape.keys.length === 0)])), + })) + } + + const uniqueKeys = shape.k === "rows" ? ["id"] : shape.keys.map(([alias]) => alias) + const total = uniqueKeys.length > 0 && uniqueKeys.every((key) => shape.orderBy.some(([alias]) => alias === key)) + sortRows(env, rows, shape.orderBy) + return { rows: page(rows, shape), ordered: total && !plan.wrap, orgs } +} diff --git a/tests/fuzz/plan.ts b/tests/fuzz/plan.ts new file mode 100644 index 0000000..8c91e23 --- /dev/null +++ b/tests/fuzz/plan.ts @@ -0,0 +1,341 @@ +// Query plans: a random query as plain data. +// +// The generator never touches the builder. It produces a `Plan` — which source, +// which conditions, what to select — that `build.ts` turns into builder calls +// and `model.ts` evaluates in JS. Keeping the plan as data is what makes a +// failure readable: fast-check shrinks the plan, and the report prints it. +// +// Expressions are generated by *sort* (number, string, nullable string, time) +// so a plan is well-typed by construction, and the builder refusing one is a +// finding rather than noise. +import fc from "fast-check" +import { nastyString, wellFormed } from "./values" + +// The fixture (tests/core-cases.ts): five orders, four customers, two orgs. +export const ORDER_COLUMNS = { + OrgId: "str", + Id: "num", + Customer: "str", + Amount: "num", + Status: "str", + Note: "nstr", + Created: "time", +} as const +export const CUSTOMER_COLUMNS = { OrgId: "str", Name: "str", Tier: "str" } as const +export type OrderColumn = keyof typeof ORDER_COLUMNS +export type CustomerColumn = keyof typeof CUSTOMER_COLUMNS +export type Sort = "num" | "str" | "nstr" | "time" + +/** Values the fixture's string columns hold, so equality and LIKE sometimes match. */ +const FIXTURE_STRINGS = ["org_1", "org_2", "acme", "globex", "initech", "umbrella", "paid", "open", "void", "rush", "it's", "gold", "silver", "bronze", "platinum"] +const T0 = Date.UTC(2026, 0, 1) +/** Around the fixture's `Created` range (00:00–00:40), at millisecond precision. */ +const time = fc.oneof( + fc.integer({ min: 0, max: 4 }).map((i) => T0 + i * 600_000), + fc.integer({ min: T0 - 3_600_000, max: T0 + 6_000_000 }), +) + +// Expressions + +export type ArithOp = "add" | "sub" | "mul" | "div" | "mod" +export type Expr = + | { readonly k: "col"; readonly col: OrderColumn } + | { readonly k: "jcol"; readonly col: CustomerColumn } + | { readonly k: "num"; readonly v: number } + | { readonly k: "str"; readonly v: string } + | { readonly k: "arith"; readonly op: ArithOp; readonly l: Expr; readonly r: Expr | number } + | { readonly k: "coalesce"; readonly e: Expr; readonly fallback: Expr } + +/** A comparison's right-hand side: a plain value, a param, or another expression. */ +export type Operand = + | { readonly k: "value"; readonly v: number | string } + /** A time, in the three forms a DateTime comparison accepts. */ + | { readonly k: "time"; readonly ms: number; readonly form: "DateTime" | "Date" | "string" } + | { readonly k: "param"; readonly kind: "int" | "string" | "dateTime"; readonly v: number | string } + | { readonly k: "expr"; readonly e: Expr } + +export type CmpOp = "eq" | "neq" | "gt" | "gte" | "lt" | "lte" +export type Cond = + | { readonly k: "cmp"; readonly op: CmpOp; readonly l: Expr; readonly r: Operand } + | { readonly k: "like"; readonly op: "like" | "notLike" | "ilike"; readonly l: Expr; readonly pattern: string } + | { readonly k: "null"; readonly op: "isNull" | "isNotNull"; readonly e: Expr } + | { readonly k: "between"; readonly neg: boolean; readonly e: Expr; readonly lo: Operand; readonly hi: Operand } + | { readonly k: "in"; readonly neg: boolean; readonly e: Expr; readonly values: ReadonlyArray } + | { readonly k: "and" | "or"; readonly style: "method" | "fn"; readonly cs: ReadonlyArray } + | { readonly k: "not"; readonly c: Cond } + | { readonly k: "const"; readonly v: boolean } + /** `$.OrgId.eq(param.string("orgId"))`: the tenant predicate, wherever it lands. */ + | { readonly k: "tenant"; readonly side: "orders" | "customers" } + +export type Agg = + | { readonly k: "count" } + | { readonly k: "countIf"; readonly c: Cond } + | { readonly k: "sum" | "min" | "max"; readonly e: Expr } + +export interface RowsShape { + readonly k: "rows" + readonly select: ReadonlyArray + readonly distinct: boolean + readonly orderBy: ReadonlyArray + readonly limit?: number + readonly offset?: number +} +export interface AggShape { + readonly k: "agg" + readonly keys: ReadonlyArray + readonly aggs: ReadonlyArray + readonly having: ReadonlyArray<{ readonly agg: Agg; readonly op: CmpOp; readonly v: number }> + readonly orderBy: ReadonlyArray + readonly limit?: number + readonly offset?: number +} + +export interface Plan { + /** The value `param.string("orgId")` is given. */ + readonly orgId: string + readonly join?: "inner" | "left" + readonly where: ReadonlyArray + readonly shape: RowsShape | AggShape + /** The same rows through a subquery: `fromQuery(query, "w").select()`, or `.select(...names)`. */ + readonly wrap: false | "all" | "names" + readonly crossTenant: boolean +} + +// Generators + +export interface GenOptions { + /** Strings for comparisons and LIKE patterns: anything, or only what sorts the same everywhere. */ + readonly strings: fc.Arbitrary +} + +/** Mostly the fixture's own numbers (Id 1–5, Amount 5/7/10/20/99), so comparisons split the rows. */ +const smallInt = fc.oneof( + { weight: 2, arbitrary: fc.constantFrom(0, 1, 2, 3, 4, 5, 6, 7, 10, 20, 99) }, + { weight: 1, arbitrary: fc.integer({ min: -20, max: 120 }) }, +) +const divisor = fc.oneof(fc.integer({ min: 1, max: 9 }), fc.integer({ min: -9, max: -1 })) + +const expr = (sort: Sort, joined: boolean, depth: number, options: GenOptions): fc.Arbitrary => { + const cols = (Object.keys(ORDER_COLUMNS) as OrderColumn[]).filter((c) => ORDER_COLUMNS[c] === sort) + const jcols = joined ? (Object.keys(CUSTOMER_COLUMNS) as CustomerColumn[]).filter((c) => CUSTOMER_COLUMNS[c] === sort) : [] + const leaves: Array> = [ + ...(cols.length > 0 ? [fc.constantFrom(...cols).map((col): Expr => ({ k: "col", col }))] : []), + ...(jcols.length > 0 ? [fc.constantFrom(...jcols).map((col): Expr => ({ k: "jcol", col }))] : []), + ] + if (sort === "num") { + leaves.push(smallInt.map((v): Expr => ({ k: "num", v }))) + if (depth > 0) { + const sub = expr("num", joined, depth - 1, options) + leaves.push( + fc.record({ + k: fc.constant("arith" as const), + op: fc.constantFrom("add", "sub", "mul", "div", "mod"), + l: sub, + r: fc.oneof(divisor, sub), + }), + ) + } + } + if (sort === "str") { + leaves.push(fc.oneof(fc.constantFrom(...FIXTURE_STRINGS), options.strings).map((v): Expr => ({ k: "str", v }))) + if (depth > 0) { + leaves.push( + fc.record({ + k: fc.constant("coalesce" as const), + e: expr("nstr", joined, depth - 1, options), + fallback: expr("str", joined, 0, options), + }), + ) + } + } + return fc.oneof(...leaves) +} + +const valueOf = (sort: Sort, options: GenOptions): fc.Arbitrary => { + switch (sort) { + case "num": + return fc.oneof( + { weight: 3, arbitrary: smallInt.map((v): Operand => ({ k: "value", v })) }, + { weight: 3, arbitrary: smallInt.map((v): Operand => ({ k: "param", kind: "int", v })) }, + // A fraction against an integer column: `Amount > 9.5`. + { weight: 1, arbitrary: fc.integer({ min: -40, max: 240 }).map((n): Operand => ({ k: "value", v: n / 2 + 0.25 })) }, + ) + case "str": + case "nstr": + return fc.oneof( + fc.oneof(fc.constantFrom(...FIXTURE_STRINGS), options.strings).map((v): Operand => ({ k: "value", v })), + fc.oneof(fc.constantFrom(...FIXTURE_STRINGS), options.strings).map((v): Operand => ({ k: "param", kind: "string", v })), + ) + case "time": + return fc.oneof( + fc.record({ k: fc.constant("time" as const), ms: time, form: fc.constantFrom("DateTime" as const, "Date" as const, "string" as const) }), + time.map((ms): Operand => ({ k: "param", kind: "dateTime", v: ms })), + ) + } +} + +/** A comparable right-hand side for `sort`: mostly values, sometimes another expression of the sort. */ +const operand = (sort: Sort, joined: boolean, options: GenOptions): fc.Arbitrary => + sort === "time" + ? valueOf(sort, options) + : fc.oneof( + { weight: 3, arbitrary: valueOf(sort, options) }, + { weight: 1, arbitrary: expr(sort === "nstr" ? "str" : sort, joined, 1, options).map((e): Operand => ({ k: "expr", e })) }, + ) + +/** LIKE patterns over the fixture's letters; `\` only ever escapes `%`, `_` or `\`, which both databases agree on. */ +const likePattern = fc + .array(fc.constantFrom("%", "_", "\\%", "\\_", "\\\\", "a", "c", "e", "m", "i", "o", "p", "AC", "G", "'", "s"), { maxLength: 5 }) + .map((parts) => parts.join("")) + +const cond = (joined: boolean, depth: number, options: GenOptions): fc.Arbitrary => { + const sorts: ReadonlyArray = ["num", "str", "nstr", "time"] + const leaf = fc.oneof( + fc.constantFrom(...sorts).chain((sort) => + fc.record({ + k: fc.constant("cmp" as const), + op: fc.constantFrom("eq", "neq", "gt", "gte", "lt", "lte"), + l: expr(sort, joined, 1, options), + r: operand(sort, joined, options), + }), + ), + fc.constantFrom("str", "nstr").chain((sort) => + fc.record({ + k: fc.constant("like" as const), + op: fc.constantFrom("like" as const, "notLike" as const, "ilike" as const), + l: expr(sort, joined, 0, options), + pattern: likePattern, + }), + ), + fc.record({ + k: fc.constant("null" as const), + op: fc.constantFrom("isNull" as const, "isNotNull" as const), + e: fc.oneof(expr("nstr", joined, 0, options), expr("str", joined, 0, options)), + }), + fc.constantFrom("num", "str", "time").chain((sort) => + fc.record({ k: fc.constant("between" as const), neg: fc.boolean(), e: expr(sort, joined, 1, options), lo: valueOf(sort, options), hi: valueOf(sort, options) }), + ), + fc.constantFrom("num", "str", "nstr").chain((sort) => + fc.record({ + k: fc.constant("in" as const), + neg: fc.boolean(), + e: expr(sort, joined, 0, options), + values: fc.array( + sort === "num" ? smallInt : fc.oneof(fc.constantFrom(...FIXTURE_STRINGS), options.strings), + { maxLength: 4 }, + ), + }), + ), + fc.boolean().map((v): Cond => ({ k: "const", v })), + fc.constantFrom({ k: "tenant", side: "orders" }, ...(joined ? [{ k: "tenant", side: "customers" } as const] : [])), + ) + if (depth === 0) return leaf + const sub = cond(joined, depth - 1, options) + return fc.oneof( + { weight: 4, arbitrary: leaf }, + { + weight: 2, + arbitrary: fc.record({ + k: fc.constantFrom("and" as const, "or" as const), + style: fc.constantFrom("method" as const, "fn" as const), + cs: fc.array(sub, { minLength: 2, maxLength: 3 }), + }), + }, + { weight: 1, arbitrary: sub.map((c): Cond => ({ k: "not", c })) }, + ) +} + +const agg = (joined: boolean, options: GenOptions): fc.Arbitrary => + fc.oneof( + fc.constant({ k: "count" }), + cond(joined, 1, options).map((c): Agg => ({ k: "countIf", c })), + fc.record({ k: fc.constantFrom("sum" as const, "min" as const, "max" as const), e: expr("num", joined, 1, options) }), + ) + +const aliasesOf = (n: number, prefix: string) => Array.from({ length: n }, (_, i) => `${prefix}${i}`) + +/** An ORDER BY over `aliases`, completed with `unique` so LIMIT/OFFSET pick a determined set. */ +const ordering = (aliases: ReadonlyArray, unique: ReadonlyArray) => + fc + .record({ + keys: fc.subarray([...aliases]).chain((keys) => fc.tuple(...keys.map((key) => fc.constant(key).chain((k) => fc.constantFrom("asc" as const, "desc" as const).map((d) => [k, d] as const))))), + limit: fc.option(fc.oneof({ weight: 4, arbitrary: fc.integer({ min: 1, max: 6 }) }, { weight: 1, arbitrary: fc.constant(0) }), { nil: undefined }), + offset: fc.option(fc.integer({ min: 0, max: 3 }), { nil: undefined, freq: 3 }), + total: fc.boolean(), + }) + .map(({ keys, limit, offset, total }) => { + const paging = limit !== undefined || offset !== undefined + const complete = paging || total + const tail = complete ? unique.filter((u) => !keys.some(([k]) => k === u)).map((u) => [u, "asc"] as const) : [] + return { orderBy: [...keys, ...tail], limit, offset } + }) + +const rowsShape = (joined: boolean, options: GenOptions): fc.Arbitrary => + fc + .array(fc.constantFrom("num", "str", "nstr", "time").chain((sort) => expr(sort, joined, 2, options)), { minLength: 0, maxLength: 4 }) + .chain((exprs) => { + // `id` is always selected: the unique key that makes an ordering total. + const select: Array = [["id", { k: "col", col: "Id" }], ...exprs.map((e, i) => [`e${i}`, e] as const)] + return fc.record({ distinct: fc.boolean(), order: ordering(select.map(([a]) => a), ["id"]) }).map( + ({ distinct, order }): RowsShape => ({ k: "rows", select, distinct, ...order }), + ) + }) + +const keyExpr = (joined: boolean): fc.Arbitrary => + fc.constantFrom( + { k: "col", col: "Customer" }, + { k: "col", col: "Status" }, + { k: "col", col: "OrgId" }, + { k: "col", col: "Note" }, + ...(joined ? [{ k: "jcol", col: "Tier" } as const, { k: "jcol", col: "Name" } as const] : []), + ) + +const aggShape = (joined: boolean, options: GenOptions): fc.Arbitrary => + fc + .record({ + keys: fc.uniqueArray(keyExpr(joined), { maxLength: 2, selector: (e) => JSON.stringify(e) }), + aggs: fc.array(agg(joined, options), { minLength: 1, maxLength: 3 }), + }) + .chain(({ keys, aggs }) => { + const keyAliases = aliasesOf(keys.length, "k") + const aggAliases = aliasesOf(aggs.length, "a") + return fc + .record({ + having: keys.length === 0 ? fc.constant([]) : fc.array(fc.record({ agg: agg(joined, options), op: fc.constantFrom("eq", "neq", "gt", "gte", "lt", "lte"), v: smallInt }), { maxLength: 2 }), + // A global aggregate is one row; only grouped results page. + order: keys.length === 0 ? fc.constant({ orderBy: [], limit: undefined, offset: undefined }) : ordering([...keyAliases, ...aggAliases], keyAliases), + }) + .map(({ having, order }): AggShape => ({ + k: "agg", + keys: keys.map((e, i) => [keyAliases[i]!, e] as const), + aggs: aggs.map((a, i) => [aggAliases[i]!, a] as const), + having, + ...order, + })) + }) + +export const plan = (options: GenOptions): fc.Arbitrary => + fc.option(fc.constantFrom("inner" as const, "left" as const), { nil: undefined, freq: 3 }).chain((join) => { + const joined = join !== undefined + return fc.record({ + orgId: fc.oneof({ weight: 6, arbitrary: fc.constant("org_1") }, { weight: 2, arbitrary: fc.constant("org_2") }, { weight: 1, arbitrary: fc.constant("org_3") }), + join: fc.constant(join), + // Short: every entry is ANDed, and a long conjunction of random predicates is almost always empty. + where: fc.oneof( + fc.constant([]), + fc.array(fc.oneof({ weight: 1, arbitrary: fc.constant({ k: "tenant", side: "orders" }) }, { weight: 3, arbitrary: cond(joined, 2, options) }), { minLength: 1, maxLength: 2 }), + ), + shape: fc.oneof({ weight: 3, arbitrary: rowsShape(joined, options) }, { weight: 2, arbitrary: aggShape(joined, options) }), + wrap: fc.constantFrom(false as const, "all" as const, "names" as const), + crossTenant: fc.boolean(), + }) + }) + +/** Every string the oracle compares by byte order: plain, so collation never enters. */ +export const plainOptions: GenOptions = { + strings: fc.string({ unit: fc.constantFrom(..."abcdeghiklmnoprstuwxy_' -"), maxLength: 8 }), +} +/** Strings from the nasty generator: for escaping, where the model only needs equality. */ +export const nastyOptions = (dialect: "clickhouse" | "postgres" | "both"): GenOptions => ({ + strings: nastyString(dialect).filter((s) => wellFormed(s)), +}) diff --git a/tests/fuzz/queries.fuzz.test.ts b/tests/fuzz/queries.fuzz.test.ts new file mode 100644 index 0000000..41a5ca5 --- /dev/null +++ b/tests/fuzz/queries.fuzz.test.ts @@ -0,0 +1,72 @@ +// Query semantics: random plans run on a real database and match the reference model. +// +// Each plan is compiled for the target dialect, executed over the core fixture, +// and its decoded rows compared with `model.ts`. On top of the row diff: +// +// tenant soundness a query that reports "single-tenant" read rows of one org only +// +// Postgres (PGlite) always runs. ClickHouse runs twice when an endpoint is set, +// once per `join_use_nulls`, because that setting changes what a LEFT JOIN fills. +import { afterAll, describe, expect, it } from "vitest" +import type { CoreContext } from "../core-cases" +import { contexts, scopeContexts } from "./contexts" +import { compilePlan } from "./build" +import { clickhouseEndpoint, closePostgres, run, type Target } from "./db" +import { config, FuzzFailure, property, skipKnown, withContext } from "./harness" +import { knownPlanIssue } from "./known-issues" +import { evaluate } from "./model" +import { nastyOptions, plainOptions, plan, type Plan } from "./plan" +import { normalize } from "./values" + +afterAll(closePostgres) + +/** Rows in a form that compares across float summation order: non-integers to 12 significant digits. */ +const canonical = (rows: ReadonlyArray): Array => + rows.map((row) => + Object.fromEntries( + Object.entries(normalize(row) as Record).map(([k, v]) => [k, typeof v === "number" && !Number.isInteger(v) ? Number(v.toPrecision(12)) : v]), + ), + ) +const sorted = (rows: Array) => rows.map((r) => JSON.stringify(r)).sort() + +const matchesModel = async (ctx: CoreContext, target: Target, input: Plan) => { + if (!config.strict) { + const known = knownPlanIssue({ target, plan: input }) + if (known !== undefined) skipKnown(known) + } + const expected = evaluate(input, target) + const result = compilePlan(ctx, input) + if (result._tag !== "Compiled") throw new FuzzFailure(`a valid plan was not compiled (${result._tag})`, { error: result.error }) + const { compiled } = result + // The scope a real application would get: the same plan over the plain tables. + const scoped = compilePlan(scopeContexts[ctx.dialect], input) + const tenantScope = scoped._tag === "Compiled" ? scoped.compiled.tenantScope : undefined + const outcome = await run(target, compiled) + await withContext({ sql: compiled.sql, parameters: compiled.parameters, tenantScope, outcome, expected: expected.rows }, () => { + expect(outcome._tag).toBe("Rows") + if (outcome._tag !== "Rows") return + const actual = canonical(outcome.rows) + const want = canonical(expected.rows) + if (expected.ordered) expect(actual).toEqual(want) + else expect(sorted(actual)).toEqual(sorted(want)) + if (tenantScope === "single-tenant") { + const foreign = [...expected.orgs].filter((org) => org !== input.orgId) + expect(foreign, "a single-tenant query read another org's rows").toEqual([]) + } + }) +} + +const targets: ReadonlyArray = clickhouseEndpoint ? ["postgres", "clickhouse", "clickhouse-join-nulls"] : ["postgres"] + +describe("queries match the model", () => { + for (const target of targets) { + const ctx = contexts[target === "postgres" ? "postgres" : "clickhouse"] + for (const [label, options] of [ + ["plain strings", plainOptions], + ["nasty strings", nastyOptions(ctx.dialect)], + ] as const) { + const name = `plans match the model (${target}, ${label})` + it(name, () => property(name, plan(options), (input) => matchesModel(ctx, target, input), { runs: 100 }), 300_000) + } + } +}) diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts new file mode 100644 index 0000000..26e11b9 --- /dev/null +++ b/tests/fuzz/regressions.ts @@ -0,0 +1,68 @@ +// Pinned counterexamples, replayed before any random input. +// +// When a property fails, its report prints the shrunk input; paste it here under +// a key that is a substring of the property's name (so one plan can pin every +// target at once). Inputs that hit an open entry in `known-issues.ts` are +// skipped until the bug is fixed, then replayed for good. +import { DateTime } from "effect" +import type { Plan, RowsShape } from "./plan" + +const rows = (shape: RowsShape): Plan => ({ orgId: "org_1", join: undefined, where: [], shape, wrap: false, crossTenant: false }) + +export const pinned: Record> = { + // Every plan property: "plans compile cleanly" and "plans match the model". + plans: [ + // Documented: a chain follows SQL precedence, `Id + 0 * 2`. The model must agree. + rows({ + k: "rows", + select: [["id", { k: "col", col: "Id" }], ["e0", { k: "arith", op: "mul", l: { k: "arith", op: "add", l: { k: "col", col: "Id" }, r: { k: "num", v: 0 } }, r: 2 }]], + distinct: false, + orderBy: [], + }), + // arith-argument-not-parenthesised: `Id - Amount + 1`. + rows({ + k: "rows", + select: [["id", { k: "col", col: "Id" }], ["e0", { k: "arith", op: "sub", l: { k: "col", col: "Id" }, r: { k: "arith", op: "add", l: { k: "col", col: "Amount" }, r: 1 } }]], + distinct: false, + orderBy: [], + }), + // fromquery-select-all. + { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), wrap: "all" }, + // clickhouse-server-and-absorption. + { + ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), + where: [ + { k: "tenant", side: "orders" }, + { k: "or", style: "method", cs: [{ k: "and", style: "method", cs: [{ k: "const", v: true }, { k: "tenant", side: "orders" }] }, { k: "tenant", side: "orders" }] }, + ], + }, + // pg-fraction-bound-to-integer. + { + ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), + where: [{ k: "cmp", op: "gt", l: { k: "col", col: "Amount" }, r: { k: "value", v: 9.75 } }], + }, + ] satisfies ReadonlyArray, + + // Literals. + "clickhouse:Float64": [5e-324, 2.1722934738885276e269], + "clickhouse:Array(Nullable(Int64)) compare": [[0, 4294967296]], + "clickhouse:String": ["__PARAM_string_x__", "\\'", "\0"], + "postgres:boolean select": [false], + "postgres:text[] select": [[]], + "postgres:timestamptz": [DateTime.makeUnsafe(Date.UTC(100, 0, 1))], + + // Hostile values. + "hostile values are refused, never thrown (clickhouse)": [ + { dialect: "clickhouse", column: "OrgId", site: "eq", value: 0n }, + { dialect: "clickhouse", column: "OrgId", site: "like", value: {} }, + { dialect: "clickhouse", column: "OrgId", site: "select", value: Number.NaN }, + ], + "hostile values are refused, never thrown (postgres)": [ + { dialect: "postgres", column: "OrgId", site: "param.dateTime", value: "" }, + { dialect: "postgres", column: "Created", site: "param.of", value: "2026-13-45 99:99:99" }, + ], +} + +/** Every pinned input whose key appears in `property`'s name. */ +export const pinnedFor = (property: string): ReadonlyArray => + Object.entries(pinned).flatMap(([key, inputs]) => (property.includes(key) ? inputs : [])) diff --git a/tests/fuzz/values.ts b/tests/fuzz/values.ts new file mode 100644 index 0000000..1a929d2 --- /dev/null +++ b/tests/fuzz/values.ts @@ -0,0 +1,214 @@ +// Value generators, and the column types the literal oracle round-trips. +import fc from "fast-check" +import { DateTime } from "effect" +import * as CH from "@maple-dev/effect-orm/clickhouse" +import * as PG from "@maple-dev/effect-orm/postgres" + +export type Dialect = "clickhouse" | "postgres" + +// Strings + +/** + * Fragments that have broken some SQL writer somewhere: quote and escape + * characters, comment openers, every placeholder syntax this library or a + * driver uses, line separators, and code points that change length between + * UTF-16 and UTF-8. + */ +const NASTY = [ + "'", "''", "\\", "\\'", "\\\\", '"', "`", "\n", "\r\n", "\t", "\b", "\f", "\v", "\x1b", "\x7f", + "%", "_", "\\%", "\\_", "--", "/*", "*/", ";", "?", "$1", "$$", ":name", "{p:String}", + "__PARAM_", "__PARAM_string_x__", "NULL", "\\N", "e'", "E'", "U&'", "
", "
", + " ", "", "​", "é", "é", "😀", "👩‍👩‍👧", "\ud800", "\udfff", "Ω", "İ", "ß", +] + +/** Any string, biased hard towards the fragments above. `\0` only where the dialect stores it. */ +export const nastyString = (dialect: Dialect | "both"): fc.Arbitrary => { + const unit = fc.oneof( + { weight: 3, arbitrary: fc.constantFrom(...NASTY) }, + { weight: 2, arbitrary: fc.string({ unit: "binary", maxLength: 3 }) }, + { weight: 2, arbitrary: fc.string({ unit: fc.constantFrom(..."abcxyz019 "), maxLength: 4 }) }, + ...(dialect === "clickhouse" ? [{ weight: 1, arbitrary: fc.constant("\0") }] : []), + ) + return fc + .array(unit, { maxLength: 6 }) + .map((parts) => parts.join("")) + // Postgres `text` cannot hold U+0000, and PGlite's TextDecoder drops a leading BOM: + // the database's and the driver's limits, not the builder's. + .filter((s) => dialect === "clickhouse" || (!s.includes("\0") && !s.startsWith("\ufeff"))) +} + +/** Strings whose byte order every target agrees on: lowercase ASCII and a few symbols. */ +export const plainString = fc.string({ unit: fc.constantFrom(..."abcdeghiklmnoprstuwxy_-' "), maxLength: 8 }) + +// Times + +const MIN_DT = Date.UTC(1970, 0, 1) +const MAX_DT = Date.UTC(2106, 1, 7, 6, 28, 15) // DateTime is a UInt32 of seconds +const MIN_DT64 = Date.UTC(1900, 0, 1) +const MAX_DT64 = Date.UTC(2299, 11, 31, 23, 59, 59, 999) +// PGlite hands timestamps to JS as `Date`s built with two-digit-year rules, so +// years 0–99 come back as 1900–1999 before the builder sees them. +const MIN_PG = Date.UTC(100, 0, 1) +const MAX_PG = Date.UTC(9999, 11, 31, 23, 59, 59, 999) + +const utc = (ms: number): DateTime.Utc => DateTime.makeUnsafe(ms) +const pad = (n: number, width = 2) => String(n).padStart(width, "0") +/** `YYYY-MM-DD hh:mm:ss[.sss]`, written here rather than by the code under test. */ +const wireDateTime = (ms: number, fraction: boolean): string => { + const d = new Date(ms) + const base = `${pad(d.getUTCFullYear(), 4)}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}` + return fraction ? `${base}.${pad(d.getUTCMilliseconds(), 3)}` : base +} + +// Comparing decoded values + +/** + * A decoded value in a form `toEqual` compares meaningfully: a `DateTime` as + * its instant, `-0` as `0`. Applied to both sides of every row comparison. + */ +export const normalize = (value: unknown): unknown => { + if (DateTime.isDateTime(value)) return `@${new Date(DateTime.toEpochMillis(value)).toISOString()}` + if (value instanceof Date) return `@${value.toISOString()}` + if (typeof value === "number") return Object.is(value, -0) ? 0 : value + if (Array.isArray(value)) return value.map(normalize) + if (value !== null && typeof value === "object") { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, normalize(v)])) + } + return value +} + +// The literal catalog + +/** + * A column type and how to put a value of it into a row *without* the builder. + * + * `source` writes a one-row, one-column (`v`) SELECT from the value's JSON — a + * path through the database's own JSON parser, independent of the literal + * writer under test — so comparing the builder's literal against `v` is not + * the builder checking itself. + */ +/** + * The JSON a reference row is parsed from. `__PARAM_` is written `\u005f_PARAM_`: + * the reference SQL is raw text, and the builder rightly refuses raw SQL that + * contains its placeholder prefix. + */ +export const referenceJson = (wire: unknown): string => + JSON.stringify({ v: wire }).replaceAll("__PARAM_", "\\u005f_PARAM_") + +/** Whether every string inside `value` is well-formed UTF-16 (no lone surrogates). */ +export const wellFormed = (value: unknown): boolean => { + // String.prototype.isWellFormed, spelled out (the tests' lib target predates it): + // iterating by code point yields a lone surrogate as itself. + if (typeof value === "string") { + for (const ch of value) { + const code = ch.codePointAt(0)! + if (code >= 0xd800 && code <= 0xdfff) return false + } + return true + } + if (Array.isArray(value)) return value.every(wellFormed) + if (value !== null && typeof value === "object") { + return Object.entries(value).every(([k, v]) => wellFormed(k) && wellFormed(v)) + } + return true +} + +export interface LiteralType { + readonly id: string + readonly dialect: Dialect + readonly type: CH.CHType + readonly value: fc.Arbitrary + /** The value as the JSON the database parses. */ + readonly wire: (value: unknown) => unknown + readonly source: (json: string, value: unknown) => string + /** Whether `$.v.eq(value)` is meaningful for this type. */ + readonly comparable: boolean +} + +const chSource = (sqlType: string) => (json: string) => + `SELECT v FROM format(JSONEachRow, 'v ${sqlType.replaceAll("'", "\\'")}', '${json.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}')` + +const pgJson = (json: string) => `CAST('${json.replaceAll("'", "''")}' AS jsonb)` +const pgScalar = (sqlType: string) => (json: string) => `SELECT CAST(${pgJson(json)}->>'v' AS ${sqlType}) AS "v"` +const pgArray = (sqlType: string) => (json: string) => + `SELECT CAST(ARRAY(SELECT jsonb_array_elements_text(${pgJson(json)}->'v')) AS ${sqlType}[]) AS "v"` + +const ch = ( + id: string, + type: CH.CHType, + value: fc.Arbitrary, + options: { wire?: (value: unknown) => unknown; comparable?: boolean; source?: (value: unknown) => string } = {}, +): LiteralType => ({ + id: `clickhouse:${id}`, + dialect: "clickhouse", + type, + value, + wire: options.wire ?? ((v) => v), + source: (json, v) => (options.source ? options.source(v) : chSource(type.sql)(json)), + comparable: options.comparable ?? true, +}) +const pg = ( + id: string, + type: CH.CHType, + value: fc.Arbitrary, + options: { wire?: (value: unknown) => unknown; array?: boolean } = {}, +): LiteralType => ({ + id: `postgres:${id}`, + dialect: "postgres", + type, + value, + wire: options.wire ?? ((v) => v), + source: options.array ? pgArray(type.sql.replace(/\[\]$/, "")) : pgScalar(type.sql), + comparable: true, +}) + +const finite = fc.double({ noNaN: true, noDefaultInfinity: true }) +const int32 = fc.integer({ min: -(2 ** 31), max: 2 ** 31 - 1 }) +const nullOr = (arb: fc.Arbitrary) => fc.option(arb, { nil: null, freq: 4 }) +const dateTimeWire = (fraction: boolean) => (value: unknown) => + wireDateTime(DateTime.toEpochMillis(value as DateTime.Utc), fraction) + +export const literalTypes: ReadonlyArray = [ + ch("String", CH.string, nastyString("clickhouse")), + ch("Int32", CH.int32, int32), + ch("Int64", CH.int64, fc.maxSafeInteger()), + ch("UInt32", CH.uint32, fc.nat({ max: 2 ** 32 - 1 })), + ch("UInt64", CH.uint64, fc.maxSafeNat()), + // ClickHouse's text float parser (JSON input, `toFloat64('…')`) can be 1 ULP off unless + // precise_float_parsing is on, so the reference row is built from the IEEE bits instead. + ch("Float64", CH.float64, finite, { + source: (v) => { + const view = new DataView(new ArrayBuffer(8)) + view.setFloat64(0, v as number) + return `SELECT reinterpretAsFloat64(toUInt64(${view.getBigUint64(0)})) AS v` + }, + }), + ch("Bool", CH.bool, fc.boolean()), + ch("DateTime", CH.dateTime, fc.integer({ min: MIN_DT / 1000, max: MAX_DT / 1000 }).map((s) => utc(s * 1000)), { + wire: dateTimeWire(false), + }), + ch("DateTime64", CH.dateTime64, fc.integer({ min: MIN_DT64, max: MAX_DT64 }).map(utc), { wire: dateTimeWire(true) }), + ch("Nullable(String)", CH.nullable(CH.string), nullOr(nastyString("clickhouse"))), + ch("Nullable(Int64)", CH.nullable(CH.int64), nullOr(fc.maxSafeInteger())), + ch("Array(String)", CH.array(CH.string), fc.array(nastyString("clickhouse"), { maxLength: 4 })), + ch("Array(Nullable(Int64))", CH.array(CH.nullable(CH.int64)), fc.array(nullOr(fc.maxSafeInteger()), { maxLength: 4 })), + ch("Array(Array(Int32))", CH.array(CH.array(CH.int32)), fc.array(fc.array(int32, { maxLength: 3 }), { maxLength: 3 })), + ch("Map(String, String)", CH.map(CH.string, CH.string), fc.dictionary(nastyString("clickhouse"), nastyString("clickhouse"), { maxKeys: 3 }), { + comparable: false, + }), + ch("Map(String, Int64)", CH.map(CH.string, CH.int64), fc.dictionary(nastyString("clickhouse"), fc.maxSafeInteger(), { maxKeys: 3 }), { + comparable: false, + }), + + pg("text", PG.text, nastyString("postgres")), + pg("int4", PG.int4, int32), + pg("int8", PG.int8, fc.maxSafeInteger()), + pg("float8", PG.float8, finite), + pg("boolean", PG.bool, fc.boolean()), + pg("timestamptz", PG.timestamptz, fc.integer({ min: MIN_PG, max: MAX_PG }).map(utc), { + wire: (value) => new Date(DateTime.toEpochMillis(value as DateTime.Utc)).toISOString(), + }), + pg("nullable(text)", PG.nullable(PG.text), nullOr(nastyString("postgres"))), + pg("nullable(int8)", PG.nullable(PG.int8), nullOr(fc.maxSafeInteger())), + pg("text[]", PG.array(PG.text), fc.array(nastyString("postgres"), { maxLength: 4 }), { array: true }), +]