From e011beedb5f71f4846f6791a7b13defda7ed6a7c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 23 Oct 2026 01:52:23 +0200 Subject: [PATCH] fix(clickhouse): write number literals that read back exactly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two spellings did not round-trip through ClickHouse's SQL parser: - a subnormal float (`5e-324`) is a SYNTAX_ERROR, and parsed from text (`toFloat64('5e-324')`) it flushes to 0; it is now written from its IEEE bits as `reinterpretAsFloat64(toUInt64())`. - a whole double past 2^53 printed as its shortest decimal (`44812297868494010`) is parsed as an exact integer the double is not (…008), so `WHERE x = value` missed the row; it is now written as the double's own integer value. Both apply to column literals, `lit`, and untyped number operands. Found by the literal fuzzer; its known-issue entry is removed. Co-Authored-By: Claude Opus 5.5 --- .changeset/exact-number-literals.md | 5 +++++ src/ch/expr.ts | 10 +++++++--- src/ch/literal.test.ts | 14 ++++++++++++++ src/ch/literal.ts | 28 +++++++++++++++++++++++++++- tests/fuzz/known-issues.ts | 8 -------- tests/fuzz/regressions.ts | 2 +- 6 files changed, 54 insertions(+), 13 deletions(-) create mode 100644 .changeset/exact-number-literals.md diff --git a/.changeset/exact-number-literals.md b/.changeset/exact-number-literals.md new file mode 100644 index 0000000..69a776f --- /dev/null +++ b/.changeset/exact-number-literals.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": patch +--- + +ClickHouse number literals read back as exactly the value you passed. A subnormal float (`5e-324`) was a syntax error and is now written from its bits; a whole number past 2^53 (`44812297868494010`) was written as its shortest decimal, which ClickHouse parses as a different integer, so `WHERE x = value` could miss the row — it is now written as the double's exact integer value. diff --git a/src/ch/expr.ts b/src/ch/expr.ts index 74eb691..8420b42 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -12,7 +12,7 @@ import { raw, str, ident, compile, as_ as sqlAs, known } from "../sql/sql-fragme import { activeSqlSyntax } from "../sql/sql-syntax" import { activeLiteralBinder } from "../sql/literal-binder" import { chDateTimeLiteral, CHFloatResult, CHNumber, string as chString, type CHType, type InferEncoded, type InferTS } from "./types" -import { encodeColumnLiteral } from "./literal" +import { encodeColumnLiteral, isSubnormal, numberText, sqlLiteral } from "./literal" import { QueryBuilderError } from "./errors" import { fail } from "./failure" import { markTenantColumn, markTenantPredicate, tenantColumnOf, tenantPredicatesOf } from "./tenant" @@ -300,7 +300,7 @@ export const isExprLike = (value: unknown): value is Expr => export function toFragment(value: unknown): SqlFragment { if (isExprLike(value)) return value.toFragment() if (typeof value === "string") return str(value) - if (typeof value === "number") return raw(String(value)) + if (typeof value === "number") return numberFragment(value) if (typeof value === "boolean") return known(() => untypedLiteral(value)) // A DateTime column compares against a DateTime value, so the literal has to // be the dialect's own form (ClickHouse's is tz-less) rather than whatever @@ -328,6 +328,10 @@ const describeUntyped = (value: unknown): string => /** A value with no column type to encode it, in the active dialect's syntax. * ClickHouse writes booleans as `1`/`0`, which is also the rendering outside * a compile. */ +/** A number as SQL that reads back exactly. A subnormal goes through the dialect: ClickHouse cannot parse one as written. */ +const numberFragment = (value: number): SqlFragment => + isSubnormal(value) ? known(() => (activeSqlSyntax()?.literal ?? sqlLiteral)(value, "a number literal")) : raw(numberText(value)) + const untypedLiteral = (value: boolean): string => activeSqlSyntax()?.literal(value, "an untyped boolean") ?? (value ? "1" : "0") @@ -639,7 +643,7 @@ export function lit(value: string | number): Expr | Expr { // a single `CH.lit("all")` in a SELECT costs the whole query its row schema, // since derivation is all-or-nothing. if (typeof value === "string") return makeExpr(str(value), chString.schema) - return makeExpr(raw(String(value)), CHNumber as Schema.Codec) + return makeExpr(typeof value === "number" ? numberFragment(value) : raw(String(value)), CHNumber as Schema.Codec) } // Subquery expressions diff --git a/src/ch/literal.test.ts b/src/ch/literal.test.ts index 5ae367c..304c0d0 100644 --- a/src/ch/literal.test.ts +++ b/src/ch/literal.test.ts @@ -122,6 +122,20 @@ describe("sqlLiteral", () => { expect(sqlLiteral([{ a: "b" }, { c: "d" }], "test")).toBe("[map('a', 'b'), map('c', 'd')]") }) + it("writes a subnormal float from its bits, the one spelling ClickHouse parses exactly", () => { + expect(sqlLiteral(5e-324, "test")).toBe("reinterpretAsFloat64(toUInt64(1))") + expect(sqlLiteral(-5e-324, "test")).toBe("reinterpretAsFloat64(toUInt64(9223372036854775809))") + expect(sqlLiteral(2.2250738585072014e-308, "test")).toBe("2.2250738585072014e-308") + expect(sqlLiteral(0.1, "test")).toBe("0.1") + }) + + it("writes a whole double past 2^53 as its exact integer, not its shortest decimal", () => { + expect(sqlLiteral(44812297868494010, "test")).toBe("44812297868494008") + expect(sqlLiteral(-(2 ** 60), "test")).toBe("-1152921504606846976") + expect(sqlLiteral(2 ** 53 - 1, "test")).toBe("9007199254740991") + expect(sqlLiteral(1e300, "test")).toBe("1e+300") + }) + it("refuses what has no ClickHouse literal", () => { const failureOf = (value: unknown) => collectFailures(() => sqlLiteral(value, "test")).failures[0]?.message expect(failureOf(Number.NaN)).toMatch(/no ClickHouse literal/) diff --git a/src/ch/literal.ts b/src/ch/literal.ts index 6da9561..053ff25 100644 --- a/src/ch/literal.ts +++ b/src/ch/literal.ts @@ -43,6 +43,32 @@ const oneLine = (failure: unknown): string => .replace(/\s*\n\s*/g, "; ") .trim() +/** Below the smallest normal double: a value ClickHouse's SQL parser cannot read back. */ +export const isSubnormal = (value: number): boolean => value !== 0 && Math.abs(value) < 2.2250738585072014e-308 + +/** + * A finite double as digits that read back as exactly that double. + * + * `String(value)` is the shortest decimal that rounds to it, which a float + * parser rounds back — but a whole number past 2^53 prints as an integer + * (`44812297868494010`), and an integer literal is parsed exactly, as a value + * the double is not (it is …008). Below 2^64 the double's own integer value is + * written instead; above, `String` already uses an exponent. + */ +export const numberText = (value: number): string => + Number.isInteger(value) && !Number.isSafeInteger(value) && Math.abs(value) < 2 ** 64 ? BigInt(value).toString() : String(value) + +/** + * A subnormal double as ClickHouse SQL. Written as a decimal (`5e-324`) it is a + * syntax error, and parsed from text (`toFloat64('5e-324')`) it flushes to 0, so + * it is rebuilt from its IEEE bits, which is exact. + */ +const subnormalLiteral = (value: number): string => { + const view = new DataView(new ArrayBuffer(8)) + view.setFloat64(0, value) + return `reinterpretAsFloat64(toUInt64(${view.getBigUint64(0)}))` +} + /** * A wire value as ClickHouse literal syntax. * @@ -62,7 +88,7 @@ export function sqlLiteral(value: unknown, context: string): string { message: `${context}: ${value} has no ClickHouse literal`, }), "NULL") } - return String(value) + return isSubnormal(value) ? subnormalLiteral(value) : numberText(value) } if (Array.isArray(value)) { diff --git a/tests/fuzz/known-issues.ts b/tests/fuzz/known-issues.ts index 8d787e9..ec5a21b 100644 --- a/tests/fuzz/known-issues.ts +++ b/tests/fuzz/known-issues.ts @@ -19,15 +19,7 @@ export interface LiteralInput { readonly path: LiteralPath } -const isFloat64 = (literal: LiteralType) => literal.dialect === "clickhouse" && literal.type.sql === "Float64" - const literalIssues: ReadonlyArray> = [ - { - id: "ch-float-subnormal-syntax", - reason: "A subnormal float (`5e-324`) written as String(value) is a ClickHouse SYNTAX_ERROR.", - matches: ({ literal, value }) => - isFloat64(literal) && typeof value === "number" && value !== 0 && Math.abs(value) < 2.2250738585072014e-308, - }, { id: "ch-array-literal-untyped", reason: diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index 26e11b9..7d8d870 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -44,7 +44,7 @@ export const pinned: Record> = { ] satisfies ReadonlyArray, // Literals. - "clickhouse:Float64": [5e-324, 2.1722934738885276e269], + "clickhouse:Float64": [5e-324, -5e-324, 44812297868494010, 2.1722934738885276e269], "clickhouse:Array(Nullable(Int64)) compare": [[0, 4294967296]], "clickhouse:String": ["__PARAM_string_x__", "\\'", "\0"], "postgres:boolean select": [false],