fix(core,driver-memory,service-analytics): sum / avg add with one compensated sum on every face (#20544)

.changeset/20544-compensated-sum-every-face.md

@@ -0,0 +1,47 @@
+---
+'@objectstack/core': minor
+'@objectstack/objectql': patch
+'@objectstack/driver-memory': patch
+'@objectstack/service-analytics': patch
+---
+
+fix: `sum` / `avg` answer the same double on every face the platform owns, added with one compensated fold that `@objectstack/core` now exports as `compensatedSum` (#20544)
+
+Clause-②: yes
+
+**New export.** `@objectstack/core` exports `compensatedSum(nums)`: the sum of
+`nums`, added in order with Kahan-Babuska-Neumaier compensation, which is the
+summation SQLite (3.43 and later) uses for its own `sum` and `avg`. It moved
+here from `@objectstack/objectql`'s rows path (`in-memory-aggregation.ts`),
+which now imports it instead of keeping a private copy.
+
+**What changed.** Three folds still added a group's values naively, and now call
+the same function:
+
+- `@objectstack/driver-memory`'s `aggregate()` and `find()` with aggregations,
+  the path `engine.aggregate` takes on an in-memory datasource;
+- `@objectstack/driver-memory`'s analytics face (`MemoryAnalyticsService`),
+  whose `sum` / `avg` measures are now a `$group` `$accumulator` in place of
+  mingo's `$sum` / `$avg`;
+- `@objectstack/service-analytics`' draft preview.
+
+Over a `number` column holding `0.1`, `0.2` and `0.3`, each of them answered
+`0.6000000000000001` / `0.20000000000000004`. They now answer `0.6` /
+`0.19999999999999998`, as SQLite and the engine's rows path do. Over
+`1e16, 1, -1e16` they answered `0` and now answer `1`. On driver-memory,
+`engine.aggregate` gave two answers depending on its path: `having { s: { $eq:
+0.6 } }` kept the group on the rows path and dropped it on the native path. It
+now keeps it on both.
+
+**What did not move.** Two addends, integers whose running total stays within
+2^53, and a non-finite total give the same answer as before. Which values count
+as addends did not change either: booleans as 1 / 0, and nulls and non-numeric
+strings left out, as each face already had it. `count`, `min` and `max` are
+untouched. The analytics face's pipeline dump (`result.sql`) now renders the
+accumulator's functions by name, so a `sum` measure and an `avg` measure still
+dump differently.
+
+**Residual.** PostgreSQL and MySQL add their doubles natively without
+compensation, and the platform does not wrap that arithmetic. So over three or
+more fractions their native path can still differ from these faces in the last
+place. An exact `$eq` on a fractional sum compares doubles; compare with a range.

packages/core/src/index.ts

@@ -49,6 +49,14 @@ export * from './utils/env.js';
 // Export timezone-aware calendar utilities (ADR-0053 Phase 2)
 export * from './utils/datetime.js';
 
+// [#20544] The ONE compensated fold for `sum` / `avg`, hoisted from
+// `@objectstack/objectql`'s rows path for the reason `bucketDateKey` above
+// was: `driver-memory`'s two faces and `service-analytics`' draft preview add
+// a group's values in JavaScript too, neither package has objectql among its
+// runtime dependencies, and a copy per face is how one `sum` came to answer
+// two doubles.
+export * from './utils/compensated-sum.js';
+
 // Export the shared batched-write helper (framework#2678)
 export * from './utils/bulk-write.js';
 

packages/core/src/utils/compensated-sum.test.ts

@@ -0,0 +1,68 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * [#20544] `compensatedSum` — the one compensated fold every platform face that
+ * adds a group's values in JavaScript calls (objectql's rows path,
+ * driver-memory's data and analytics faces, service-analytics' draft preview).
+ *
+ * The expected values are SQLite's own `sum` answers, measured by #20489 with
+ * better-sqlite3 (SQLite 3.53.4), sql.js (3.49.1) and @libsql/client (3.45.1);
+ * the naive fold each face used before is asserted beside them, so a fixture
+ * that cannot tell the two folds apart fails here instead of passing vacuously.
+ * Each face pins the same fixtures through its own door, in its own package.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { compensatedSum } from './compensated-sum';
+
+/** The fold the faces used before: in order, one addition at a time. */
+const naiveSum = (xs: readonly number[]) => xs.reduce((a, b) => a + b, 0);
+
+describe('[#20544] compensatedSum — the one compensated fold', () => {
+  it("the card's fixture: 0.1 + 0.2 + 0.3 is SQLite's 0.6, not the naive 0.6000000000000001", () => {
+    expect(naiveSum([0.1, 0.2, 0.3])).toBe(0.6000000000000001);
+    expect(compensatedSum([0.1, 0.2, 0.3])).toBe(0.6);
+    // The mean every avg arm divides out of it: SQLite's 0.19999999999999998.
+    expect(compensatedSum([0.1, 0.2, 0.3]) / 3).toBe(0.19999999999999998);
+  });
+
+  it('a large cancellation keeps the small addend: 1e16 + 1 - 1e16 is 1, not 0', () => {
+    expect(naiveSum([1e16, 1, -1e16])).toBe(0);
+    expect(compensatedSum([1e16, 1, -1e16])).toBe(1);
+    expect(compensatedSum([1e16, 0.5, -1e16])).toBe(0.5);
+  });
+
+  it('two addends are unchanged: the compensated a + b is the naive one', () => {
+    for (const pair of [[0.1, 0.2], [0.7, 0.1], [1e16, 1], [-0.3, 0.1]]) {
+      expect(compensatedSum(pair), `${pair}`).toBe(naiveSum(pair));
+    }
+    expect(compensatedSum([0.1, 0.2])).toBe(0.30000000000000004);
+  });
+
+  it('integers whose partial sums stay within 2^53 are unchanged', () => {
+    for (const ints of [[1, 2, 3, 40, 500], [-7, 3, 12, 0, 9_000_000_000]]) {
+      const s = compensatedSum(ints);
+      expect(s, `${ints}`).toBe(naiveSum(ints));
+      expect(Number.isInteger(s)).toBe(true);
+    }
+    expect(compensatedSum([1, 2, 3, 40, 500])).toBe(546);
+  });
+
+  it('an empty list is 0, and one addend is itself', () => {
+    expect(compensatedSum([])).toBe(0);
+    expect(compensatedSum([0.1])).toBe(0.1);
+    expect(compensatedSum([-2.5])).toBe(-2.5);
+  });
+
+  it('a non-finite total is the naive one, as SQLite returns its running sum when the error term overflows', () => {
+    for (const values of [
+      [Infinity, 1, 2],
+      [1, -Infinity, 0.3],
+      [1e308, 1e308, -1e308],
+      [Infinity, -Infinity, 1],
+      [NaN, 0.1, 0.2],
+    ]) {
+      expect(Object.is(compensatedSum(values), naiveSum(values)), `${values}`).toBe(true);
+    }
+  });
+});

packages/core/src/utils/compensated-sum.ts

@@ -0,0 +1,60 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * [#20489, #20544] The sum of `nums`, added in order with
+ * Kahan-Babuska-Neumaier compensation — the summation SQLite (3.43 and later)
+ * uses for its own `sum` and `avg`, transcribed from its
+ * `kahanBabuskaNeumaierStep` and the finalizers' overflow guard.
+ *
+ * Why: a naive fold (`reduce((a, b) => a + b, 0)`) and SQLite answer two
+ * different doubles for the same values. A `number` column holding `0.1`,
+ * `0.2` and `0.3` sums to `0.6` on SQLite and to `0.6000000000000001` naively
+ * (`avg` `0.19999999999999998` against `0.20000000000000004`), so
+ * `having { s: { $eq: 0.6 } }` kept a group on one face and dropped it on
+ * another. Compensated, the faces agree, and the answer is the more accurate
+ * one (`1e16 + 1 - 1e16` is `1`, not `0`).
+ *
+ * ## Why it lives here
+ *
+ * Every platform fold that adds a group's values in JavaScript calls this one
+ * function, so a `sum` is the same double on every face the platform owns:
+ *
+ * - `@objectstack/objectql`'s rows path (`in-memory-aggregation.ts`, the fold
+ *   `engine.aggregate` runs itself), where it was written;
+ * - `@objectstack/driver-memory`'s native `aggregate` (`memory-driver.ts`,
+ *   `computeAggregate`) and its analytics face (`memory-analytics.ts`, the
+ *   `sum` / `avg` measure accumulator);
+ * - `@objectstack/service-analytics`' draft preview (`preview-evaluator.ts`,
+ *   the `sum` / `avg` arms).
+ *
+ * Neither `driver-memory` nor `service-analytics` has objectql among its
+ * runtime dependencies, and this is the package all three already stand on —
+ * the same reason `bucketDateKey` lives here. A second transcription is how a
+ * face comes to answer its own double again.
+ *
+ * ## What does not move
+ *
+ * Two addends (the compensated `a + b` IS the naive one), integers whose
+ * partial sums stay within 2^53 (every addition is exact), and a non-finite
+ * total. `s` below is exactly the naive running sum; once it overflows or
+ * meets a NaN, the error term is non-finite and the naive answer is returned
+ * as it was, which is SQLite's rule too. An empty list sums to `0`. Which
+ * values count as addends, and what an empty group answers, stay each
+ * caller's own rule: this function only adds.
+ *
+ * ⚠️ Residual, stated: PostgreSQL and MySQL add their doubles natively without
+ * compensation, and the platform does not wrap that arithmetic, so over three
+ * or more fractions their native path can still differ from this one in the
+ * last place. An exact `$eq` on a fractional sum compares doubles; compare
+ * with a range.
+ */
+export function compensatedSum(nums: readonly number[]): number {
+  let s = 0;
+  let c = 0;
+  for (const r of nums) {
+    const t = s + r;
+    c += Math.abs(s) > Math.abs(r) ? (s - t) + r : (r - t) + s;
+    s = t;
+  }
+  return Number.isFinite(c) ? s + c : s;
+}

packages/drivers/driver-memory/src/memory-analytics.ts

@@ -28,6 +28,10 @@ import {
   bucketDateKey,
   isBucketGranularity,
   type BucketGranularity,
+  // [#20544] The ONE compensated fold for `sum` / `avg`, the one objectql's
+  // rows path, this package's data face and SQLite add with — see
+  // {@link compensatedAddendAccumulator}.
+  compensatedSum,
 } from '@objectstack/core';
 // [#16178] The pipeline below is split at its `$group` when a time dimension
 // buckets, so the bucket key can be folded in JS between the two halves — mingo
@@ -647,6 +651,80 @@ function numericAggregandExpr(path: string): Record<string, unknown> {
   return { $cond: [{ $eq: [{ $type: path }, 'bool'] }, { $cond: [path, 1, 0] }, path] };
 }
 
+/**
+ * [#20544] A `sum` or `avg` measure as ONE `$group` accumulator that adds with
+ * `@objectstack/core`'s {@link compensatedSum} — the fold objectql's rows path,
+ * this package's data face (`memory-driver.ts`, `computeAggregate`) and SQLite
+ * add with.
+ *
+ * ## What it replaced
+ *
+ * mingo's `$sum` and `$avg` add in a plain loop, so this face answered
+ * `0.6000000000000001` / `0.20000000000000004` over `0.1`, `0.2` and `0.3`
+ * where the rows path and SQLite answer `0.6` / `0.19999999999999998`, and
+ * `1e16, 1, -1e16` summed to `0` rather than `1`.
+ *
+ * ## Why an `$accumulator`, measured against the other two routes (mingo 7.2.4)
+ *
+ * - **A post-group recompute** is ruled out by the reason in
+ *   {@link numericAggregandExpr}'s header: it runs after the pipeline's own
+ *   `$sort` and `$limit`, so `order` over a `sum` measure would rank the value
+ *   the measure does not answer.
+ * - **A custom accumulator operator** cannot replace `$sum` / `$avg` through
+ *   the `mingo` entry point this package imports: its `Aggregator` merges the
+ *   default operators first (`Context.from`), and `addOps` keeps an operator
+ *   already present, so a caller's context can only ADD a name. A new name
+ *   would have to be registered where each `Aggregator` is built — the
+ *   driver's public `aggregate()` and {@link MemoryAnalyticsService}'s own
+ *   time-bucket half — widening the pipeline dialect the driver accepts.
+ * - **`$accumulator`** is in mingo's default operator set and needs
+ *   `scriptEnabled`, which `ComputeOptions.init` defaults to `true`; both
+ *   `Aggregator`s this face runs take the default options. It stays inside the
+ *   `$group` stage, so every later stage sees the finished number.
+ *
+ * ## What does not move
+ *
+ * Only the addition. The aggregand is {@link numericAggregandExpr}, as before,
+ * and the addends are the values mingo's own `$sum` / `$avg` add: numbers,
+ * NaN excluded (mingo's `isNumber`), so null, a missing key and a non-numeric
+ * string stay ignored. `avg` over no addend is `null`, `sum` over none is `0`,
+ * exactly as mingo answered. The values are added in the group's row order,
+ * the order mingo's `$push` collects them in, so the naive running sum inside
+ * {@link compensatedSum} is the one `$sum` computed.
+ *
+ * The functions are named, and {@link pipelineDumpReplacer} renders a function
+ * by its name, so the pipeline dump still says which fold a measure runs.
+ */
+function compensatedAddendAccumulator(path: string, fn: 'sum' | 'avg'): Record<string, unknown> {
+  return {
+    $accumulator: {
+      init: startAddends,
+      accumulateArgs: [numericAggregandExpr(path)],
+      accumulate: collectAddend,
+      finalize: fn === 'sum' ? compensatedSumOfAddends : compensatedMeanOfAddends,
+      lang: 'js',
+    },
+  };
+}
+
+function startAddends(): number[] {
+  return [];
+}
+
+/** mingo's `isNumber`: the values its `$sum` and `$avg` add. */
+function collectAddend(addends: number[], value: unknown): number[] {
+  if (typeof value === 'number' && !Number.isNaN(value)) addends.push(value);
+  return addends;
+}
+
+function compensatedSumOfAddends(addends: readonly number[]): number {
+  return compensatedSum(addends);
+}
+
+function compensatedMeanOfAddends(addends: readonly number[]): number | null {
+  return addends.length === 0 ? null : compensatedSum(addends) / addends.length;
+}
+
 /**
  * [#7853] A `JSON.stringify` replacer that renders a `RegExp` operand instead of
  * dropping it — the one value type the pipeline dump carries that
@@ -700,7 +778,13 @@ function numericAggregandExpr(path: string): Record<string, unknown> {
  * EXECUTION, before this dump is ever built, so no replacer here reaches it.
  */
 function pipelineDumpReplacer(_key: string, value: unknown): unknown {
-  return value instanceof RegExp ? `/${value.source}/${value.flags}` : value;
+  if (value instanceof RegExp) return `/${value.source}/${value.flags}`;
+  // [#20544] A function is the other value `JSON.stringify` erases, and the
+  // `sum` / `avg` `$accumulator` carries three ({@link
+  // compensatedAddendAccumulator}). Dropped, the two measures dump identically;
+  // by name, the dump still says which fold each one runs.
+  if (typeof value === 'function') return `[function ${value.name}]`;
+  return value;
 }
 
 /**
@@ -1728,10 +1812,12 @@ export class MemoryAnalyticsService implements IAnalyticsService {
     switch (measure.type) {
       case 'count':
         return { $sum: 1 };
+      // [#20544] Compensated, as every other face the platform owns adds —
+      // see {@link compensatedAddendAccumulator}.
       case 'sum':
-        return { $sum: numericAggregandExpr(`$${fieldPath}`) };
+        return compensatedAddendAccumulator(`$${fieldPath}`, 'sum');
       case 'avg':
-        return { $avg: numericAggregandExpr(`$${fieldPath}`) };
+        return compensatedAddendAccumulator(`$${fieldPath}`, 'avg');
       // [#11152] `min`/`max` take the SAME boolean coercion as `sum`/`avg` —
       // maintainer ruling 2026-08-28 (superseding #11249's `false`/`true`):
       // booleans aggregate as NUMBERS on every face, no per-aggregate