fix(webapp): stop dropping the newest bucket from queue backlog sparklines

The sparkline grid floors its start and ceils its end onto the bucket
boundary so repeated loads share ClickHouse query-cache entries, but the
bucket count was measured from the raw requested range. Since the range
starts at "now minus the period", it is almost never already on a
boundary, so the aligned grid spans one more bucket than the count
allowed and the newest bucket was discarded on every load, for every
period. Depth is forward filled, so the sparkline showed the previous
bucket's depth rather than looking broken, and a throttle in the newest
bucket was invisible.

The grid arithmetic moves to a pure module so it can be tested, and the
count is now measured across the aligned grid.

Author: ericallam

apps/webapp/app/presenters/v3/QueueMetricsPresenter.server.ts

@@ -1,6 +1,7 @@
 import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
 import { logger } from "~/services/logger.server";
+import { computeSparklineGrid } from "~/v3/queueSparklineGrid";
 
 export type QueueListMetric = {
   p50WaitMs: number | null;
@@ -24,8 +25,6 @@ export type QueueListMetrics = {
   byQueue: Map<string, QueueListMetric>;
 };
 
-const SPARKLINE_POINTS = 48;
-
 function formatClickhouseDateTime(date: Date): string {
   return date.toISOString().slice(0, 19).replace("T", " ");
 }
@@ -51,13 +50,9 @@ export class QueueMetricsPresenter {
     from: Date;
     to: Date;
   }): Promise<QueueListMetrics> {
-    const rangeSeconds = Math.max(60, Math.round((to.getTime() - from.getTime()) / 1000));
-    const bucketSeconds = Math.max(60, Math.round(rangeSeconds / SPARKLINE_POINTS));
-    const numBuckets = Math.max(1, Math.ceil(rangeSeconds / bucketSeconds));
-    const gridStartSeconds =
-      Math.floor(Math.floor(from.getTime() / 1000) / bucketSeconds) * bucketSeconds;
-    const bucketStartMs = gridStartSeconds * 1000;
-    const bucketIntervalMs = bucketSeconds * 1000;
+    const grid = computeSparklineGrid(from, to);
+    const { bucketSeconds, bucketIntervalMs, bucketStartMs } = grid;
+    const numBuckets = grid.bucketCount;
 
     const empty: QueueListMetrics = {
       bucketStartMs,
@@ -75,9 +70,7 @@ export class QueueMetricsPresenter {
         "queueMetrics"
       );
 
-      // End bound snaps up to the bucket grid so repeated loads within a bucket produce
-      // identical params and share ClickHouse query-cache entries.
-      const endMs = Math.ceil(to.getTime() / bucketIntervalMs) * bucketIntervalMs;
+      const endMs = grid.endMs;
       const ids = {
         organizationId: environment.organizationId,
         projectId: environment.projectId,

apps/webapp/app/v3/queueSparklineGrid.ts

@@ -0,0 +1,54 @@
+/**
+ * Bucket grid for the Queues list sparklines. Pure so it can be unit tested: the presenter that
+ * uses it reaches ClickHouse, and the grid arithmetic is where the off-by-one lives.
+ *
+ * The grid start floors to a bucket boundary and the end ceils to one, so repeated loads inside a
+ * bucket produce identical query params and share ClickHouse query-cache entries. That means the
+ * span covered by the grid is wider than the requested range whenever `from` is not already on a
+ * boundary, which is almost always. `bucketCount` is therefore measured from the aligned grid
+ * start rather than from the raw range, otherwise the newest bucket lands at an index the caller
+ * treats as out of range and its data is silently dropped.
+ */
+
+export const SPARKLINE_POINTS = 48;
+
+const MIN_RANGE_SECONDS = 60;
+const MIN_BUCKET_SECONDS = 60;
+
+export type SparklineGrid = {
+  bucketSeconds: number;
+  bucketIntervalMs: number;
+  /** Aligned grid start; at or before the requested `from`. */
+  bucketStartMs: number;
+  /** Aligned grid end; at or after the requested `to`. */
+  endMs: number;
+  /** Buckets spanning bucketStartMs..endMs, so every bucket the query can return has an index. */
+  bucketCount: number;
+};
+
+export function computeSparklineGrid(from: Date, to: Date): SparklineGrid {
+  const rangeSeconds = Math.max(
+    MIN_RANGE_SECONDS,
+    Math.round((to.getTime() - from.getTime()) / 1000)
+  );
+  const bucketSeconds = Math.max(MIN_BUCKET_SECONDS, Math.round(rangeSeconds / SPARKLINE_POINTS));
+  const bucketIntervalMs = bucketSeconds * 1000;
+
+  const gridStartSeconds =
+    Math.floor(Math.floor(from.getTime() / 1000) / bucketSeconds) * bucketSeconds;
+  const bucketStartMs = gridStartSeconds * 1000;
+  const endMs = Math.ceil(to.getTime() / bucketIntervalMs) * bucketIntervalMs;
+
+  const bucketCount = Math.max(1, Math.round((endMs - bucketStartMs) / bucketIntervalMs));
+
+  return { bucketSeconds, bucketIntervalMs, bucketStartMs, endMs, bucketCount };
+}
+
+/** Index of a bucket on the grid, or null when it falls outside it. */
+export function bucketIndex(grid: SparklineGrid, bucketMs: number): number | null {
+  const index = Math.round((bucketMs - grid.bucketStartMs) / grid.bucketIntervalMs);
+  if (index < 0 || index >= grid.bucketCount) {
+    return null;
+  }
+  return index;
+}

apps/webapp/test/queueSparklineGrid.test.ts

@@ -0,0 +1,70 @@
+import { describe, expect, it } from "vitest";
+import { bucketIndex, computeSparklineGrid, SPARKLINE_POINTS } from "~/v3/queueSparklineGrid";
+
+const PERIODS_MS = {
+  "30m": 30 * 60 * 1000,
+  "1h": 60 * 60 * 1000,
+  "1d": 24 * 60 * 60 * 1000,
+  "7d": 7 * 24 * 60 * 60 * 1000,
+  "30d": 30 * 24 * 60 * 60 * 1000,
+};
+
+/** Newest bucket the ClickHouse query can return: the last full slot before the grid end. */
+function newestBucketMs(grid: ReturnType<typeof computeSparklineGrid>): number {
+  return grid.endMs - grid.bucketIntervalMs;
+}
+
+describe("computeSparklineGrid", () => {
+  it("keeps the newest bucket for every period, at any start offset", () => {
+    for (const [label, span] of Object.entries(PERIODS_MS)) {
+      for (let offset = 0; offset < 120; offset++) {
+        const toMs = 1785000000000 + offset * 1371;
+        const grid = computeSparklineGrid(new Date(toMs - span), new Date(toMs));
+
+        expect(
+          bucketIndex(grid, newestBucketMs(grid)),
+          `${label} dropped its newest bucket at offset ${offset}`
+        ).not.toBeNull();
+      }
+    }
+  });
+
+  it("indexes every bucket the query can return, and nothing outside the grid", () => {
+    const toMs = 1785000000123;
+    const grid = computeSparklineGrid(new Date(toMs - PERIODS_MS["1d"]), new Date(toMs));
+
+    expect(bucketIndex(grid, grid.bucketStartMs)).toBe(0);
+    expect(bucketIndex(grid, newestBucketMs(grid))).toBe(grid.bucketCount - 1);
+    expect(bucketIndex(grid, grid.bucketStartMs - grid.bucketIntervalMs)).toBeNull();
+    expect(bucketIndex(grid, grid.endMs)).toBeNull();
+  });
+
+  it("aligns the grid outward from the requested range", () => {
+    const from = new Date(1785000000123);
+    const to = new Date(from.getTime() + PERIODS_MS["1h"]);
+    const grid = computeSparklineGrid(from, to);
+
+    expect(grid.bucketStartMs).toBeLessThanOrEqual(from.getTime());
+    expect(grid.endMs).toBeGreaterThanOrEqual(to.getTime());
+    expect(grid.bucketStartMs % grid.bucketIntervalMs).toBe(0);
+    expect(grid.endMs % grid.bucketIntervalMs).toBe(0);
+  });
+
+  it("targets the sparkline resolution without going below a one minute bucket", () => {
+    const toMs = 1785000000000;
+    const dayGrid = computeSparklineGrid(new Date(toMs - PERIODS_MS["1d"]), new Date(toMs));
+    expect(dayGrid.bucketSeconds).toBe(1800);
+    expect(dayGrid.bucketCount).toBeGreaterThanOrEqual(SPARKLINE_POINTS);
+
+    const shortGrid = computeSparklineGrid(new Date(toMs - 60_000), new Date(toMs));
+    expect(shortGrid.bucketSeconds).toBe(60);
+    expect(shortGrid.bucketCount).toBeGreaterThanOrEqual(1);
+  });
+
+  it("never returns a zero-bucket grid for a degenerate range", () => {
+    const at = new Date(1785000000000);
+    const grid = computeSparklineGrid(at, at);
+    expect(grid.bucketCount).toBeGreaterThanOrEqual(1);
+    expect(grid.bucketSeconds).toBe(60);
+  });
+});