Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 15 additions & 28 deletions workers/grouper/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import TimeMs from '../../../lib/utils/time';
import DataFilter from './data-filter';
import RedisHelper from './redisHelper';
import { computeDelta } from './utils/repetitionDiff';
import { bucketTimestampMs } from './utils/bucketTimestamp';
import { rightTrim } from '../../../lib/utils/string';
import { hasValue } from '../../../lib/utils/hasValue';

Expand All @@ -48,6 +49,11 @@ const CACHE_CLEANUP_INTERVAL_SECONDS = 30;
*/
const DB_DUPLICATE_KEY_ERROR = '11000';

/**
* Retention period for daily Redis TimeSeries metrics in days
*/
const DAILY_METRICS_RETENTION_DAYS = 90;

/**
* Maximum length for backtrace code line or title
*/
Expand Down Expand Up @@ -343,37 +349,18 @@ export default class GrouperWorker extends Worker {
};

const series = [
{
key: minutelyKey,
label: 'minutely',
retentionMs: TimeMs.DAY,
},
{
key: hourlyKey,
label: 'hourly',
retentionMs: TimeMs.WEEK,
},
{
key: dailyKey,
label: 'daily',
// eslint-disable-next-line @typescript-eslint/no-magic-numbers
retentionMs: 90 * TimeMs.DAY,
},
{ key: minutelyKey, label: 'minutely', retentionMs: TimeMs.DAY, timestampMs: bucketTimestampMs('minutely') },
{ key: hourlyKey, label: 'hourly', retentionMs: TimeMs.WEEK, timestampMs: bucketTimestampMs('hourly') },
{ key: dailyKey, label: 'daily', retentionMs: DAILY_METRICS_RETENTION_DAYS * TimeMs.DAY, timestampMs: bucketTimestampMs('daily') },
];

const operations = series.map(({ key, label, retentionMs }) => ({
label,
promise: this.redis.safeTsAdd(key, 1, labels, retentionMs),
}));

const results = await Promise.allSettled(operations.map((op) => op.promise));

results.forEach((result, index) => {
if (result.status === 'rejected') {
const { label } = operations[index];
this.logger.error(`Failed to add ${label} TS for ${metricType}`, result.reason);
for (const { key, label, retentionMs, timestampMs } of series) {
try {
await this.redis.safeTsAdd(key, 1, labels, retentionMs, timestampMs);
} catch (error) {
this.logger.error(`Failed to add ${label} TS for ${metricType}`, error);
}
});
}
}

/**
Expand Down
6 changes: 4 additions & 2 deletions workers/grouper/src/redisHelper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -237,14 +237,16 @@ export default class RedisHelper {
* @param value - value to add
* @param labels - labels to attach to the time series
* @param retentionMs - optional retention in milliseconds
* @param timestampMs - timestamp in milliseconds; defaults to current time
*/
public async safeTsAdd(
key: string,
value: number,
labels: Record<string, string>,
retentionMs = 0
retentionMs = 0,
timestampMs = 0
): Promise<void> {
const timestamp = Date.now();
const timestamp = timestampMs === 0 ? Date.now() : timestampMs;

/**
* Create key if not exists — then call increment
Expand Down
17 changes: 17 additions & 0 deletions workers/grouper/src/utils/bucketTimestamp.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import TimeMs from '../../../../lib/utils/time';

/**
* Returns the current time truncated to the start of the given granularity
* bucket in milliseconds (UTC). All events within the same bucket share one
* timestamp so ON_DUPLICATE SUM accumulates them into a single sample.
*
* @param granularity - time granularity level
* @param now - current timestamp in ms, defaults to Date.now()
*/
export function bucketTimestampMs(granularity: 'minutely' | 'hourly' | 'daily', now = Date.now()): number {
switch (granularity) {
case 'hourly': return now - (now % TimeMs.HOUR);
case 'daily': return now - (now % TimeMs.DAY);
default: return now - (now % TimeMs.MINUTE); // minutely
}
}
65 changes: 65 additions & 0 deletions workers/grouper/tests/bucketTimestamp.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import '../../../env-test';
import { bucketTimestampMs } from '../src/utils/bucketTimestamp';

describe('bucketTimestampMs', () => {
/**
* 2026-04-14T15:37:42.500Z
* minute start: 2026-04-14T15:37:00.000Z
* hour start: 2026-04-14T15:00:00.000Z
* day start: 2026-04-14T00:00:00.000Z
*/
const now = new Date('2026-04-14T15:37:42.500Z').getTime();

it('truncates to the start of the current minute', () => {
const expected = new Date('2026-04-14T15:37:00.000Z').getTime();

expect(bucketTimestampMs('minutely', now)).toBe(expected);
});

it('truncates to the start of the current hour', () => {
const expected = new Date('2026-04-14T15:00:00.000Z').getTime();

expect(bucketTimestampMs('hourly', now)).toBe(expected);
});

it('truncates to the start of the current day (UTC midnight)', () => {
const expected = new Date('2026-04-14T00:00:00.000Z').getTime();

expect(bucketTimestampMs('daily', now)).toBe(expected);
});

it('returns the same value for two calls within the same minute', () => {
const t1 = new Date('2026-04-14T15:37:00.000Z').getTime();
const t2 = new Date('2026-04-14T15:37:59.999Z').getTime();

expect(bucketTimestampMs('minutely', t1)).toBe(bucketTimestampMs('minutely', t2));
});

it('returns different values for two calls in different minutes', () => {
const t1 = new Date('2026-04-14T15:37:59.999Z').getTime();
const t2 = new Date('2026-04-14T15:38:00.000Z').getTime();

expect(bucketTimestampMs('minutely', t1)).not.toBe(bucketTimestampMs('minutely', t2));
});

it('returns the same value for two calls within the same hour', () => {
const t1 = new Date('2026-04-14T15:00:00.000Z').getTime();
const t2 = new Date('2026-04-14T15:59:59.999Z').getTime();

expect(bucketTimestampMs('hourly', t1)).toBe(bucketTimestampMs('hourly', t2));
});

it('returns the same value for two calls within the same day', () => {
const t1 = new Date('2026-04-14T00:00:00.000Z').getTime();
const t2 = new Date('2026-04-14T23:59:59.999Z').getTime();

expect(bucketTimestampMs('daily', t1)).toBe(bucketTimestampMs('daily', t2));
});

it('returns different values for two calls on different days', () => {
const t1 = new Date('2026-04-14T23:59:59.999Z').getTime();
const t2 = new Date('2026-04-15T00:00:00.000Z').getTime();

expect(bucketTimestampMs('daily', t1)).not.toBe(bucketTimestampMs('daily', t2));
});
});
9 changes: 6 additions & 3 deletions workers/grouper/tests/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -763,21 +763,24 @@ describe('GrouperWorker', () => {
`ts:project-events-accepted:${projectIdMock}:minutely`,
1,
expectedLabels,
TimeMs.DAY
TimeMs.DAY,
expect.any(Number),
);
expect(safeTsAddSpy).toHaveBeenNthCalledWith(
2,
`ts:project-events-accepted:${projectIdMock}:hourly`,
1,
expectedLabels,
TimeMs.WEEK
TimeMs.WEEK,
expect.any(Number),
);
expect(safeTsAddSpy).toHaveBeenNthCalledWith(
3,
`ts:project-events-accepted:${projectIdMock}:daily`,
1,
expectedLabels,
90 * TimeMs.DAY
90 * TimeMs.DAY,
expect.any(Number),
);
} finally {
safeTsAddSpy.mockRestore();
Expand Down
Loading