diff --git a/apps/start/src/routes/_app.$organizationId.$projectId.reports_.$reportId.tsx b/apps/start/src/routes/_app.$organizationId.$projectId.reports_.$reportId.tsx
index 714ec89f1..ae258b296 100644
--- a/apps/start/src/routes/_app.$organizationId.$projectId.reports_.$reportId.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId.reports_.$reportId.tsx
@@ -28,6 +28,7 @@ export const Route = createFileRoute(
},
validateSearch: z.object({
dashboardId: z.string().optional(),
+ range: z.string().optional(),
}),
pendingComponent: FullPageLoadingState,
});
diff --git a/apps/start/src/utils/csv-download.ts b/apps/start/src/utils/csv-download.ts
new file mode 100644
index 000000000..39b3efbd7
--- /dev/null
+++ b/apps/start/src/utils/csv-download.ts
@@ -0,0 +1,138 @@
+import type { RouterOutputs } from '@/trpc/client';
+
+type IChartData = RouterOutputs['chart']['chart'];
+type FunnelData = RouterOutputs['chart']['funnel'];
+type CohortData = RouterOutputs['chart']['cohort'];
+type ConversionData = RouterOutputs['chart']['conversion'];
+
+function escapeCsvValue(value: string | number | null | undefined): string {
+ if (value === null || value === undefined) return '';
+ const str = String(value);
+ if (str.includes(',') || str.includes('"') || str.includes('\n')) {
+ return `"${str.replace(/"/g, '""')}"`;
+ }
+ return str;
+}
+
+function buildCSV(rows: (string | number | null | undefined)[][]): string {
+ return rows.map((row) => row.map(escapeCsvValue).join(',')).join('\n');
+}
+
+export function downloadCSV(content: string, filename: string): void {
+ const blob = new Blob([content], { type: 'text/csv;charset=utf-8;' });
+ const url = URL.createObjectURL(blob);
+ const link = document.createElement('a');
+ link.href = url;
+ link.download = filename;
+ document.body.appendChild(link);
+ link.click();
+ document.body.removeChild(link);
+ URL.revokeObjectURL(url);
+}
+
+export function chartDataToCSV(data: IChartData): string {
+ if (!data.series.length) return '';
+
+ const allDates = [
+ ...new Set(data.series.flatMap((s) => s.data.map((d) => d.date))),
+ ].sort();
+
+ // Rows = series, columns = dates
+ const rows = data.series.map((serie) => {
+ const label = serie.names.join(' / ') || serie.event.name;
+ const values = allDates.map((date) => {
+ const point = serie.data.find((d) => d.date === date);
+ return point?.count ?? '';
+ });
+ return [label, ...values];
+ });
+
+ return buildCSV([['Series', ...allDates], ...rows]);
+}
+
+export function funnelDataToCSV(data: FunnelData): string {
+ if (!data.current.length) return '';
+
+ const hasBreakdowns = data.current.length > 1;
+
+ if (hasBreakdowns) {
+ // Columns = steps, rows = breakdown × metric
+ const steps = data.current[0]!.steps.map((s) => s.event.displayName);
+ const header = ['Breakdown / Metric', ...steps];
+ const rows: (string | number | null | undefined)[][] = [];
+
+ for (const variant of data.current) {
+ const label = variant.breakdowns.join(' / ') || '(all)';
+ rows.push([`${label} (Count)`, ...variant.steps.map((s) => s.count)]);
+ rows.push([`${label} (Conversion %)`, ...variant.steps.map((s) => s.percent)]);
+ rows.push([`${label} (Dropped After)`, ...variant.steps.map((s) => s.dropoffCount)]);
+ rows.push([`${label} (Dropoff %)`, ...variant.steps.map((s) => s.dropoffPercent)]);
+ }
+
+ return buildCSV([header, ...rows]);
+ }
+
+ // Columns = steps, rows = metrics
+ const funnel = data.current[0]!;
+ const steps = funnel.steps.map((s) => s.event.displayName);
+ const header = ['Metric', ...steps];
+ const rows = [
+ ['Count', ...funnel.steps.map((s) => s.count)],
+ ['Conversion %', ...funnel.steps.map((s) => s.percent)],
+ ['Dropped After', ...funnel.steps.map((s) => s.dropoffCount)],
+ ['Dropoff %', ...funnel.steps.map((s) => s.dropoffPercent)],
+ ];
+
+ return buildCSV([header, ...rows]);
+}
+
+export function cohortDataToCSV(data: CohortData): string {
+ if (!data.length) return '';
+
+ const cohortDates = data.map((row) => row.cohort_interval);
+ const maxPeriods = data[0]?.values.length ?? 0;
+
+ // Rows = periods, columns = cohort dates
+ const totalRow: (string | number | null | undefined)[] = [
+ 'Total Profiles',
+ ...data.map((row) => row.sum),
+ ];
+
+ const periodRows = Array.from({ length: maxPeriods }, (_, i) => {
+ const label = i === 0 ? 'Period <1' : `Period ${i}`;
+ return [label, ...data.map((row) => row.values[i] ?? '')];
+ });
+
+ return buildCSV([['Cohort Date', ...cohortDates], totalRow, ...periodRows]);
+}
+
+export function cohortMembersToCSV(profileIds: string[]): string {
+ if (!profileIds.length) return '';
+ return buildCSV([['profile_id'], ...profileIds.map((id) => [id])]);
+}
+
+export function conversionDataToCSV(data: ConversionData): string {
+ if (!data.current.length) return '';
+
+ const allDates = data.current[0]?.data.map((d) => d.date) ?? [];
+
+ // Rows = series × metric, columns = dates
+ const rows = data.current.flatMap((serie) => {
+ const label = serie.breakdowns.join(' / ') || serie.id;
+ const rateRow = [
+ `${label} (Rate %)`,
+ ...allDates.map((date) => serie.data.find((d) => d.date === date)?.rate ?? ''),
+ ];
+ const convRow = [
+ `${label} (Conversions)`,
+ ...allDates.map((date) => serie.data.find((d) => d.date === date)?.conversions ?? ''),
+ ];
+ const totalRow = [
+ `${label} (Total)`,
+ ...allDates.map((date) => serie.data.find((d) => d.date === date)?.total ?? ''),
+ ];
+ return [rateRow, convRow, totalRow];
+ });
+
+ return buildCSV([['Series', ...allDates], ...rows]);
+}
diff --git a/apps/start/src/utils/title.ts b/apps/start/src/utils/title.ts
index e735636e2..1974618d6 100644
--- a/apps/start/src/utils/title.ts
+++ b/apps/start/src/utils/title.ts
@@ -94,6 +94,7 @@ export const PAGE_TITLES = {
PROFILES: 'Profiles',
PROFILE_EVENTS: 'Profile events',
PROFILE_DETAILS: 'Profile details',
+ COHORTS: 'Cohorts',
// Sub-sections
CONVERSIONS: 'Conversions',
diff --git a/apps/start/vite.config.ts b/apps/start/vite.config.ts
index 3b0bc073a..ef94a3d14 100644
--- a/apps/start/vite.config.ts
+++ b/apps/start/vite.config.ts
@@ -21,6 +21,7 @@ if (process.env.NITRO) {
nitroV2Plugin({
preset: 'node-server',
compatibilityDate: '2025-10-21',
+ serveStatic: true,
}),
);
} else {
diff --git a/apps/worker/package.json b/apps/worker/package.json
index c8ae019e3..ab4c23550 100644
--- a/apps/worker/package.json
+++ b/apps/worker/package.json
@@ -14,6 +14,7 @@
"@bull-board/api": "6.14.0",
"@bull-board/express": "6.14.0",
"@openpanel/common": "workspace:*",
+ "@openpanel/constants": "workspace:*",
"@openpanel/db": "workspace:*",
"@openpanel/email": "workspace:*",
"@openpanel/integrations": "workspace:^",
diff --git a/apps/worker/src/boot-cron.ts b/apps/worker/src/boot-cron.ts
index 9650598e8..d1ce19aac 100644
--- a/apps/worker/src/boot-cron.ts
+++ b/apps/worker/src/boot-cron.ts
@@ -22,7 +22,7 @@ export async function bootCron() {
{
name: 'flush',
type: 'flushEvents',
- pattern: 1000 * 10,
+ pattern: 1000 * 5,
},
{
name: 'flush',
@@ -34,6 +34,11 @@ export async function bootCron() {
type: 'flushSessions',
pattern: 1000 * 10,
},
+ // {
+ // name: 'customAlerts',
+ // type: 'customAlerts',
+ // pattern: '*/15 * * * *',
+ // },
];
if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') {
diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.ts
index 55652de8d..42896e240 100644
--- a/apps/worker/src/boot-workers.ts
+++ b/apps/worker/src/boot-workers.ts
@@ -64,6 +64,76 @@ function getEnabledQueues(): QueueName[] {
return queues;
}
+function getPodIndex(): number {
+ const hostname = process.env.HOSTNAME || '';
+
+ // Extract numeric index from StatefulSet pod name (e.g., openpanel-worker-0 -> 0)
+ const match = hostname.match(/-(\d+)$/);
+ if (match) {
+ return Number.parseInt(match[1], 10);
+ }
+
+ // Fallback to 0 if no numeric suffix found
+ logger.warn('Could not extract pod index from hostname, defaulting to 0', {
+ hostname,
+ });
+ return 0;
+}
+
+function getAutoPartitionedQueues(): QueueName[] {
+ const enabledQueues = getEnabledQueues();
+
+ const enableShardDistribution =
+ process.env.ENABLE_SHARD_DISTRIBUTION === 'true';
+
+ if (!enableShardDistribution) {
+ return enabledQueues;
+ }
+
+ if (!enabledQueues.includes('events')) {
+ return enabledQueues;
+ }
+
+ const totalPods = Number.parseInt(process.env.TOTAL_POD || '1', 10);
+
+ if (totalPods <= 1) {
+ logger.info('Shard distribution disabled: TOTAL_POD not set or = 1', {
+ totalShards: EVENTS_GROUP_QUEUES_SHARDS,
+ });
+ return enabledQueues;
+ }
+
+ const podIndex = getPodIndex();
+ const shardsPerPod = Math.floor(EVENTS_GROUP_QUEUES_SHARDS / totalPods);
+ const remainderShards = EVENTS_GROUP_QUEUES_SHARDS % totalPods;
+ const extraShard = podIndex < remainderShards ? 1 : 0;
+ const startShard =
+ podIndex * shardsPerPod + Math.min(podIndex, remainderShards);
+ const endShard = startShard + shardsPerPod + extraShard;
+
+ const specificShards = Array.from(
+ { length: endShard - startShard },
+ (_, i) => `events_${startShard + i}`,
+ );
+
+ const partitionedQueues = [
+ ...enabledQueues.filter((q) => q !== 'events'),
+ ...specificShards,
+ ];
+
+ logger.info('Shard distribution enabled', {
+ hostname: process.env.HOSTNAME,
+ podIndex,
+ totalPods,
+ totalShards: EVENTS_GROUP_QUEUES_SHARDS,
+ assignedShards: `${startShard}-${endShard - 1}`,
+ shardsCount: endShard - startShard,
+ queues: partitionedQueues,
+ });
+
+ return partitionedQueues;
+}
+
/**
* Gets the concurrency setting for a queue from environment variables.
* Env var format: {QUEUE_NAME}_CONCURRENCY (e.g., EVENTS_0_CONCURRENCY=32)
@@ -83,7 +153,11 @@ function getConcurrencyFor(queueName: string, defaultValue = 1): number {
}
export async function bootWorkers() {
- const enabledQueues = getEnabledQueues();
+ const enableShardDistribution =
+ process.env.ENABLE_SHARD_DISTRIBUTION === 'true';
+ const enabledQueues = enableShardDistribution
+ ? getAutoPartitionedQueues()
+ : getEnabledQueues();
const workers: (Worker | GroupWorker
)[] = [];
@@ -182,9 +256,15 @@ export async function bootWorkers() {
const importWorker = new Worker(importQueue.name, importJob, {
...workerOptions,
concurrency,
+ lockDuration: 300000, // 5 minutes - for testing to see actual errors faster
+ stalledInterval: 150000, // 2.5 minutes - check for stalled jobs
});
workers.push(importWorker);
- logger.info('Started worker for import', { concurrency });
+ logger.info('Started worker for import', {
+ concurrency,
+ lockDuration: 300000,
+ stalledInterval: 150000,
+ });
}
if (workers.length === 0) {
diff --git a/apps/worker/src/jobs/cron.custom-alerts.ts b/apps/worker/src/jobs/cron.custom-alerts.ts
new file mode 100644
index 000000000..c06a788fd
--- /dev/null
+++ b/apps/worker/src/jobs/cron.custom-alerts.ts
@@ -0,0 +1,503 @@
+import {
+ type AlertFrequency,
+ ALERT_FREQUENCY_MS,
+ ALERT_FREQUENCY_TO_CURRENT_RANGE,
+ ALERT_FREQUENCY_TO_INTERVAL,
+ ALERT_FREQUENCY_TO_RANGE,
+ ANOMALY_HISTORY_COUNT,
+ CONFIDENCE_Z_SCORES,
+} from '@openpanel/constants';
+import {
+ ChartEngine,
+ conversionService,
+ createNotification,
+ db,
+ funnelService,
+ getDatesFromRange,
+ getReportById,
+ getSettingsForProject,
+} from '@openpanel/db';
+
+import { logger } from '../utils/logger';
+
+// Inline types — @openpanel/validation is not a direct worker dependency
+type INotificationRuleThresholdConfig = {
+ type: 'threshold';
+ reportId: string;
+ operator: 'above' | 'below';
+ value: number;
+ frequency: AlertFrequency;
+};
+
+type INotificationRuleAnomalyConfig = {
+ type: 'anomaly';
+ reportId: string;
+ confidence: '95' | '98' | '99';
+ frequency: AlertFrequency;
+};
+
+/**
+ * Main custom alerts cron job
+ * Runs every 15 minutes, evaluates all threshold and anomaly rules
+ */
+export async function customAlerts() {
+ const rules = await db.notificationRule.findMany({
+ where: {
+ config: {
+ path: ['type'],
+ string_contains: '', // We'll filter in code since Prisma JSON filtering is limited
+ },
+ },
+ include: {
+ integrations: true,
+ project: {
+ select: {
+ name: true,
+ organizationId: true,
+ },
+ },
+ },
+ });
+
+ // Filter to only threshold and anomaly rules
+ const alertRules = rules.filter((rule) => {
+ const config = rule.config as { type?: string };
+ return config.type === 'threshold' || config.type === 'anomaly';
+ });
+
+ if (alertRules.length === 0) {
+ logger.debug('No custom alert rules found, skipping');
+ return;
+ }
+
+ logger.info(`[custom-alerts] Starting evaluation of ${alertRules.length} rules`);
+
+ let evaluated = 0;
+ let skippedFrequency = 0;
+ let skippedNotCrossed = 0;
+ let triggered = 0;
+ let errored = 0;
+
+ for (const rule of alertRules) {
+ try {
+ const config = rule.config as
+ | INotificationRuleThresholdConfig
+ | INotificationRuleAnomalyConfig;
+
+ // Frequency check: skip if not enough time has elapsed
+ if (rule.lastNotifiedAt) {
+ const elapsed = Date.now() - rule.lastNotifiedAt.getTime();
+ if (elapsed < ALERT_FREQUENCY_MS[config.frequency]) {
+ skippedFrequency++;
+ logger.debug(
+ `[custom-alerts] Rule "${rule.name}" (${rule.id}): skipped — frequency limit not elapsed (${Math.round(elapsed / 1000 / 60)}min / ${config.frequency})`,
+ );
+ continue;
+ }
+ }
+
+ // Fetch the report this alert is tied to
+ const report = await getReportById(config.reportId);
+ if (!report) {
+ logger.warn(
+ `[custom-alerts] Rule "${rule.name}" (${rule.id}): report ${config.reportId} not found — skipping`,
+ );
+ errored++;
+ continue;
+ }
+
+ evaluated++;
+ let shouldAlert = false;
+ let title = '';
+ let message = '';
+
+ if (config.type === 'threshold') {
+ const result = await evaluateThreshold(report, config);
+ shouldAlert = result.shouldAlert;
+ title = result.title;
+ message = result.message;
+
+ logger.info(
+ `[custom-alerts] Rule "${rule.name}" (${rule.id}): threshold ${config.operator} ${config.value} — current: ${result.currentValue} — ${shouldAlert ? 'TRIGGERED' : 'not crossed'}`,
+ );
+ } else if (config.type === 'anomaly') {
+ const result = await evaluateAnomaly(report, config);
+ shouldAlert = result.shouldAlert;
+ title = result.title;
+ message = result.message;
+
+ logger.info(
+ `[custom-alerts] Rule "${rule.name}" (${rule.id}): anomaly ${config.confidence}% — current: ${result.currentValue} — band: [${result.lowerBound}, ${result.upperBound}] — ${shouldAlert ? 'TRIGGERED' : 'within range'}`,
+ );
+ }
+
+ if (!shouldAlert) {
+ skippedNotCrossed++;
+ continue;
+ }
+
+ triggered++;
+
+ // Build dashboard link scoped to the alert's evaluation window
+ const dashboardUrl =
+ process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL;
+ const organizationId = rule.project?.organizationId;
+ const alertRange = ALERT_FREQUENCY_TO_CURRENT_RANGE[config.frequency as AlertFrequency];
+ const reportLink =
+ dashboardUrl && organizationId
+ ? `${dashboardUrl}/${organizationId}/${rule.projectId}/reports/${config.reportId}?range=${alertRange}`
+ : '';
+
+ // Build rich notification message
+ const projectName = rule.project?.name || '';
+ const now = new Date();
+ const timeStr = now.toLocaleDateString('en-US', {
+ weekday: 'short',
+ year: 'numeric',
+ month: 'short',
+ day: '2-digit',
+ hour: '2-digit',
+ minute: '2-digit',
+ timeZoneName: 'short',
+ });
+
+ const lines = [message];
+ if (reportLink) {
+ lines.push(`Report: ${reportLink}`);
+ }
+ lines.push(`Time: ${timeStr}`);
+ if (projectName) {
+ lines.push(`Project: ${projectName}`);
+ }
+ const fullMessage = lines.join('\n');
+
+ const integrationCount = rule.integrations.length + (rule.sendToApp ? 1 : 0);
+ logger.info(
+ `[custom-alerts] Rule "${rule.name}" (${rule.id}): sending ${integrationCount} notification(s)`,
+ );
+
+ const promises = rule.integrations.map((integration) =>
+ createNotification({
+ title,
+ message: fullMessage,
+ projectId: rule.projectId,
+ integrationId: integration.id,
+ notificationRuleId: rule.id,
+ payload: null,
+ }),
+ );
+
+ if (rule.sendToApp) {
+ promises.push(
+ createNotification({
+ title,
+ message: fullMessage,
+ projectId: rule.projectId,
+ integrationId: 'app',
+ notificationRuleId: rule.id,
+ payload: null,
+ }),
+ );
+ }
+
+ await Promise.all(promises);
+
+ // Update lastNotifiedAt
+ await db.notificationRule.update({
+ where: { id: rule.id },
+ data: { lastNotifiedAt: new Date() },
+ });
+
+ logger.info(
+ `[custom-alerts] Rule "${rule.name}" (${rule.id}): lastNotifiedAt updated`,
+ );
+ } catch (error) {
+ errored++;
+ logger.error(
+ `[custom-alerts] Rule "${rule.name}" (${rule.id}): error — ${error}`,
+ );
+ }
+ }
+
+ logger.info(
+ `[custom-alerts] Done — ${alertRules.length} rules: ${evaluated} evaluated, ${triggered} triggered, ${skippedFrequency} skipped (frequency), ${skippedNotCrossed} skipped (not crossed), ${errored} errored`,
+ );
+}
+
+/**
+ * Evaluate a threshold alert rule
+ * Runs the report query for the current frequency window and compares against the fixed threshold
+ */
+async function evaluateThreshold(
+ report: NonNullable>>,
+ config: INotificationRuleThresholdConfig,
+) {
+ const freq = config.frequency;
+ const interval = ALERT_FREQUENCY_TO_INTERVAL[freq];
+
+ // Conversion charts: use conversionService to get the actual conversion rate (%)
+ if (report.chartType === 'conversion') {
+ const { timezone } = await getSettingsForProject(report.projectId);
+ const { startDate, endDate } = getDatesFromRange(
+ ALERT_FREQUENCY_TO_CURRENT_RANGE[freq] as any,
+ timezone,
+ );
+
+ const series = await conversionService.getConversion({
+ projectId: report.projectId,
+ startDate,
+ endDate,
+ series: report.series,
+ breakdowns: report.breakdowns,
+ interval: interval as any,
+ timezone,
+ funnelGroup: report.funnelGroup,
+ funnelWindow: report.funnelWindow,
+ globalFilters: report.globalFilters,
+ holdProperties: report.holdProperties,
+ measuring: 'conversion_rate',
+ limit: report.limit,
+ cohortFilters: report.cohortFilters,
+ });
+
+ const data = series[0]?.data ?? [];
+ const lastPoint = data[data.length - 1];
+ const currentValue = lastPoint?.rate ?? 0;
+
+ const crossed =
+ config.operator === 'above'
+ ? currentValue > config.value
+ : currentValue < config.value;
+
+ return {
+ shouldAlert: crossed,
+ currentValue,
+ title: `Alert: ${report.name}`,
+ message: `The current value for ${report.name} is ${currentValue.toFixed(2)}%. Triggered when the current value is ${config.operator} ${config.value}%.`,
+ };
+ }
+
+ // Funnel charts: use funnelService to get the overall conversion % from the last step
+ if (report.chartType === 'funnel') {
+ const { timezone } = await getSettingsForProject(report.projectId);
+ const { startDate, endDate } = getDatesFromRange(
+ ALERT_FREQUENCY_TO_CURRENT_RANGE[freq] as any,
+ timezone,
+ );
+
+ const funnelResult = await funnelService.getFunnel({
+ projectId: report.projectId,
+ startDate,
+ endDate,
+ series: report.series,
+ breakdowns: report.breakdowns,
+ interval: interval as any,
+ range: report.range,
+ chartType: report.chartType,
+ metric: report.metric as any,
+ previous: false,
+ timezone,
+ funnelGroup: report.funnelGroup,
+ funnelWindow: report.funnelWindow,
+ globalFilters: report.globalFilters,
+ holdProperties: report.holdProperties,
+ measuring: 'conversion_rate',
+ limit: report.limit,
+ cohortFilters: report.cohortFilters,
+ });
+
+ const currentValue = funnelResult[0]?.lastStep?.percent ?? 0;
+
+ const crossed =
+ config.operator === 'above'
+ ? currentValue > config.value
+ : currentValue < config.value;
+
+ return {
+ shouldAlert: crossed,
+ currentValue,
+ title: `Alert: ${report.name}`,
+ message: `The current value for ${report.name} is ${currentValue.toFixed(2)}%. Triggered when the current value is ${config.operator} ${config.value}%.`,
+ };
+ }
+
+ // All other chart types: use ChartEngine with raw metric values
+ const range = ALERT_FREQUENCY_TO_CURRENT_RANGE[freq];
+
+ const result = await ChartEngine.execute({
+ projectId: report.projectId,
+ series: report.series,
+ breakdowns: report.breakdowns,
+ chartType: report.chartType,
+ interval: interval as any,
+ range: range as any,
+ previous: false,
+ formula: report.formula,
+ metric: report.metric as any,
+ globalFilters: report.globalFilters,
+ holdProperties: report.holdProperties,
+ measuring: 'conversion_rate',
+ limit: report.limit,
+ cohortFilters: report.cohortFilters,
+ });
+
+ const currentValue = result.series[0]?.metrics?.sum ?? 0;
+ const metricKey = (report.metric as string) || 'sum';
+ const metricValue =
+ (result.series[0]?.metrics as unknown as Record)?.[metricKey] ??
+ currentValue;
+
+ const crossed =
+ config.operator === 'above'
+ ? metricValue > config.value
+ : metricValue < config.value;
+
+ return {
+ shouldAlert: crossed,
+ currentValue: metricValue,
+ title: `Alert: ${report.name}`,
+ message: `The current value for ${report.name} is ${metricValue.toFixed(2)}. Triggered when the current value is ${config.operator} ${config.value}.`,
+ };
+}
+
+/**
+ * Evaluate an anomaly detection alert rule
+ * Runs the report query for historical periods, computes confidence band, checks if current value is outside
+ */
+async function evaluateAnomaly(
+ report: NonNullable>>,
+ config: INotificationRuleAnomalyConfig,
+) {
+ const freq = config.frequency;
+ const historicalRange = ALERT_FREQUENCY_TO_RANGE[freq];
+ const interval = ALERT_FREQUENCY_TO_INTERVAL[freq];
+
+ // Conversion charts: use conversionService and compare on rate (%) not raw counts
+ if (report.chartType === 'conversion') {
+ const { timezone } = await getSettingsForProject(report.projectId);
+ const { startDate, endDate } = getDatesFromRange(historicalRange as any, timezone);
+
+ const seriesResult = await conversionService.getConversion({
+ projectId: report.projectId,
+ startDate,
+ endDate,
+ series: report.series,
+ breakdowns: report.breakdowns,
+ interval: interval as any,
+ timezone,
+ funnelGroup: report.funnelGroup,
+ funnelWindow: report.funnelWindow,
+ globalFilters: report.globalFilters,
+ holdProperties: report.holdProperties,
+ measuring: 'conversion_rate',
+ limit: report.limit,
+ cohortFilters: report.cohortFilters,
+ });
+
+ const data = seriesResult[0]?.data ?? [];
+ if (data.length < 3) {
+ logger.warn(
+ `Not enough historical data for anomaly detection on report ${report.id}`,
+ );
+ return { shouldAlert: false, currentValue: 0, lowerBound: 0, upperBound: 0, title: '', message: '' };
+ }
+
+ // dataPoints are conversion rates (percentages, e.g. 88.63)
+ const dataPoints = data.map((d) => d.rate);
+ const historicalValues = dataPoints.slice(
+ 0,
+ Math.min(dataPoints.length - 1, ANOMALY_HISTORY_COUNT),
+ );
+ const currentValue = dataPoints[dataPoints.length - 1] ?? 0;
+
+ if (historicalValues.length < 3) {
+ return { shouldAlert: false, currentValue: 0, lowerBound: 0, upperBound: 0, title: '', message: '' };
+ }
+
+ const mean = historicalValues.reduce((sum, v) => sum + v, 0) / historicalValues.length;
+ const variance =
+ historicalValues.reduce((sum, v) => sum + (v - mean) ** 2, 0) /
+ historicalValues.length;
+ const stddev = Math.sqrt(variance);
+
+ const zScore = CONFIDENCE_Z_SCORES[config.confidence] ?? 1.96;
+ const lowerBound = mean - zScore * stddev;
+ const upperBound = mean + zScore * stddev;
+ const isAnomaly = currentValue < lowerBound || currentValue > upperBound;
+
+ return {
+ shouldAlert: isAnomaly,
+ currentValue,
+ lowerBound,
+ upperBound,
+ title: `Alert: ${report.name}`,
+ message: `The current value for ${report.name} is ${currentValue.toFixed(2)}%. Triggered when the current value is not within forecasted range [${lowerBound.toFixed(2)}%, ${upperBound.toFixed(2)}%].`,
+ };
+ }
+
+ // Funnel charts: no time-series data available — anomaly detection not supported
+ if (report.chartType === 'funnel') {
+ logger.warn(
+ `[custom-alerts] Anomaly detection is not supported for funnel charts (report ${report.id}) — skipping`,
+ );
+ return { shouldAlert: false, currentValue: 0, lowerBound: 0, upperBound: 0, title: '', message: '' };
+ }
+
+ // All other chart types: use ChartEngine with raw counts
+ const result = await ChartEngine.execute({
+ projectId: report.projectId,
+ series: report.series,
+ breakdowns: report.breakdowns,
+ chartType: report.chartType,
+ interval: interval as any,
+ range: historicalRange as any,
+ previous: false,
+ formula: report.formula,
+ metric: report.metric as any,
+ globalFilters: report.globalFilters,
+ holdProperties: report.holdProperties,
+ measuring: 'conversion_rate',
+ limit: report.limit,
+ cohortFilters: report.cohortFilters,
+ });
+
+ const series = result.series[0];
+ if (!series || !series.data || series.data.length < 3) {
+ logger.warn(
+ `Not enough historical data for anomaly detection on report ${report.id}`,
+ );
+ return { shouldAlert: false, currentValue: 0, lowerBound: 0, upperBound: 0, title: '', message: '' };
+ }
+
+ const dataPoints = series.data.map((d) => d.count);
+ const historicalValues = dataPoints.slice(
+ 0,
+ Math.min(dataPoints.length - 1, ANOMALY_HISTORY_COUNT),
+ );
+ const currentValue = dataPoints[dataPoints.length - 1] ?? 0;
+
+ if (historicalValues.length < 3) {
+ return { shouldAlert: false, currentValue: 0, lowerBound: 0, upperBound: 0, title: '', message: '' };
+ }
+
+ const mean =
+ historicalValues.reduce((sum, v) => sum + v, 0) / historicalValues.length;
+ const variance =
+ historicalValues.reduce((sum, v) => sum + (v - mean) ** 2, 0) /
+ historicalValues.length;
+ const stddev = Math.sqrt(variance);
+
+ const zScore = CONFIDENCE_Z_SCORES[config.confidence] ?? 1.96;
+ const lowerBound = mean - zScore * stddev;
+ const upperBound = mean + zScore * stddev;
+ const isAnomaly = currentValue < lowerBound || currentValue > upperBound;
+
+ return {
+ shouldAlert: isAnomaly,
+ currentValue,
+ lowerBound,
+ upperBound,
+ title: `Alert: ${report.name}`,
+ message: `The current value for ${report.name} is ${currentValue.toFixed(2)}. Triggered when the current value is not within forecasted range [${lowerBound.toFixed(2)}, ${upperBound.toFixed(2)}].`,
+ };
+}
diff --git a/apps/worker/src/jobs/cron.materialize-columns.ts b/apps/worker/src/jobs/cron.materialize-columns.ts
new file mode 100644
index 000000000..4e83fa3e6
--- /dev/null
+++ b/apps/worker/src/jobs/cron.materialize-columns.ts
@@ -0,0 +1,36 @@
+import { materializeColumnsService } from '@openpanel/db';
+import { logger } from '@/utils/logger';
+
+export async function materializeColumns(options: {
+ dryRun?: boolean;
+ threshold?: number;
+}) {
+ const { dryRun = false, threshold = 150 } = options;
+
+ logger.info('Starting materialized columns cron job', { dryRun, threshold });
+
+ try {
+ const result = await materializeColumnsService.analyze({
+ dryRun,
+ threshold,
+ });
+
+ logger.info('Materialized columns analysis complete', {
+ candidatesFound: result.candidates.length,
+ materialized: result.materialized,
+ dryRun,
+ });
+
+ // Log the report for visibility
+ console.log(result.report);
+
+ return {
+ success: true,
+ candidates: result.candidates.length,
+ materialized: result.materialized.length,
+ };
+ } catch (error) {
+ logger.error('Materialized columns cron job failed', { error });
+ throw error;
+ }
+}
diff --git a/apps/worker/src/jobs/cron.ts b/apps/worker/src/jobs/cron.ts
index b50e3beb5..6b0f371da 100644
--- a/apps/worker/src/jobs/cron.ts
+++ b/apps/worker/src/jobs/cron.ts
@@ -3,7 +3,9 @@ import type { Job } from 'bullmq';
import { eventBuffer, profileBuffer, sessionBuffer } from '@openpanel/db';
import type { CronQueuePayload } from '@openpanel/queue';
+import { customAlerts } from './cron.custom-alerts';
import { jobdeleteProjects } from './cron.delete-projects';
+import { materializeColumns } from './cron.materialize-columns';
import { ping } from './cron.ping';
import { salt } from './cron.salt';
@@ -27,5 +29,14 @@ export async function cronJob(job: Job) {
case 'deleteProjects': {
return await jobdeleteProjects(job);
}
+ case 'materializeColumns': {
+ return await materializeColumns({
+ dryRun: job.data.dryRun ?? false,
+ threshold: job.data.threshold ?? 150,
+ });
+ }
+ case 'customAlerts': {
+ return await customAlerts();
+ }
}
}
diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.ts
index 4965acac9..35e4954fe 100644
--- a/apps/worker/src/jobs/events.incoming-event.ts
+++ b/apps/worker/src/jobs/events.incoming-event.ts
@@ -11,11 +11,7 @@ import {
parseUserAgent,
} from '@openpanel/common/server';
import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db';
-import {
- checkNotificationRulesForEvent,
- createEvent,
- sessionBuffer,
-} from '@openpanel/db';
+import { createEvent, sessionBuffer } from '@openpanel/db';
import type { ILogger } from '@openpanel/logger';
import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
import * as R from 'ramda';
@@ -34,11 +30,7 @@ async function createEventAndNotify(
logger: ILogger,
) {
logger.info('Creating event', { event: payload });
- const [event] = await Promise.all([
- createEvent(payload),
- checkNotificationRulesForEvent(payload).catch(() => {}),
- ]);
- return event;
+ return createEvent(payload);
}
const parseRevenue = (revenue: unknown): number | undefined => {
diff --git a/apps/worker/src/jobs/events.incoming-events.test.ts b/apps/worker/src/jobs/events.incoming-events.test.ts
index 1ee1ebb90..838dcd30f 100644
--- a/apps/worker/src/jobs/events.incoming-events.test.ts
+++ b/apps/worker/src/jobs/events.incoming-events.test.ts
@@ -21,7 +21,6 @@ vi.mock('@openpanel/db', async () => {
return {
...actual,
createEvent: vi.fn(),
- checkNotificationRulesForEvent: vi.fn().mockResolvedValue(true),
sessionBuffer: {
getExistingSession: vi.fn(),
},
diff --git a/apps/worker/src/jobs/import.ts b/apps/worker/src/jobs/import.ts
index f03a053cc..b9986bd87 100644
--- a/apps/worker/src/jobs/import.ts
+++ b/apps/worker/src/jobs/import.ts
@@ -139,7 +139,7 @@ export async function importJob(job: Job) {
// Phase 1: Fetch & Transform - Process events in batches
if (shouldRunStep('loading')) {
- const eventBatch: any = [];
+ let eventBatch: any = [];
for await (const rawEvent of providerInstance.parseSource(
resumeLoadingFrom,
)) {
@@ -158,7 +158,13 @@ export async function importJob(job: Job) {
// Process batch when it reaches the batch size
if (eventBatch.length >= BATCH_SIZE) {
- jobLogger.info('Processing batch', { batchSize: eventBatch.length });
+ const memUsage = process.memoryUsage();
+ jobLogger.info('Processing batch', {
+ batchSize: eventBatch.length,
+ heapUsedMB: Math.round(memUsage.heapUsed / 1024 / 1024),
+ heapTotalMB: Math.round(memUsage.heapTotal / 1024 / 1024),
+ externalMB: Math.round(memUsage.external / 1024 / 1024),
+ });
const transformedEvents: IClickhouseEvent[] = eventBatch.map(
(
@@ -170,7 +176,7 @@ export async function importJob(job: Job) {
await insertImportBatch(transformedEvents, importId);
processedEvents += eventBatch.length;
- eventBatch.length = 0;
+ eventBatch = [];
const createdAt = new Date(transformedEvents[0]?.created_at || '')
.toISOString()
@@ -190,6 +196,14 @@ export async function importJob(job: Job) {
// Process remaining events in the last batch
if (eventBatch.length > 0) {
+ const memUsage = process.memoryUsage();
+ jobLogger.info('Processing final batch', {
+ batchSize: eventBatch.length,
+ heapUsedMB: Math.round(memUsage.heapUsed / 1024 / 1024),
+ heapTotalMB: Math.round(memUsage.heapTotal / 1024 / 1024),
+ externalMB: Math.round(memUsage.external / 1024 / 1024),
+ });
+
const transformedEvents = eventBatch.map(
(
// @ts-expect-error
@@ -200,7 +214,7 @@ export async function importJob(job: Job) {
await insertImportBatch(transformedEvents, importId);
processedEvents += eventBatch.length;
- eventBatch.length = 0;
+ eventBatch = [];
const createdAt = new Date(transformedEvents[0]?.created_at || '')
.toISOString()
diff --git a/apps/worker/src/jobs/sessions.ts b/apps/worker/src/jobs/sessions.ts
index 84b35cbb4..c0e14dc17 100644
--- a/apps/worker/src/jobs/sessions.ts
+++ b/apps/worker/src/jobs/sessions.ts
@@ -38,6 +38,7 @@ const updateEventsCount = cacheable(async function updateEventsCount(
});
if (!organization) {
+ logger.warn('updateEventsCount: Organization not found', { projectId });
return;
}
diff --git a/packages/constants/index.ts b/packages/constants/index.ts
index 17cd2d15e..c2545d964 100644
--- a/packages/constants/index.ts
+++ b/packages/constants/index.ts
@@ -91,6 +91,8 @@ export const operators = {
lt: 'Less than',
gte: 'Greater than or equal to',
lte: 'Less than or equal to',
+ inCohort: 'In cohort',
+ notInCohort: 'Not in cohort',
} as const;
export const chartTypes = {
@@ -245,3 +247,62 @@ export function getDefaultIntervalByDates(
return null;
}
+
+// ── Custom Alerts ───────────────────────────────────────────────────
+
+export type AlertFrequency = 'hour' | 'day' | 'week' | 'month';
+
+/**
+ * Maps alert frequency to the range used for historical data (anomaly detection)
+ */
+export const ALERT_FREQUENCY_TO_RANGE: Record = {
+ hour: '7d',
+ day: '30d',
+ week: '30d',
+ month: '180d',
+};
+
+/**
+ * Maps alert frequency to the range used for current period evaluation (threshold)
+ */
+export const ALERT_FREQUENCY_TO_CURRENT_RANGE: Record =
+ {
+ hour: 'lastHour',
+ day: 'today',
+ week: '7d',
+ month: '30d',
+ };
+
+/**
+ * Maps alert frequency to the ChartEngine interval
+ */
+export const ALERT_FREQUENCY_TO_INTERVAL: Record = {
+ hour: 'hour',
+ day: 'day',
+ week: 'week',
+ month: 'month',
+};
+
+/**
+ * How many historical data points to use for anomaly detection
+ */
+export const ANOMALY_HISTORY_COUNT = 30;
+
+/**
+ * Z-scores for confidence levels used in anomaly detection
+ */
+export const CONFIDENCE_Z_SCORES: Record = {
+ '95': 1.96,
+ '98': 2.326,
+ '99': 2.576,
+};
+
+/**
+ * Frequency duration in milliseconds — used for rate limiting notifications
+ */
+export const ALERT_FREQUENCY_MS: Record = {
+ hour: 60 * 60 * 1000,
+ day: 24 * 60 * 60 * 1000,
+ week: 7 * 24 * 60 * 60 * 1000,
+ month: 30 * 24 * 60 * 60 * 1000,
+};
diff --git a/packages/db/code-migrations/10-backfill-profile-event-summary.ts b/packages/db/code-migrations/10-backfill-profile-event-summary.ts
new file mode 100644
index 000000000..d5869cb2e
--- /dev/null
+++ b/packages/db/code-migrations/10-backfill-profile-event-summary.ts
@@ -0,0 +1,170 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import {
+ chMigrationClient,
+ runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+/**
+ * Migration 10: Backfill profile_event_summary_mv with historical data
+ *
+ * Context: profile_event_summary_mv was created with populate: false to avoid OOM
+ * This means it only captures NEW events, missing all historical data.
+ * This migration backfills the MV with historical events for cohort queries.
+ *
+ * Hard backstop: January 1, 2025 (won't backfill data before this date)
+ *
+ * Steps:
+ * 1. Check the date range of events in the events table
+ * 2. Backfill historical data day-by-day in batches to avoid OOM
+ * 3. Generate SQL file for review before execution
+ */
+export async function up() {
+ const isClustered = getIsCluster();
+ const sqls: string[] = [];
+
+ // Hard backstop date - don't backfill before this
+ const BACKSTOP_DATE = new Date('2025-01-01');
+
+ // Step 1: Check the actual date range in the events table
+ console.log('🔍 Checking date range in events table...');
+
+ const checkDataQuery = await chMigrationClient.query({
+ query: `
+ SELECT
+ min(toDate(created_at)) as min_date,
+ max(toDate(created_at)) as max_date,
+ count() as total_events,
+ uniq(profile_id) as total_profiles
+ FROM events
+ WHERE profile_id != device_id
+ AND toDate(created_at) >= '2025-01-01'
+ `,
+ format: 'JSONEachRow',
+ });
+
+ const dataRange = await checkDataQuery.json<{
+ min_date: string;
+ max_date: string;
+ total_events: string;
+ total_profiles: string;
+ }>();
+
+ if (dataRange[0]?.min_date && dataRange[0]?.max_date) {
+ let startDate = new Date(dataRange[0].min_date);
+ const endDate = new Date(dataRange[0].max_date);
+
+ // Enforce backstop date
+ if (startDate < BACKSTOP_DATE) {
+ console.log(`⚠️ Data exists before backstop date, limiting to ${BACKSTOP_DATE.toISOString().split('T')[0]}`);
+ startDate = BACKSTOP_DATE;
+ }
+
+ const totalEvents = Number(dataRange[0].total_events);
+ const totalProfiles = Number(dataRange[0].total_profiles);
+ const daysDiff = Math.ceil((endDate.getTime() - startDate.getTime()) / (1000 * 60 * 60 * 24)) + 1;
+
+ console.log('========================================');
+ console.log('📊 Backfill Plan for profile_event_summary_mv:');
+ console.log(` Backstop Date: ${BACKSTOP_DATE.toISOString().split('T')[0]} (hard limit)`);
+ console.log(` Start Date: ${startDate.toISOString().split('T')[0]}`);
+ console.log(` End Date: ${endDate.toISOString().split('T')[0]} (inclusive)`);
+ console.log(` Days: ${daysDiff} days`);
+ console.log(` Events: ${totalEvents.toLocaleString()} total events`);
+ console.log(` Profiles: ${totalProfiles.toLocaleString()} unique profiles`);
+ console.log('========================================');
+ console.log('');
+
+ // Step 2: Generate day-by-day INSERT statements
+ // Process from most recent to oldest to prioritize recent data for cohorts
+ const targetTable = isClustered ? 'profile_event_summary_mv_replicated' : 'profile_event_summary_mv';
+ const backfillSqls: string[] = [];
+
+ let currentDate = new Date(endDate); // Start from endDate (most recent)
+
+ while (currentDate >= startDate) {
+ const dateStr = currentDate.toISOString().split('T')[0];
+
+ const sql = `INSERT INTO ${targetTable}
+ SELECT
+ project_id,
+ profile_id,
+ name,
+ toStartOfDay(created_at) AS event_date,
+ countState() AS event_count,
+ minState(created_at) AS first_event_time,
+ maxState(created_at) AS last_event_time,
+ sumState(duration) AS total_duration
+ FROM events
+ WHERE toDate(created_at) = '${dateStr}'
+ AND profile_id != device_id
+ GROUP BY project_id, profile_id, name, event_date`;
+
+ backfillSqls.push(sql);
+ currentDate.setDate(currentDate.getDate() - 1);
+ }
+
+ sqls.push(...backfillSqls);
+
+ console.log(`📝 Generated ${sqls.length} daily INSERT statements`);
+ } else {
+ console.log('⚠️ No data found in events table since 2025-01-01, skipping backfill');
+ }
+
+ // Step 3: Write SQL to file for review
+ const sqlFilePath = path.join(__filename.replace('.ts', '.sql'));
+ fs.writeFileSync(
+ sqlFilePath,
+ sqls
+ .map((sql) =>
+ sql
+ .trim()
+ .replace(/;$/, '')
+ .replace(/\n{2,}/g, '\n')
+ .concat(';'),
+ )
+ .join('\n\n---\n\n'),
+ );
+
+ console.log(`📄 SQL written to: ${sqlFilePath}`);
+ console.log('');
+
+ // Step 4: Execute if not in dry-run mode
+ if (!process.argv.includes('--dry')) {
+ console.log('🚀 Starting backfill execution...');
+ console.log('⏱️ This may take a while depending on data volume');
+ console.log('');
+
+ let completed = 0;
+ const total = sqls.length;
+
+ for (const sql of sqls) {
+ await runClickhouseMigrationCommands([sql]);
+ completed++;
+
+ // Show progress every 10%
+ if (completed % Math.ceil(total / 10) === 0 || completed === total) {
+ const percentage = Math.round((completed / total) * 100);
+ console.log(` Progress: ${completed}/${total} days (${percentage}%)`);
+ }
+ }
+
+ console.log('');
+ console.log('✅ Migration completed successfully!');
+ console.log('');
+ console.log('📋 Next steps:');
+ console.log(' 1. Verify data in profile_event_summary_mv');
+ console.log(' 2. Test cohort queries with historical data');
+ console.log(' 3. Check cohort member counts');
+ } else {
+ console.log('🔍 DRY RUN MODE: SQL generated but not executed');
+ console.log(' Review the SQL file and run without --dry to execute');
+ }
+}
+
+export async function down() {
+ console.log('⚠️ Down migration not supported for backfill operations');
+ console.log(' Data has been inserted into the MV and cannot be easily rolled back');
+ console.log(' If needed, drop and recreate the MV: profile_event_summary_mv');
+}
diff --git a/packages/db/code-migrations/11-backfill-profile-event-property-v2.ts b/packages/db/code-migrations/11-backfill-profile-event-property-v2.ts
new file mode 100644
index 000000000..2ccd87172
--- /dev/null
+++ b/packages/db/code-migrations/11-backfill-profile-event-property-v2.ts
@@ -0,0 +1,308 @@
+import {
+ chMigrationClient,
+ runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+/**
+ * Backfill profile_event_property_summary_mv in hourly batches
+ *
+ * Safe to run on existing table - AggregatingMergeTree will merge states correctly!
+ *
+ * Usage:
+ * # For clustered (production):
+ * npm run migrate -- 11 --start-date=2025-01-01 --end-date=2026-01-10 --table=profile_event_property_summary_mv_replicated
+ *
+ * # For self-hosted:
+ * npm run migrate -- 11 --start-date=2025-01-01 --end-date=2026-01-10 --table=profile_event_property_summary_mv
+ *
+ * # Optional flags:
+ * npm run migrate -- 11 ... --dry (dry run only)
+ * npm run migrate -- 11 ... --batch-hours=2 (larger batches)
+ * npm run migrate -- 11 ... --skip-existing (skip dates that have data)
+ */
+
+interface BackfillOptions {
+ startDate: string;
+ endDate: string;
+ targetTable: string;
+ isDryRun: boolean;
+ batchSizeHours: number;
+}
+
+function parseArgs(): BackfillOptions {
+ const args = process.argv;
+
+ const startDateArg = args.find(arg => arg.startsWith('--start-date='));
+ const endDateArg = args.find(arg => arg.startsWith('--end-date='));
+ const tableArg = args.find(arg => arg.startsWith('--table='));
+ const batchSizeArg = args.find(arg => arg.startsWith('--batch-hours='));
+
+ if (!startDateArg || !endDateArg || !tableArg) {
+ console.error('❌ Missing required arguments');
+ console.log('');
+ console.log('Usage:');
+ console.log(' npm run migrate -- 11 \\');
+ console.log(' --start-date=2025-01-01 \\');
+ console.log(' --end-date=2026-01-10 \\');
+ console.log(' --table=profile_event_property_summary_mv_v2_replicated');
+ console.log('');
+ console.log('Optional flags:');
+ console.log(' --batch-hours=1 (default: 1, process N hours at a time)');
+ console.log(' --dry (generate SQL only, don\'t execute)');
+ console.log('');
+ process.exit(1);
+ }
+
+ return {
+ startDate: startDateArg.split('=')[1]!,
+ endDate: endDateArg.split('=')[1]!,
+ targetTable: tableArg.split('=')[1]!,
+ isDryRun: args.includes('--dry'),
+ batchSizeHours: batchSizeArg ? parseInt(batchSizeArg.split('=')[1]!) : 1,
+ };
+}
+
+export async function up() {
+ const options = parseArgs();
+
+ console.log('🚀 Profile Event Property Summary Backfill');
+ console.log('='.repeat(60));
+ console.log(`📅 Date Range: ${options.startDate} to ${options.endDate}`);
+ console.log(`📦 Target Table: ${options.targetTable}`);
+ console.log(`⏱️ Batch Size: ${options.batchSizeHours} hour(s)`);
+ console.log(`🔧 Mode: ${options.isDryRun ? 'DRY RUN' : 'EXECUTION'}`);
+ console.log('='.repeat(60));
+ console.log('');
+
+ // Skip analysis (too memory intensive for large datasets)
+ console.log('⏩ Skipping analysis - will process all data in date range');
+ console.log('');
+
+ // Generate hourly batches
+ const batches = generateHourlyBatches(
+ options.startDate,
+ options.endDate,
+ options.batchSizeHours,
+ options.targetTable
+ );
+
+ console.log(`📦 Generated ${batches.length} batches`);
+ console.log(`⏱️ Est. Time: ${formatEstimatedTime(batches.length)} (at ~30s/batch)`);
+ console.log('');
+
+ if (options.isDryRun) {
+ console.log('🔍 DRY RUN - Sample batch SQL:');
+ console.log('─'.repeat(80));
+ console.log(batches[0]?.sql.trim() || 'No batches generated');
+ console.log('─'.repeat(80));
+ console.log('');
+ console.log(`💡 Total batches: ${batches.length}`);
+ console.log('💡 Remove --dry to execute');
+ return;
+ }
+
+ // Step 3: Execute batches
+ await executeBatches(batches, options.targetTable);
+}
+
+async function analyzeDataRange(startDate: string, endDate: string) {
+ console.log('🔍 Analyzing data...');
+
+ const query = `
+ SELECT
+ count() as total_events,
+ uniq(profile_id) as total_profiles,
+ sum(length(mapKeys(properties))) as total_properties
+ FROM events
+ PREWHERE toDate(created_at) >= '${startDate}'
+ AND toDate(created_at) <= '${endDate}'
+ AND profile_id != device_id
+ WHERE arrayExists(k -> k != '', mapKeys(properties))
+ AND arrayExists(v -> v != '', mapValues(properties))
+ `;
+
+ const result = await chMigrationClient.query({ query, format: 'JSONEachRow' });
+ const data = await result.json<{
+ total_events: string;
+ total_profiles: string;
+ total_properties: string;
+ }>();
+
+ const row = data[0];
+ if (!row || row.total_events === '0') {
+ return { hasData: false, totalEvents: 0, totalProfiles: 0, avgProperties: 0 };
+ }
+
+ const totalEvents = Number(row.total_events);
+ const totalProperties = Number(row.total_properties);
+
+ return {
+ hasData: true,
+ totalEvents,
+ totalProfiles: Number(row.total_profiles),
+ avgProperties: totalProperties / totalEvents,
+ };
+}
+
+function generateHourlyBatches(
+ startDate: string,
+ endDate: string,
+ batchSizeHours: number,
+ targetTable: string
+) {
+ const batches: Array<{ startTime: string; endTime: string; sql: string }> = [];
+
+ const start = new Date(startDate + 'T00:00:00Z');
+ const end = new Date(endDate + 'T23:59:59Z');
+ let current = new Date(start);
+
+ while (current < end) {
+ const batchEnd = new Date(current);
+ batchEnd.setHours(current.getHours() + batchSizeHours);
+
+ // Ensure we don't go past the end date
+ if (batchEnd > end) {
+ batchEnd.setTime(end.getTime());
+ }
+
+ const startStr = current.toISOString().slice(0, 19).replace('T', ' ');
+ const endStr = batchEnd.toISOString().slice(0, 19).replace('T', ' ');
+
+ const sql = `
+INSERT INTO ${targetTable}
+SELECT
+ project_id,
+ profile_id,
+ name,
+ property_key,
+ property_value,
+ toStartOfDay(created_at) AS event_date,
+ countState() AS event_count,
+ minState(created_at) AS first_event_time,
+ maxState(created_at) AS last_event_time
+FROM events
+ARRAY JOIN
+ mapKeys(properties) AS property_key,
+ mapValues(properties) AS property_value
+PREWHERE
+ created_at >= toDateTime64('${startStr}', 3)
+ AND created_at < toDateTime64('${endStr}', 3)
+ AND profile_id != device_id
+ AND name IN (
+ 'userFlags',
+ 'appsflyerInstallWebhook',
+ '_firstSeen',
+ 'screenVisible',
+ 'trialScreenOpen',
+ 'trialStarted',
+ 'subscriptionPurchased',
+ 'trialConverted',
+ 'supportChatOpened',
+ 'showOpen',
+ 'reelOpen'
+ )
+WHERE
+ property_key != ''
+ AND property_value != ''
+ AND property_key IN (
+ 'subscription-v3-enabled',
+ 'af_adset_id',
+ 'screenName',
+ 'starter-shows',
+ 'reels-entry-point-experiment',
+ 'playlist'
+ )
+GROUP BY project_id, profile_id, name, property_key, property_value, event_date
+SETTINGS
+ max_memory_usage = 10000000000,
+ max_execution_time = 1800,
+ max_threads = 8`;
+
+ batches.push({ startTime: startStr, endTime: endStr, sql });
+
+ // Advance to next batch start
+ current = new Date(batchEnd);
+
+ // Break if we've reached or passed the end (prevent infinite loop)
+ if (current >= end) {
+ break;
+ }
+ }
+
+ return batches;
+}
+
+async function executeBatches(
+ batches: Array<{ startTime: string; endTime: string; sql: string }>,
+ targetTable: string
+) {
+ console.log('🚀 Starting execution...');
+ console.log('💡 AggregatingMergeTree will merge any duplicate data');
+ console.log('');
+
+ let completed = 0;
+ const total = batches.length;
+ const startTime = Date.now();
+
+ for (const batch of batches) {
+ try {
+ // Execute INSERT (no skip - reprocess all data, AggregatingMergeTree will merge)
+ const execStart = Date.now();
+ await runClickhouseMigrationCommands([batch.sql]);
+ const execTime = Date.now() - execStart;
+
+ completed++;
+
+ if (completed % 50 === 0 || execTime > 60000 || completed === total) {
+ logProgress(completed, total, startTime, `${batch.startTime} to ${batch.endTime}`, execTime);
+ }
+
+ } catch (error: any) {
+ console.error(`\n❌ Error: ${batch.startTime}`, error.message);
+ throw error;
+ }
+ }
+
+ const totalTime = Date.now() - startTime;
+
+ console.log('');
+ console.log('✅ Backfill complete!');
+ console.log(` Time: ${formatTime(Math.round(totalTime / 1000))}`);
+ console.log(` Processed: ${completed}/${total} batches`);
+ console.log(` Avg/batch: ${Math.round(totalTime / completed / 1000)}s`);
+ console.log('');
+}
+
+function logProgress(
+ completed: number,
+ total: number,
+ startTime: number,
+ label?: string,
+ execTime?: number
+) {
+ const pct = Math.round((completed / total) * 100);
+ const elapsed = (Date.now() - startTime) / 1000;
+ const remaining = ((total - completed) / completed) * elapsed;
+
+ let msg = ` [${pct}%] ${completed}/${total} | ETA: ${formatTime(Math.round(remaining))}`;
+ if (label && execTime) msg += `\n Last: ${label} (${Math.round(execTime / 1000)}s)`;
+
+ console.log(msg);
+}
+
+function formatTime(sec: number): string {
+ if (sec < 60) return `${sec}s`;
+ const min = Math.floor(sec / 60);
+ if (min < 60) return `${min}m`;
+ const hrs = Math.floor(min / 60);
+ return `${hrs}h ${min % 60}m`;
+}
+
+function formatEstimatedTime(batches: number): string {
+ return formatTime(batches * 30);
+}
+
+export async function down() {
+ console.log('⚠️ No down migration - backfill is data only');
+}
diff --git a/packages/db/code-migrations/12-backfill-profile-event-summary-v2.ts b/packages/db/code-migrations/12-backfill-profile-event-summary-v2.ts
new file mode 100644
index 000000000..fbf2b1a47
--- /dev/null
+++ b/packages/db/code-migrations/12-backfill-profile-event-summary-v2.ts
@@ -0,0 +1,217 @@
+import {
+ chMigrationClient,
+ runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+/**
+ * Backfill profile_event_summary_mv_v2 in daily batches
+ *
+ * This MV is for simple event-based cohorts WITHOUT property filters
+ * Much simpler than profile_event_property_summary_mv - no ARRAY JOIN
+ *
+ * Usage:
+ * # For clustered (production):
+ * npm run migrate -- 12 --start-date=2025-01-01 --end-date=2026-01-10 --table=profile_event_summary_mv_v2_replicated
+ *
+ * # For self-hosted:
+ * npm run migrate -- 12 --start-date=2025-01-01 --end-date=2026-01-10 --table=profile_event_summary_mv_v2
+ *
+ * # Optional flags:
+ * npm run migrate -- 12 ... --dry (dry run only)
+ */
+
+interface BackfillOptions {
+ startDate: string;
+ endDate: string;
+ targetTable: string;
+ isDryRun: boolean;
+}
+
+function parseArgs(): BackfillOptions {
+ const args = process.argv;
+
+ const startDateArg = args.find(arg => arg.startsWith('--start-date='));
+ const endDateArg = args.find(arg => arg.startsWith('--end-date='));
+ const tableArg = args.find(arg => arg.startsWith('--table='));
+
+ if (!startDateArg || !endDateArg || !tableArg) {
+ console.error('❌ Missing required arguments');
+ console.log('');
+ console.log('Usage:');
+ console.log(' npm run migrate -- 12 \\');
+ console.log(' --start-date=2025-01-01 \\');
+ console.log(' --end-date=2026-01-10 \\');
+ console.log(' --table=profile_event_summary_mv_v2');
+ console.log('');
+ console.log('Optional flags:');
+ console.log(' --dry (generate SQL only, don\'t execute)');
+ console.log('');
+ process.exit(1);
+ }
+
+ return {
+ startDate: startDateArg.split('=')[1]!,
+ endDate: endDateArg.split('=')[1]!,
+ targetTable: tableArg.split('=')[1]!,
+ isDryRun: args.includes('--dry'),
+ };
+}
+
+export async function up() {
+ const options = parseArgs();
+
+ console.log('🚀 Profile Event Summary Backfill (Daily Batches)');
+ console.log('='.repeat(60));
+ console.log(`📅 Date Range: ${options.startDate} to ${options.endDate}`);
+ console.log(`📦 Target Table: ${options.targetTable}`);
+ console.log(`🔧 Mode: ${options.isDryRun ? 'DRY RUN' : 'EXECUTION'}`);
+ console.log('='.repeat(60));
+ console.log('');
+
+ // Skip analysis
+ console.log('⏩ Skipping analysis - will process all events in date range');
+ console.log('');
+
+ // Generate daily batches
+ const batches = generateDailyBatches(
+ options.startDate,
+ options.endDate,
+ options.targetTable
+ );
+
+ console.log(`📦 Generated ${batches.length} daily batches`);
+ console.log(`⏱️ Est. Time: ${formatEstimatedTime(batches.length)} (at ~2min/day)`);
+ console.log('');
+
+ if (options.isDryRun) {
+ console.log('🔍 DRY RUN - Sample batch SQL:');
+ console.log('─'.repeat(80));
+ console.log(batches[0]?.sql.trim() || 'No batches generated');
+ console.log('─'.repeat(80));
+ console.log('');
+ console.log(`💡 Total batches: ${batches.length}`);
+ console.log('💡 Remove --dry to execute');
+ return;
+ }
+
+ // Execute batches
+ await executeBatches(batches);
+}
+
+function generateDailyBatches(
+ startDate: string,
+ endDate: string,
+ targetTable: string
+) {
+ const batches: Array<{ date: string; sql: string }> = [];
+
+ const start = new Date(startDate);
+ const end = new Date(endDate);
+ let current = new Date(start);
+
+ while (current <= end) {
+ const dateStr = current.toISOString().split('T')[0];
+
+ const sql = `
+INSERT INTO ${targetTable}
+SELECT
+ project_id,
+ profile_id,
+ name,
+ toStartOfDay(created_at) AS event_date,
+ countState() AS event_count,
+ minState(created_at) AS first_event_time,
+ maxState(created_at) AS last_event_time,
+ sumState(duration) AS total_duration
+FROM events
+PREWHERE
+ toDate(created_at) = '${dateStr}'
+ AND profile_id != device_id
+GROUP BY project_id, profile_id, name, event_date
+SETTINGS
+ max_memory_usage = 30000000000,
+ max_execution_time = 7200,
+ max_threads = 16`;
+
+ batches.push({ date: dateStr, sql });
+
+ // Move to next day
+ current.setDate(current.getDate() + 1);
+ }
+
+ return batches;
+}
+
+async function executeBatches(
+ batches: Array<{ date: string; sql: string }>
+) {
+ console.log('🚀 Starting execution...');
+ console.log('💡 AggregatingMergeTree will merge any duplicate data');
+ console.log('');
+
+ let completed = 0;
+ const total = batches.length;
+ const startTime = Date.now();
+
+ for (const batch of batches) {
+ try {
+ // Execute INSERT
+ const execStart = Date.now();
+ await runClickhouseMigrationCommands([batch.sql]);
+ const execTime = Date.now() - execStart;
+
+ completed++;
+
+ // Show progress every 10 batches or if query took > 2 minutes
+ if (completed % 10 === 0 || execTime > 120000 || completed === total) {
+ logProgress(completed, total, startTime, batch.date, execTime);
+ }
+
+ } catch (error: any) {
+ console.error(`\n❌ Error processing ${batch.date}:`, error.message);
+ throw error;
+ }
+ }
+
+ const totalTime = Date.now() - startTime;
+
+ console.log('');
+ console.log('✅ Backfill complete!');
+ console.log(` Time: ${formatTime(Math.round(totalTime / 1000))}`);
+ console.log(` Processed: ${completed}/${total} days`);
+ console.log(` Avg/day: ${Math.round(totalTime / completed / 1000)}s`);
+ console.log('');
+}
+
+function logProgress(
+ completed: number,
+ total: number,
+ startTime: number,
+ date: string,
+ execTime: number
+) {
+ const pct = Math.round((completed / total) * 100);
+ const elapsed = (Date.now() - startTime) / 1000;
+ const remaining = ((total - completed) / completed) * elapsed;
+
+ const msg = ` [${pct}%] ${completed}/${total} | ETA: ${formatTime(Math.round(remaining))}\n Last: ${date} (${Math.round(execTime / 1000)}s)`;
+ console.log(msg);
+}
+
+function formatTime(sec: number): string {
+ if (sec < 60) return `${sec}s`;
+ const min = Math.floor(sec / 60);
+ if (min < 60) return `${min}m`;
+ const hrs = Math.floor(min / 60);
+ return `${hrs}h ${min % 60}m`;
+}
+
+function formatEstimatedTime(days: number): string {
+ // Assume ~2 minutes per day on average
+ return formatTime(days * 120);
+}
+
+export async function down() {
+ console.log('⚠️ No down migration - backfill is data only');
+}
diff --git a/packages/db/code-migrations/13-deduplicate-events.ts b/packages/db/code-migrations/13-deduplicate-events.ts
new file mode 100644
index 000000000..4222ea8f7
--- /dev/null
+++ b/packages/db/code-migrations/13-deduplicate-events.ts
@@ -0,0 +1,211 @@
+import {
+ chMigrationClient,
+ runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+/**
+ * IST Data Loss Validation & Recovery
+ *
+ * Validates how much data was lost due to IST filter in moveImportsToProduction,
+ * and optionally recovers the lost events from events_imports_v2 into events_tmp.
+ *
+ * Lost events = those where toDate(created_at) != toDate(addHours(created_at, 5.5))
+ * i.e., events between 18:30-23:59 UTC (00:00-05:30 IST next day)
+ *
+ * Usage:
+ * Validate only (dry run):
+ * pnpm migrate:deploy:code -- 13 --cluster --dry --date=2026-01-05
+ *
+ * Validate and recover:
+ * pnpm migrate:deploy:code -- 13 --cluster --date=2026-01-05 --no-record
+ */
+
+const IMPORTS_TABLE = 'events_imports_v2';
+const TMP_TABLE = 'events_tmp';
+
+const DEDUP_KEY =
+ 'project_id, name, device_id, profile_id, toStartOfSecond(created_at), path, properties';
+
+const COLUMNS = `id, name, sdk_name, sdk_version, device_id, profile_id, project_id,
+ session_id, path, origin, referrer, referrer_name, referrer_type,
+ duration, created_at, country, city, region, longitude, latitude,
+ os, os_version, browser, browser_version, device, brand, model, imported_at`;
+
+function parseArgs() {
+ const args = process.argv;
+ const dateArg = args.find((a: string) => a.startsWith('--date='));
+
+ if (!dateArg) {
+ console.error('Missing required --date=YYYY-MM-DD argument');
+ console.error(' Example: pnpm migrate:deploy:code -- 13 --cluster --date=2026-01-05');
+ process.exit(1);
+ }
+
+ const date = dateArg!.split('=')[1]!;
+
+ return {
+ date,
+ isCluster: getIsCluster(),
+ isDry: args.includes('--dry'),
+ };
+}
+
+export async function up() {
+ const { date, isCluster, isDry } = parseArgs();
+
+ console.log('='.repeat(60));
+ console.log(' IST DATA LOSS VALIDATION');
+ console.log(` Date: ${date}`);
+ console.log(` Mode: ${isDry ? 'DRY RUN (validate only)' : 'EXECUTE (validate + recover)'}`);
+ console.log('='.repeat(60));
+
+ // Step 0: Check events_tmp current state for this date
+ console.log(`\n[Step 0] Current events_tmp counts for UTC ${date}:`);
+ const tmpResult = await chMigrationClient.query({
+ query: `
+ SELECT
+ name,
+ count() as total,
+ uniq(${DEDUP_KEY}) as unique_events
+ FROM ${TMP_TABLE}
+ WHERE toDate(created_at) = '${date}'
+ GROUP BY name
+ ORDER BY total DESC`,
+ format: 'JSONEachRow',
+ });
+ const tmpData = await tmpResult.json<{ name: string; total: string; unique_events: string }>();
+ const tmpMap = new Map(tmpData.map((r) => [r.name, { total: Number(r.total), unique: Number(r.unique_events) }]));
+
+ console.log(`\n ${'Event'.padEnd(25)} ${'Total'.padStart(10)} ${'Unique'.padStart(10)}`);
+ console.log(' ' + '-'.repeat(47));
+ for (const row of tmpData) {
+ console.log(
+ ` ${row.name.padEnd(25)} ${Number(row.total).toLocaleString().padStart(10)} ${Number(row.unique_events).toLocaleString().padStart(10)}`,
+ );
+ }
+
+ // Step 1: Check events_imports_v2 full counts for this date (source of truth)
+ console.log(`\n[Step 1] events_imports_v2 vs events_tmp for UTC ${date}:`);
+ const importsResult = await chMigrationClient.query({
+ query: `
+ SELECT
+ name,
+ count() as total,
+ uniq(${DEDUP_KEY}) as unique_events
+ FROM ${IMPORTS_TABLE}
+ WHERE toDate(created_at) = '${date}'
+ GROUP BY name
+ ORDER BY total DESC`,
+ format: 'JSONEachRow',
+ });
+ const importsData = await importsResult.json<{ name: string; total: string; unique_events: string }>();
+
+ console.log(`\n ${'Event'.padEnd(25)} ${'imp_total'.padStart(10)} ${'imp_uniq'.padStart(10)} ${'tmp_total'.padStart(10)} ${'tmp_uniq'.padStart(10)} ${'Missing'.padStart(10)}`);
+ console.log(' ' + '-'.repeat(77));
+ for (const row of importsData) {
+ const impTotal = Number(row.total);
+ const impUniq = Number(row.unique_events);
+ const tmp = tmpMap.get(row.name) ?? { total: 0, unique: 0 };
+ const missingUniq = impUniq - tmp.unique;
+ console.log(
+ ` ${row.name.padEnd(25)} ${impTotal.toLocaleString().padStart(10)} ${impUniq.toLocaleString().padStart(10)} ${tmp.total.toLocaleString().padStart(10)} ${tmp.unique.toLocaleString().padStart(10)} ${missingUniq.toLocaleString().padStart(10)}`,
+ );
+ }
+
+ // Step 2: Check lost events (18:30-23:59 UTC = where UTC date != IST date)
+ console.log(`\n[Step 2] Lost events (18:30-23:59 UTC, not moved to events_tmp):`);
+ const lostResult = await chMigrationClient.query({
+ query: `
+ SELECT
+ name,
+ count() as lost_total
+ FROM ${IMPORTS_TABLE}
+ WHERE toDate(created_at) = '${date}'
+ AND toDate(created_at) != toDate(addHours(created_at, 5.5))
+ GROUP BY name
+ ORDER BY lost_total DESC`,
+ format: 'JSONEachRow',
+ });
+ const lostData = await lostResult.json<{ name: string; lost_total: string }>();
+
+ let totalLost = 0;
+ console.log(`\n ${'Event'.padEnd(25)} ${'Lost'.padStart(10)}`);
+ console.log(' ' + '-'.repeat(37));
+ for (const row of lostData) {
+ const lost = Number(row.lost_total);
+ totalLost += lost;
+ console.log(
+ ` ${row.name.padEnd(25)} ${lost.toLocaleString().padStart(10)}`,
+ );
+ }
+ console.log(' ' + '-'.repeat(37));
+ console.log(` ${'TOTAL'.padEnd(25)} ${totalLost.toLocaleString().padStart(10)}`);
+
+ if (isDry) {
+ console.log('\n[DRY RUN] Recovery SQL that would execute:');
+ console.log(`
+ INSERT INTO ${TMP_TABLE} (${COLUMNS})
+ SELECT ${COLUMNS}
+ FROM ${IMPORTS_TABLE}
+ WHERE toDate(created_at) = '${date}'
+ AND toDate(created_at) != toDate(addHours(created_at, 5.5))
+ SETTINGS max_memory_usage = 40000000000, max_execution_time = 18000;`);
+ console.log('\n Run without --dry to execute recovery.');
+ return;
+ }
+
+ // Step 3: Recover lost events
+ if (totalLost === 0) {
+ console.log('\n No lost events to recover!');
+ return;
+ }
+
+ console.log(`\n[Step 3] Recovering ${totalLost.toLocaleString()} lost events into ${TMP_TABLE}...`);
+ await runClickhouseMigrationCommands([
+ `INSERT INTO ${TMP_TABLE} (${COLUMNS})
+ SELECT ${COLUMNS}
+ FROM ${IMPORTS_TABLE}
+ WHERE toDate(created_at) = '${date}'
+ AND toDate(created_at) != toDate(addHours(created_at, 5.5))
+ SETTINGS
+ max_memory_usage = 40000000000,
+ max_execution_time = 18000`,
+ ]);
+
+ // Step 4: Verify recovery
+ console.log(`\n[Step 4] Verifying recovery - events_tmp counts after:`);
+ const afterResult = await chMigrationClient.query({
+ query: `
+ SELECT
+ name,
+ count() as total,
+ uniq(${DEDUP_KEY}) as unique_events
+ FROM ${TMP_TABLE}
+ WHERE toDate(created_at) = '${date}'
+ GROUP BY name
+ ORDER BY total DESC`,
+ format: 'JSONEachRow',
+ });
+ const afterData = await afterResult.json<{ name: string; total: string; unique_events: string }>();
+
+ console.log(`\n ${'Event'.padEnd(25)} ${'Before'.padStart(10)} ${'After'.padStart(10)} ${'Recovered'.padStart(10)} ${'Unique'.padStart(10)}`);
+ console.log(' ' + '-'.repeat(67));
+ for (const row of afterData) {
+ const after = Number(row.total);
+ const unique = Number(row.unique_events);
+ const before = (tmpMap.get(row.name) ?? { total: 0 }).total;
+ const recovered = after - before;
+ console.log(
+ ` ${row.name.padEnd(25)} ${before.toLocaleString().padStart(10)} ${after.toLocaleString().padStart(10)} ${recovered.toLocaleString().padStart(10)} ${unique.toLocaleString().padStart(10)}`,
+ );
+ }
+
+ console.log('\n' + '='.repeat(60));
+ console.log(' RECOVERY COMPLETE');
+ console.log('='.repeat(60));
+}
+
+export async function down() {
+ console.log('No down migration');
+}
diff --git a/packages/db/code-migrations/14-move-events-to-tmp.ts b/packages/db/code-migrations/14-move-events-to-tmp.ts
new file mode 100644
index 000000000..135660cd9
--- /dev/null
+++ b/packages/db/code-migrations/14-move-events-to-tmp.ts
@@ -0,0 +1,126 @@
+import {
+ chMigrationClient,
+ runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+/**
+ * Move events to events_tmp2 for a given date
+ *
+ * Copies data as-is from events to events_tmp2 (no dedup).
+ * Used to bring Jan 16-31 data into events_tmp2.
+ *
+ * Usage:
+ * Dry run:
+ * pnpm migrate:deploy:code -- 14 --cluster --dry --date=2026-01-16
+ *
+ * Execute:
+ * pnpm migrate:deploy:code -- 14 --cluster --date=2026-01-16 --no-record
+ */
+
+const TMP_TABLE = 'events_tmp2';
+const EVENTS_TABLE = 'events';
+
+function parseArgs() {
+ const args = process.argv;
+ const dateArg = args.find((a: string) => a.startsWith('--date='));
+
+ if (!dateArg) {
+ console.error('Missing required --date=YYYY-MM-DD argument');
+ console.error(' Example: pnpm migrate:deploy:code -- 14 --cluster --date=2026-01-16');
+ process.exit(1);
+ }
+
+ const date = dateArg!.split('=')[1]!;
+
+ return {
+ date,
+ isCluster: getIsCluster(),
+ isDry: args.includes('--dry'),
+ };
+}
+
+export async function up() {
+ const { date, isDry } = parseArgs();
+
+ console.log('='.repeat(60));
+ console.log(' MOVE EVENTS TO EVENTS_TMP2');
+ console.log(` Date: ${date}`);
+ console.log(` Mode: ${isDry ? 'DRY RUN' : 'EXECUTE'}`);
+ console.log('='.repeat(60));
+
+ // Step 0: Check source counts
+ console.log(`\n[Step 0] events count for ${date}:`);
+ const srcResult = await chMigrationClient.query({
+ query: `
+ SELECT
+ name,
+ count() as total
+ FROM ${EVENTS_TABLE}
+ WHERE toDate(created_at) = '${date}'
+ GROUP BY name
+ ORDER BY total DESC`,
+ format: 'JSONEachRow',
+ });
+ const srcData = await srcResult.json<{ name: string; total: string }>();
+
+ let srcTotal = 0;
+ console.log(`\n ${'Event'.padEnd(25)} ${'Total'.padStart(10)}`);
+ console.log(' ' + '-'.repeat(37));
+ for (const row of srcData) {
+ const total = Number(row.total);
+ srcTotal += total;
+ console.log(` ${row.name.padEnd(25)} ${total.toLocaleString().padStart(10)}`);
+ }
+ console.log(' ' + '-'.repeat(37));
+ console.log(` ${'TOTAL'.padEnd(25)} ${srcTotal.toLocaleString().padStart(10)}`);
+
+ if (srcTotal === 0) {
+ console.log('\n No events found for this date!');
+ return;
+ }
+
+ if (isDry) {
+ console.log('\n[DRY RUN] SQL that would execute:');
+ console.log(`
+ INSERT INTO ${TMP_TABLE}
+ SELECT * FROM ${EVENTS_TABLE}
+ WHERE toDate(created_at) = '${date}'
+ SETTINGS max_memory_usage = 40000000000, max_execution_time = 18000;`);
+ return;
+ }
+
+ // Step 1: Copy data
+ console.log(`\n[Step 1] Copying ${srcTotal.toLocaleString()} events into ${TMP_TABLE}...`);
+ await runClickhouseMigrationCommands([
+ `INSERT INTO ${TMP_TABLE}
+ SELECT * FROM ${EVENTS_TABLE}
+ WHERE toDate(created_at) = '${date}'
+ SETTINGS
+ max_memory_usage = 40000000000,
+ max_execution_time = 18000`,
+ ]);
+
+ // Step 2: Verify
+ console.log(`\n[Step 2] Verifying events_tmp2 count for ${date}:`);
+ const dstResult = await chMigrationClient.query({
+ query: `
+ SELECT count() as total
+ FROM ${TMP_TABLE}
+ WHERE toDate(created_at) = '${date}'`,
+ format: 'JSONEachRow',
+ });
+ const dstData = await dstResult.json<{ total: string }>();
+ const dstTotal = Number(dstData[0]?.total ?? 0);
+
+ console.log(`\n events: ${srcTotal.toLocaleString()}`);
+ console.log(` events_tmp2: ${dstTotal.toLocaleString()}`);
+
+ console.log('\n' + '='.repeat(60));
+ console.log(' COPY COMPLETE');
+ console.log('='.repeat(60));
+}
+
+export async function down() {
+ console.log('No down migration');
+}
diff --git a/packages/db/code-migrations/15-validate-optimize-dedup.ts b/packages/db/code-migrations/15-validate-optimize-dedup.ts
new file mode 100644
index 000000000..6b019e896
--- /dev/null
+++ b/packages/db/code-migrations/15-validate-optimize-dedup.ts
@@ -0,0 +1,203 @@
+import {
+ chMigrationClient,
+} from '../src/clickhouse/migration';
+
+/**
+ * Validate OPTIMIZE TABLE DEDUPLICATE impact on events_tmp
+ *
+ * Shows duplicate counts per day and per date range (Jan 1-15 vs Jan 16-31)
+ * to prove that OPTIMIZE on partition 202601 won't affect Jan 16-31 data.
+ *
+ * Usage:
+ * Validate (default):
+ * pnpm migrate:deploy:code -- 15 --cluster --no-record
+ *
+ * Execute OPTIMIZE after validation:
+ * pnpm migrate:deploy:code -- 15 --cluster --no-record --execute
+ */
+
+const TMP_TABLE = 'events_tmp';
+
+const DEDUP_KEY =
+ 'project_id, name, device_id, profile_id, created_at, path';
+
+function parseArgs() {
+ const args = process.argv;
+ return {
+ shouldExecute: args.includes('--execute'),
+ };
+}
+
+export async function up() {
+ const { shouldExecute } = parseArgs();
+
+ console.log('='.repeat(60));
+ console.log(' OPTIMIZE DEDUPLICATE VALIDATION');
+ console.log(` Table: ${TMP_TABLE}`);
+ console.log(` Mode: ${shouldExecute ? 'VALIDATE + EXECUTE' : 'VALIDATE ONLY'}`);
+ console.log('='.repeat(60));
+
+ // Step 1: Per-day breakdown — total vs unique
+ console.log(`\n[Step 1] Per-day duplicate analysis for January 2026:`);
+ const perDayResult = await chMigrationClient.query({
+ query: `
+ SELECT
+ toDate(created_at) as date,
+ count() as total,
+ uniqExact(${DEDUP_KEY}) as unique_events,
+ count() - uniqExact(${DEDUP_KEY}) as duplicates
+ FROM ${TMP_TABLE}
+ WHERE toYYYYMM(created_at) = 202601
+ GROUP BY date
+ ORDER BY date ASC`,
+ format: 'JSONEachRow',
+ });
+ const perDayData = await perDayResult.json<{
+ date: string;
+ total: string;
+ unique_events: string;
+ duplicates: string;
+ }>();
+
+ console.log(
+ `\n ${'Date'.padEnd(14)} ${'Total'.padStart(12)} ${'Unique'.padStart(12)} ${'Duplicates'.padStart(12)} ${'Dup %'.padStart(8)}`,
+ );
+ console.log(' ' + '-'.repeat(60));
+
+ let grandTotal = 0;
+ let grandUnique = 0;
+ let grandDuplicates = 0;
+ for (const row of perDayData) {
+ const total = Number(row.total);
+ const unique = Number(row.unique_events);
+ const dups = Number(row.duplicates);
+ const dupPct = total > 0 ? ((dups / total) * 100).toFixed(1) : '0.0';
+ grandTotal += total;
+ grandUnique += unique;
+ grandDuplicates += dups;
+ console.log(
+ ` ${row.date.padEnd(14)} ${total.toLocaleString().padStart(12)} ${unique.toLocaleString().padStart(12)} ${dups.toLocaleString().padStart(12)} ${(dupPct + '%').padStart(8)}`,
+ );
+ }
+ console.log(' ' + '-'.repeat(60));
+ const grandDupPct =
+ grandTotal > 0
+ ? ((grandDuplicates / grandTotal) * 100).toFixed(1)
+ : '0.0';
+ console.log(
+ ` ${'TOTAL'.padEnd(14)} ${grandTotal.toLocaleString().padStart(12)} ${grandUnique.toLocaleString().padStart(12)} ${grandDuplicates.toLocaleString().padStart(12)} ${(grandDupPct + '%').padStart(8)}`,
+ );
+
+ // Step 2: Date range summary — Jan 1-15 vs Jan 16-31
+ console.log(`\n[Step 2] Date range summary (Jan 1-15 vs Jan 16-31):`);
+ const rangeResult = await chMigrationClient.query({
+ query: `
+ SELECT
+ if(toDate(created_at) <= '2026-01-15', 'Jan 01-15', 'Jan 16-31') as date_range,
+ count() as total,
+ uniqExact(${DEDUP_KEY}) as unique_events,
+ count() - uniqExact(${DEDUP_KEY}) as duplicates
+ FROM ${TMP_TABLE}
+ WHERE toYYYYMM(created_at) = 202601
+ GROUP BY date_range
+ ORDER BY date_range ASC`,
+ format: 'JSONEachRow',
+ });
+ const rangeData = await rangeResult.json<{
+ date_range: string;
+ total: string;
+ unique_events: string;
+ duplicates: string;
+ }>();
+
+ console.log(
+ `\n ${'Range'.padEnd(14)} ${'Total'.padStart(12)} ${'Unique'.padStart(12)} ${'Duplicates'.padStart(12)} ${'Dup %'.padStart(8)}`,
+ );
+ console.log(' ' + '-'.repeat(60));
+ for (const row of rangeData) {
+ const total = Number(row.total);
+ const unique = Number(row.unique_events);
+ const dups = Number(row.duplicates);
+ const dupPct = total > 0 ? ((dups / total) * 100).toFixed(1) : '0.0';
+ console.log(
+ ` ${row.date_range.padEnd(14)} ${total.toLocaleString().padStart(12)} ${unique.toLocaleString().padStart(12)} ${dups.toLocaleString().padStart(12)} ${(dupPct + '%').padStart(8)}`,
+ );
+ }
+
+ // Step 3: Safety check — verify Jan 16-31 has 0 duplicates
+ const jan16_31 = rangeData.find((r) => r.date_range === 'Jan 16-31');
+ const jan16_31_dups = jan16_31 ? Number(jan16_31.duplicates) : 0;
+
+ if (jan16_31_dups > 0) {
+ console.log(`\n ⚠ WARNING: Jan 16-31 has ${jan16_31_dups.toLocaleString()} duplicates!`);
+ console.log(' OPTIMIZE DEDUPLICATE on partition 202601 WILL affect Jan 16-31 data.');
+ console.log(' Review the per-day breakdown above before proceeding.');
+ } else {
+ console.log(`\n ✓ Jan 16-31 has 0 duplicates — OPTIMIZE will NOT affect it.`);
+ }
+
+ if (!shouldExecute) {
+ console.log('\n[VALIDATE ONLY] To execute OPTIMIZE, re-run with --execute flag:');
+ console.log(
+ ' pnpm migrate:deploy:code -- 15 --cluster --no-record --execute',
+ );
+ return;
+ }
+
+ // Step 4: Execute OPTIMIZE DEDUPLICATE
+ console.log(`\n[Step 3] Running OPTIMIZE TABLE DEDUPLICATE on partition 202601...`);
+ console.log(' This may take several minutes...');
+
+ await chMigrationClient.query({
+ query: `
+ OPTIMIZE TABLE ${TMP_TABLE}
+ PARTITION '202601'
+ FINAL
+ DEDUPLICATE BY ${DEDUP_KEY}`,
+ });
+
+ console.log(' OPTIMIZE complete!');
+
+ // Step 5: Verify after OPTIMIZE
+ console.log(`\n[Step 4] Post-OPTIMIZE verification:`);
+ const afterResult = await chMigrationClient.query({
+ query: `
+ SELECT
+ if(toDate(created_at) <= '2026-01-15', 'Jan 01-15', 'Jan 16-31') as date_range,
+ count() as total,
+ uniqExact(${DEDUP_KEY}) as unique_events,
+ count() - uniqExact(${DEDUP_KEY}) as duplicates
+ FROM ${TMP_TABLE}
+ WHERE toYYYYMM(created_at) = 202601
+ GROUP BY date_range
+ ORDER BY date_range ASC`,
+ format: 'JSONEachRow',
+ });
+ const afterData = await afterResult.json<{
+ date_range: string;
+ total: string;
+ unique_events: string;
+ duplicates: string;
+ }>();
+
+ console.log(
+ `\n ${'Range'.padEnd(14)} ${'Total'.padStart(12)} ${'Unique'.padStart(12)} ${'Duplicates'.padStart(12)}`,
+ );
+ console.log(' ' + '-'.repeat(52));
+ for (const row of afterData) {
+ const total = Number(row.total);
+ const unique = Number(row.unique_events);
+ const dups = Number(row.duplicates);
+ console.log(
+ ` ${row.date_range.padEnd(14)} ${total.toLocaleString().padStart(12)} ${unique.toLocaleString().padStart(12)} ${dups.toLocaleString().padStart(12)}`,
+ );
+ }
+
+ console.log('\n' + '='.repeat(60));
+ console.log(' OPTIMIZE DEDUPLICATE COMPLETE');
+ console.log('='.repeat(60));
+}
+
+export async function down() {
+ console.log('No down migration');
+}
diff --git a/packages/db/code-migrations/16-dedup-tmp-to-tmp2.ts b/packages/db/code-migrations/16-dedup-tmp-to-tmp2.ts
new file mode 100644
index 000000000..3fa7efe1e
--- /dev/null
+++ b/packages/db/code-migrations/16-dedup-tmp-to-tmp2.ts
@@ -0,0 +1,132 @@
+import {
+ chMigrationClient,
+ runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+/**
+ * Move events from events_tmp to events_tmp2 with dedup
+ *
+ * Reads from events_tmp, deduplicates using LIMIT 1 BY, writes to events_tmp2.
+ * After all days are done, REPLACE PARTITION from events_tmp2 into events.
+ *
+ * Usage:
+ * Dry run:
+ * pnpm migrate:deploy:code -- 16 --cluster --dry --date=2026-01-01 --no-record
+ *
+ * Execute:
+ * pnpm migrate:deploy:code -- 16 --cluster --date=2026-01-01 --no-record
+ */
+
+const DEDUP_KEY =
+ 'project_id, name, device_id, profile_id, session_id, created_at, path, properties';
+
+const SRC_TABLE = 'events_tmp';
+const DST_TABLE = 'events_tmp2';
+
+function parseArgs() {
+ const args = process.argv;
+ const dateArg = args.find((a: string) => a.startsWith('--date='));
+
+ if (!dateArg) {
+ console.error('Missing required --date=YYYY-MM-DD argument');
+ console.error(' Example: pnpm migrate:deploy:code -- 16 --cluster --date=2026-01-01 --no-record');
+ process.exit(1);
+ }
+
+ const date = dateArg!.split('=')[1]!;
+
+ return {
+ date,
+ isCluster: getIsCluster(),
+ isDry: args.includes('--dry'),
+ };
+}
+
+export async function up() {
+ const { date, isDry } = parseArgs();
+
+ console.log('='.repeat(60));
+ console.log(' DEDUP: events_tmp → events_tmp2');
+ console.log(` Date: ${date}`);
+ console.log(` Mode: ${isDry ? 'DRY RUN' : 'EXECUTE'}`);
+ console.log('='.repeat(60));
+
+ // Step 0: Show current counts in events_tmp for this date
+ console.log(`\n[Step 0] events_tmp counts for ${date}:`);
+ const beforeResult = await chMigrationClient.query({
+ query: `
+ SELECT
+ count() as total,
+ uniq(${DEDUP_KEY}) as unique_events,
+ count() - uniq(${DEDUP_KEY}) as duplicates
+ FROM ${SRC_TABLE}
+ WHERE toDate(created_at) = '${date}'`,
+ format: 'JSONEachRow',
+ });
+ const beforeData = await beforeResult.json<{
+ total: string;
+ unique_events: string;
+ duplicates: string;
+ }>();
+
+ const srcTotal = Number(beforeData[0]?.total ?? 0);
+ const srcUnique = Number(beforeData[0]?.unique_events ?? 0);
+ const srcDups = Number(beforeData[0]?.duplicates ?? 0);
+
+ console.log(`\n Total: ${srcTotal.toLocaleString()}`);
+ console.log(` Unique: ${srcUnique.toLocaleString()}`);
+ console.log(` Duplicates: ${srcDups.toLocaleString()}`);
+
+ if (srcTotal === 0) {
+ console.log('\n No events found for this date!');
+ return;
+ }
+
+ if (isDry) {
+ console.log('\n[DRY RUN] SQL that would execute:');
+ console.log(`
+ INSERT INTO ${DST_TABLE}
+ SELECT * FROM ${SRC_TABLE}
+ WHERE toDate(created_at) = '${date}'
+ LIMIT 1 BY ${DEDUP_KEY}
+ SETTINGS max_memory_usage = 40000000000, max_execution_time = 18000;`);
+ return;
+ }
+
+ // Step 1: Insert deduplicated data
+ console.log(`\n[Step 1] Inserting deduped ${date} into ${DST_TABLE}...`);
+ await runClickhouseMigrationCommands([
+ `INSERT INTO ${DST_TABLE}
+ SELECT * FROM ${SRC_TABLE}
+ WHERE toDate(created_at) = '${date}'
+ LIMIT 1 BY ${DEDUP_KEY}
+ SETTINGS
+ max_memory_usage = 40000000000,
+ max_execution_time = 18000`,
+ ]);
+
+ // Step 2: Verify counts
+ console.log(`\n[Step 2] Verifying ${DST_TABLE} count for ${date}:`);
+ const afterResult = await chMigrationClient.query({
+ query: `
+ SELECT count() as total
+ FROM ${DST_TABLE}
+ WHERE toDate(created_at) = '${date}'`,
+ format: 'JSONEachRow',
+ });
+ const afterData = await afterResult.json<{ total: string }>();
+ const dstTotal = Number(afterData[0]?.total ?? 0);
+
+ console.log(`\n events_tmp: ${srcTotal.toLocaleString()}`);
+ console.log(` events_tmp2: ${dstTotal.toLocaleString()}`);
+ console.log(` Removed: ${(srcTotal - dstTotal).toLocaleString()}`);
+
+ console.log('\n' + '='.repeat(60));
+ console.log(' DEDUP COMPLETE');
+ console.log('='.repeat(60));
+}
+
+export async function down() {
+ console.log('No down migration');
+}
diff --git a/packages/db/code-migrations/4-cohorts.ts b/packages/db/code-migrations/4-cohorts.ts
new file mode 100644
index 000000000..9c09d650f
--- /dev/null
+++ b/packages/db/code-migrations/4-cohorts.ts
@@ -0,0 +1,129 @@
+import {
+ createMaterializedView,
+ createTable,
+ dropTable,
+ getExistingTables,
+ runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+export async function up() {
+ const replicatedVersion = '1';
+ const existingTables = await getExistingTables();
+ const isClustered = getIsCluster();
+
+ const sqls: string[] = [];
+
+ // 1. Create cohort_members table
+ // Stores which profile_ids belong to which cohorts
+ // Uses ReplacingMergeTree for efficient updates (version-based deduplication)
+ if (!existingTables.includes('cohort_members_distributed') && !existingTables.includes('cohort_members')) {
+ sqls.push(
+ ...createTable({
+ name: 'cohort_members',
+ columns: [
+ 'project_id String CODEC(ZSTD(3))',
+ 'cohort_id String CODEC(ZSTD(3))',
+ 'profile_id String CODEC(ZSTD(3))',
+ 'matched_at DateTime DEFAULT now()',
+ 'matching_properties Map(String, String) CODEC(ZSTD(3))',
+ 'version UInt64 DEFAULT 1',
+ ],
+ indices: [
+ 'INDEX idx_profile profile_id TYPE bloom_filter GRANULARITY 1',
+ 'INDEX idx_cohort cohort_id TYPE bloom_filter GRANULARITY 1',
+ ],
+ engine: 'ReplacingMergeTree(version)',
+ orderBy: ['project_id', 'cohort_id', 'profile_id'],
+ partitionBy: 'toYYYYMM(matched_at)',
+ settings: {
+ index_granularity: 8192,
+ },
+ distributionHash: 'cityHash64(project_id, cohort_id)',
+ replicatedVersion,
+ isClustered,
+ }),
+ );
+ }
+
+ // 2. Create cohort_metadata table
+ // Caches cohort sizes and sample profile IDs for fast lookups
+ if (!existingTables.includes('cohort_metadata_distributed') && !existingTables.includes('cohort_metadata')) {
+ sqls.push(
+ ...createTable({
+ name: 'cohort_metadata',
+ columns: [
+ 'project_id String',
+ 'cohort_id String',
+ 'member_count UInt64',
+ 'last_computed_at DateTime',
+ 'sample_profiles Array(String)',
+ 'version UInt64 DEFAULT 1',
+ ],
+ engine: 'ReplacingMergeTree(version)',
+ orderBy: ['project_id', 'cohort_id'],
+ settings: {
+ index_granularity: 8192,
+ },
+ distributionHash: 'cityHash64(project_id, cohort_id)',
+ replicatedVersion,
+ isClustered,
+ }),
+ );
+ }
+
+ // 3. Create profile_event_summary_mv materialized view
+ // Aggregates events by profile for fast cohort queries
+ // NOTE: This is different from cohort_events_mv (which is used for retention analysis)
+ // This MV is specifically for cohort computation with frequency filters
+ // populate: false - will build incrementally from new events to avoid OOM
+ if (!existingTables.includes('profile_event_summary_mv_distributed') && !existingTables.includes('profile_event_summary_mv')) {
+ sqls.push(
+ ...createMaterializedView({
+ name: 'profile_event_summary_mv',
+ tableName: 'events',
+ engine: 'AggregatingMergeTree()',
+ orderBy: ['project_id', 'profile_id', 'name', 'event_date'],
+ partitionBy: 'toYYYYMM(event_date)',
+ query: `SELECT
+ project_id,
+ profile_id,
+ name,
+ toStartOfDay(created_at) AS event_date,
+ countState() AS event_count,
+ minState(created_at) AS first_event_time,
+ maxState(created_at) AS last_event_time,
+ sumState(duration) AS total_duration
+ FROM {events}
+ WHERE profile_id != device_id
+ GROUP BY project_id, profile_id, name, event_date`,
+ distributionHash: 'cityHash64(project_id, profile_id)',
+ replicatedVersion,
+ isClustered,
+ populate: false, // Don't populate historical data - build incrementally
+ }),
+ );
+ }
+
+ await runClickhouseMigrationCommands(sqls);
+}
+
+export async function down() {
+ // Down migration is not typically used in production
+ // But included for development/testing rollback
+ const isClustered = getIsCluster();
+
+ const sqls = [
+ dropTable('profile_event_summary_mv_distributed', isClustered),
+ dropTable('profile_event_summary_mv_replicated', isClustered),
+ dropTable('profile_event_summary_mv', isClustered),
+ dropTable('cohort_metadata_distributed', isClustered),
+ dropTable('cohort_metadata_replicated', isClustered),
+ dropTable('cohort_metadata', isClustered),
+ dropTable('cohort_members_distributed', isClustered),
+ dropTable('cohort_members_replicated', isClustered),
+ dropTable('cohort_members', isClustered),
+ ];
+
+ await runClickhouseMigrationCommands(sqls);
+}
diff --git a/packages/db/code-migrations/6-add-imports-table-v2.ts b/packages/db/code-migrations/6-add-imports-table-v2.ts
new file mode 100644
index 000000000..2689a9706
--- /dev/null
+++ b/packages/db/code-migrations/6-add-imports-table-v2.ts
@@ -0,0 +1,90 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { TABLE_NAMES } from '../src/clickhouse/client';
+import {
+ createTable,
+ modifyTTL,
+ runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+export async function up() {
+ const isClustered = getIsCluster();
+
+ const sqls: string[] = [
+ ...createTable({
+ name: 'events_imports_v2',
+ columns: [
+ // Same columns as events table
+ '`id` UUID DEFAULT generateUUIDv4()',
+ '`name` LowCardinality(String)',
+ '`sdk_name` LowCardinality(String)',
+ '`sdk_version` LowCardinality(String)',
+ '`device_id` String CODEC(ZSTD(3))',
+ '`profile_id` String CODEC(ZSTD(3))',
+ '`project_id` String CODEC(ZSTD(3))',
+ '`session_id` String CODEC(LZ4)',
+ '`path` String CODEC(ZSTD(3))',
+ '`origin` String CODEC(ZSTD(3))',
+ '`referrer` String CODEC(ZSTD(3))',
+ '`referrer_name` String CODEC(ZSTD(3))',
+ '`referrer_type` LowCardinality(String)',
+ '`duration` UInt64 CODEC(Delta(4), LZ4)',
+ '`properties` Map(String, String) CODEC(ZSTD(3))',
+ '`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',
+ '`country` LowCardinality(FixedString(2))',
+ '`city` String',
+ '`region` LowCardinality(String)',
+ '`longitude` Nullable(Float32) CODEC(Gorilla, LZ4)',
+ '`latitude` Nullable(Float32) CODEC(Gorilla, LZ4)',
+ '`os` LowCardinality(String)',
+ '`os_version` LowCardinality(String)',
+ '`browser` LowCardinality(String)',
+ '`browser_version` LowCardinality(String)',
+ '`device` LowCardinality(String)',
+ '`brand` LowCardinality(String)',
+ '`model` LowCardinality(String)',
+ '`imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4)',
+
+ // Additional metadata columns for import tracking
+ '`import_id` String CODEC(ZSTD(3))',
+ "`import_status` LowCardinality(String) DEFAULT 'pending'",
+ '`imported_at_meta` DateTime DEFAULT now()',
+ ],
+ orderBy: ['created_at', 'name'], // Changed: Removed import_id from ORDER BY since it's now in partition key
+ partitionBy: 'import_id', // Changed: Was toYYYYMM(imported_at_meta) - now partition by import_id for better memory management
+ settings: {
+ index_granularity: 8192,
+ },
+ distributionHash: 'cityHash64(import_id)',
+ replicatedVersion: '2', // Version 2
+ isClustered,
+ }),
+ ];
+
+ // Add TTL policy for auto-cleanup after 7 days
+ sqls.push(
+ modifyTTL({
+ tableName: 'events_imports_v2',
+ isClustered,
+ ttl: 'imported_at_meta + INTERVAL 7 DAY',
+ }),
+ );
+
+ fs.writeFileSync(
+ path.join(__filename.replace('.ts', '.sql')),
+ sqls
+ .map((sql) =>
+ sql
+ .trim()
+ .replace(/;$/, '')
+ .replace(/\n{2,}/g, '\n')
+ .concat(';'),
+ )
+ .join('\n\n---\n\n'),
+ );
+
+ if (!process.argv.includes('--dry')) {
+ await runClickhouseMigrationCommands(sqls);
+ }
+}
diff --git a/packages/db/code-migrations/9-events-daily-stats.ts b/packages/db/code-migrations/9-events-daily-stats.ts
new file mode 100644
index 000000000..c295dfc51
--- /dev/null
+++ b/packages/db/code-migrations/9-events-daily-stats.ts
@@ -0,0 +1,270 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import {
+ chMigrationClient,
+ createMaterializedView,
+ moveDataBetweenTables,
+ runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+/**
+ * Migration 9: Create events_daily_stats to include ALL events
+ *
+ * Prerequisites: Drop existing events_daily_stats before running this migration
+ *
+ * This migration creates events_daily_stats MV that includes ALL events for complete analytics.
+ * Previous version excluded session_start, session_end, and screen_view events.
+ *
+ * Steps:
+ * 1. Optionally delete existing data up to a certain date
+ * 2. Optionally create events_daily_stats without POPULATE (starts capturing new data)
+ * 3. Backfill historical data from events table using day-by-day batch inserts
+ *
+ * Usage:
+ * # Full migration (create MV + backfill all data)
+ * pnpm tsx packages/db/code-migrations/9-events-daily-stats.ts
+ *
+ * # Backfill specific date range, skip MV creation, delete old data
+ * pnpm tsx packages/db/code-migrations/9-events-daily-stats.ts --start 2024-01-01 --end 2026-01-27 --delete-till 2026-01-28 --skip-mv
+ *
+ * # Backfill without delete
+ * pnpm tsx packages/db/code-migrations/9-events-daily-stats.ts --start 2024-01-01 --end 2026-01-27 --skip-delete --skip-mv
+ *
+ * # Dry run to see what will be executed
+ * pnpm tsx packages/db/code-migrations/9-events-daily-stats.ts --start 2024-01-01 --end 2026-01-27 --dry
+ */
+export async function up() {
+ const isClustered = getIsCluster();
+ const sqls: string[] = [];
+
+ // Parse command line arguments
+ const args = process.argv;
+ const startDateArg = args.find((arg, idx) => args[idx - 1] === '--start');
+ const endDateArg = args.find((arg, idx) => args[idx - 1] === '--end');
+ const deleteDateArg = args.find((arg, idx) => args[idx - 1] === '--delete-till');
+ const skipDelete = args.includes('--skip-delete');
+ const skipMv = args.includes('--skip-mv');
+
+ const targetTable = isClustered ? 'events_daily_stats_replicated' : 'events_daily_stats';
+
+ // Step 1: Delete existing data in batches (optional)
+ // Only delete dates in the backfill range, not everything up to delete-till date
+ if (!skipDelete && deleteDateArg) {
+ console.log(`🗑️ Preparing to delete data for backfill range (day-by-day)`);
+ console.log('');
+ }
+
+ // Step 2: Create MV without POPULATE (so it starts capturing new data immediately)
+ if (!skipMv) {
+ const mvStatements = createMaterializedView({
+ name: 'events_daily_stats',
+ tableName: 'events',
+ orderBy: ['project_id', 'name', 'date'],
+ partitionBy: 'toYYYYMM(date)',
+ query: `SELECT
+ project_id,
+ name,
+ toDate(created_at) as date,
+ uniqState(profile_id) as unique_profiles_state,
+ uniqState(session_id) as unique_sessions_state,
+ countState() as event_count
+ FROM {events}
+ GROUP BY project_id, name, date`,
+ distributionHash: 'cityHash64(project_id, name, date)',
+ replicatedVersion: '1',
+ isClustered,
+ populate: false, // Don't use POPULATE to avoid timeout
+ });
+
+ sqls.push(...mvStatements);
+ }
+
+ // Step 3: Backfill historical data from events table
+ // First, check the actual date range in the events table
+ const checkDataQuery = await chMigrationClient.query({
+ query: `
+ SELECT
+ min(toDate(created_at)) as min_date,
+ max(toDate(created_at)) as max_date,
+ count() as total_events
+ FROM events
+ `,
+ format: 'JSONEachRow',
+ });
+
+ const dataRange = await checkDataQuery.json<{
+ min_date: string;
+ max_date: string;
+ total_events: string;
+ }>();
+
+ if (dataRange[0]?.min_date && dataRange[0]?.max_date) {
+ // Determine start and end dates based on arguments or data range
+ let startDate: Date;
+ let endDate: Date;
+
+ if (startDateArg) {
+ startDate = new Date(startDateArg + 'T00:00:00Z'); // Use UTC
+ } else {
+ startDate = new Date(dataRange[0].min_date + 'T00:00:00Z');
+ }
+
+ if (endDateArg) {
+ endDate = new Date(endDateArg + 'T00:00:00Z'); // Use UTC
+ } else {
+ // If no end date specified, use max date from events table
+ // But typically you'd want yesterday to avoid conflicts with MV
+ endDate = new Date(dataRange[0].max_date + 'T00:00:00Z');
+ }
+
+ // Validate date range
+ if (startDate > endDate) {
+ console.error('❌ Error: Start date must be before or equal to end date');
+ console.error(` Start: ${startDate.toISOString().split('T')[0]}`);
+ console.error(` End: ${endDate.toISOString().split('T')[0]}`);
+ process.exit(1);
+ }
+
+ const totalEvents = Number(dataRange[0].total_events);
+ const backfillDays = Math.ceil((endDate.getTime() - startDate.getTime()) / (1000 * 60 * 60 * 24)) + 1;
+
+ // Generate DELETE statements for the backfill range (if delete not skipped)
+ const deleteStartIndex = sqls.length; // Track where deletes start
+ if (!skipDelete && deleteDateArg) {
+ let currentDeleteDate = new Date(startDate);
+ while (currentDeleteDate <= endDate) {
+ const dateStr = currentDeleteDate.toISOString().split('T')[0];
+ const deleteSql = `ALTER TABLE ${targetTable} DELETE WHERE date = toDate('${dateStr}')`;
+ sqls.push(deleteSql);
+ currentDeleteDate.setDate(currentDeleteDate.getDate() + 1);
+ }
+ }
+ const deleteOps = sqls.length - deleteStartIndex;
+
+ // Show the plan AFTER generating deletes
+ console.log('========================================');
+ console.log('📊 Backfill Plan:');
+ console.log(` Target Table: ${targetTable}`);
+ console.log(` Delete Ops: ${deleteOps > 0 ? `${deleteOps} days` : 'None'}`);
+ console.log(` Create MV: ${skipMv ? 'Skipped' : 'Yes'}`);
+ console.log(` Backfill Start: ${startDate.toISOString().split('T')[0]}`);
+ console.log(` Backfill End: ${endDate.toISOString().split('T')[0]} (inclusive)`);
+ console.log(` Backfill Days: ${backfillDays} days`);
+ console.log(` Chunking: Hourly (24 chunks per day)`);
+ console.log(` Total Chunks: ${backfillDays * 24} hourly inserts`);
+ console.log(` Total Events: ${totalEvents.toLocaleString()} in events table`);
+ console.log('========================================');
+ console.log('');
+
+ // Generate INSERT statements chunked by hour to reduce memory usage
+ // Process each day in 24 hourly chunks to avoid ClickHouse memory limits
+ const backfillSqls: string[] = [];
+
+ let currentDate = new Date(startDate); // Start from startDate (go forward)
+
+ while (currentDate <= endDate) {
+ const dateStr = currentDate.toISOString().split('T')[0];
+
+ // Process each day in 24 hourly chunks
+ for (let hour = 0; hour < 24; hour++) {
+ const hourStart = `${dateStr} ${String(hour).padStart(2, '0')}:00:00`;
+ const hourEnd = `${dateStr} ${String(hour).padStart(2, '0')}:59:59`;
+
+ const sql = `INSERT INTO ${targetTable}
+ SELECT
+ project_id,
+ name,
+ toDate(created_at) as date,
+ uniqState(profile_id) as unique_profiles_state,
+ uniqState(session_id) as unique_sessions_state,
+ countState() as event_count
+ FROM events
+ WHERE created_at >= toDateTime('${hourStart}')
+ AND created_at <= toDateTime('${hourEnd}')
+ GROUP BY project_id, name, date`;
+
+ backfillSqls.push(sql);
+ }
+
+ currentDate.setDate(currentDate.getDate() + 1);
+ }
+
+ sqls.push(...backfillSqls);
+
+ // Add OPTIMIZE TABLE to merge hourly chunks into daily aggregates
+ const optimizeSql = `OPTIMIZE TABLE ${targetTable} FINAL`;
+ sqls.push(optimizeSql);
+ console.log('Added OPTIMIZE TABLE command to merge hourly chunks');
+ console.log('');
+ } else {
+ console.log('No data found in the specified date range, skipping backfill');
+ }
+
+ // Write SQL to file for review
+ const sqlFilePath = path.join(__filename.replace('.ts', '.sql').replace('.js', '.sql'));
+ fs.writeFileSync(
+ sqlFilePath,
+ sqls
+ .map((sql) =>
+ sql
+ .trim()
+ .replace(/;$/, '')
+ .replace(/\n{2,}/g, '\n')
+ .concat(';'),
+ )
+ .join('\n\n---\n\n'),
+ );
+
+ // Count operation types
+ const deleteOpsTotal = sqls.filter(sql => sql.includes('DELETE')).length;
+ const mvOpsTotal = sqls.filter(sql => sql.includes('CREATE') || sql.includes('MATERIALIZED VIEW')).length;
+ const insertOpsTotal = sqls.filter(sql => sql.includes('INSERT')).length;
+
+ const optimizeOpsTotal = sqls.filter(sql => sql.includes('OPTIMIZE')).length;
+
+ console.log(`Generated ${sqls.length} SQL statements:`);
+ console.log(` - ${deleteOpsTotal} DELETE operations (day-by-day)`);
+ console.log(` - ${mvOpsTotal} MV creation operations`);
+ console.log(` - ${insertOpsTotal} INSERT operations (hourly chunks)`);
+ console.log(` - ${optimizeOpsTotal} OPTIMIZE operations (merge chunks)`);
+ console.log(`SQL written to: ${sqlFilePath}`);
+ console.log('');
+
+ // Execute if not in dry-run mode
+ if (!process.argv.includes('--dry')) {
+ console.log('🚀 Executing migration...');
+ console.log('');
+
+ // Execute with progress tracking
+ let completed = 0;
+ const total = sqls.length;
+
+ for (const sql of sqls) {
+ await runClickhouseMigrationCommands([sql]);
+ completed++;
+
+ // Show progress every 100 queries, every 10% or on last query
+ const shouldLog = completed % 100 === 0 ||
+ completed === total ||
+ Math.floor((completed / total) * 10) > Math.floor(((completed - 1) / total) * 10);
+
+ if (shouldLog) {
+ const percentage = ((completed / total) * 100).toFixed(1);
+ console.log(`Progress: ${completed}/${total} (${percentage}%)`);
+ }
+ }
+
+ console.log('');
+ console.log('✅ Migration completed successfully!');
+ console.log('');
+ console.log('Next steps:');
+ console.log('1. Verify data in events_daily_stats:');
+ console.log(` SELECT date, count() FROM ${targetTable} GROUP BY date ORDER BY date`);
+ console.log('2. Check a few sample queries to ensure accuracy');
+ } else {
+ console.log('🔍 Dry-run mode: SQL generated but not executed');
+ console.log('');
+ console.log('To execute, run without --dry flag');
+ }
+}
diff --git a/packages/db/index.ts b/packages/db/index.ts
index 58042d3f3..53418ed0f 100644
--- a/packages/db/index.ts
+++ b/packages/db/index.ts
@@ -29,3 +29,6 @@ export * from './src/clickhouse/query-builder';
export * from './src/services/import.service';
export * from './src/services/overview.service';
export * from './src/session-context';
+export * from './src/services/materialize-columns.service';
+export * from './src/services/cohort.service';
+export * from './src/services/custom-event.service';
diff --git a/packages/db/package.json b/packages/db/package.json
index fb4cd393e..86f9f89e2 100644
--- a/packages/db/package.json
+++ b/packages/db/package.json
@@ -9,6 +9,7 @@
"migrate:deploy:code": "pnpm with-env jiti ./code-migrations/migrate.ts",
"migrate:deploy:db": "pnpm with-env prisma migrate deploy",
"migrate:deploy": "pnpm migrate:deploy:db && pnpm migrate:deploy:code",
+ "materialize:analyze": "pnpm with-env jiti ./src/cli/materialize.ts",
"typecheck": "tsc --noEmit",
"with-env": "dotenv -e ../../.env -c --"
},
diff --git a/packages/db/prisma/migrations/20260124115911_add_cohorts_table/migration.sql b/packages/db/prisma/migrations/20260124115911_add_cohorts_table/migration.sql
new file mode 100644
index 000000000..d0c2ea4f0
--- /dev/null
+++ b/packages/db/prisma/migrations/20260124115911_add_cohorts_table/migration.sql
@@ -0,0 +1,22 @@
+-- CreateTable
+CREATE TABLE "cohorts" (
+ "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+ "name" TEXT NOT NULL,
+ "description" TEXT,
+ "projectId" TEXT NOT NULL,
+ "definition" JSONB NOT NULL,
+ "isStatic" BOOLEAN NOT NULL DEFAULT false,
+ "computeOnDemand" BOOLEAN NOT NULL DEFAULT false,
+ "profileCount" INTEGER NOT NULL DEFAULT 0,
+ "lastComputedAt" TIMESTAMP(3),
+ "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+ CONSTRAINT "cohorts_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "cohorts_projectId_idx" ON "cohorts"("projectId");
+
+-- AddForeignKey
+ALTER TABLE "cohorts" ADD CONSTRAINT "cohorts_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
diff --git a/packages/db/prisma/migrations/20260226120000_add_last_notified_at_to_notification_rules/migration.sql b/packages/db/prisma/migrations/20260226120000_add_last_notified_at_to_notification_rules/migration.sql
new file mode 100644
index 000000000..849aaafa0
--- /dev/null
+++ b/packages/db/prisma/migrations/20260226120000_add_last_notified_at_to_notification_rules/migration.sql
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "notification_rules" ADD COLUMN "lastNotifiedAt" TIMESTAMP(3);
diff --git a/packages/db/prisma/migrations/20260304120000_add_global_filters_to_reports/migration.sql b/packages/db/prisma/migrations/20260304120000_add_global_filters_to_reports/migration.sql
new file mode 100644
index 000000000..115a095a8
--- /dev/null
+++ b/packages/db/prisma/migrations/20260304120000_add_global_filters_to_reports/migration.sql
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "reports" ADD COLUMN "globalFilters" JSONB NOT NULL DEFAULT '[]';
diff --git a/packages/db/prisma/migrations/20260305120000_add_hold_properties_to_reports/migration.sql b/packages/db/prisma/migrations/20260305120000_add_hold_properties_to_reports/migration.sql
new file mode 100644
index 000000000..52dd9fc92
--- /dev/null
+++ b/packages/db/prisma/migrations/20260305120000_add_hold_properties_to_reports/migration.sql
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "reports" ADD COLUMN "holdProperties" JSONB NOT NULL DEFAULT '[]';
diff --git a/packages/db/prisma/migrations/20260310120000_add_measuring_to_reports/migration.sql b/packages/db/prisma/migrations/20260310120000_add_measuring_to_reports/migration.sql
new file mode 100644
index 000000000..7d8abcdea
--- /dev/null
+++ b/packages/db/prisma/migrations/20260310120000_add_measuring_to_reports/migration.sql
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "reports" ADD COLUMN "measuring" TEXT;
diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma
index 38a29105d..a3d6285bb 100644
--- a/packages/db/prisma/schema.prisma
+++ b/packages/db/prisma/schema.prisma
@@ -196,6 +196,8 @@ model Project {
notificationRules NotificationRule[]
notifications Notification[]
imports Import[]
+ cohorts Cohort[]
+ customEvents CustomEvent[]
// When deleteAt > now(), the project will be deleted
deleteAt DateTime?
@@ -320,7 +322,10 @@ model Report {
previous Boolean @default(false)
criteria String?
funnelGroup String?
- funnelWindow Float?
+ funnelWindow Float?
+ globalFilters Json @default("[]")
+ holdProperties Json @default("[]")
+ measuring String?
dashboardId String
dashboard Dashboard @relation(fields: [dashboardId], references: [id], onDelete: Cascade)
@@ -385,6 +390,27 @@ model EventMeta {
@@map("event_meta")
}
+model CustomEvent {
+ id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
+ name String
+ description String?
+ projectId String
+ project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+ // Custom event definition (JSON structure)
+ /// [IPrismaCustomEventDefinition]
+ definition Json
+
+ // Mark as conversion event for filtering
+ conversion Boolean @default(false)
+
+ createdAt DateTime @default(now())
+ updatedAt DateTime @default(now()) @updatedAt
+
+ @@unique([name, projectId])
+ @@map("custom_events")
+}
+
model Reference {
id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
title String
@@ -414,11 +440,12 @@ model NotificationRule {
sendToApp Boolean @default(false)
sendToEmail Boolean @default(false)
/// [IPrismaNotificationRuleConfig]
- config Json
- template String?
- createdAt DateTime @default(now())
- updatedAt DateTime @default(now()) @updatedAt
- notifications Notification[]
+ config Json
+ template String?
+ lastNotifiedAt DateTime?
+ createdAt DateTime @default(now())
+ updatedAt DateTime @default(now()) @updatedAt
+ notifications Notification[]
@@map("notification_rules")
}
@@ -497,3 +524,45 @@ model Import {
@@map("imports")
}
+
+model MaterializedColumn {
+ id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
+ targetTable String @default("events") // "events" | "profiles"
+ propertyKey String // e.g., "utm_source" (without "properties." prefix)
+ columnName String // e.g., "utm_source" (usually same as propertyKey)
+ cardinality Int // Number of unique values
+ usageCount Int // How many reports use this property
+ benefitScore Float // Calculated benefit score
+ estimatedSize BigInt // Estimated storage cost in bytes
+ status String // "active", "pending", "failed"
+ createdAt DateTime @default(now())
+ materializedAt DateTime? // When ALTER TABLE was executed
+ lastAnalyzedAt DateTime @default(now()) @updatedAt
+
+ @@unique([targetTable, propertyKey])
+ @@map("materialized_columns")
+}
+
+model Cohort {
+ id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
+ name String
+ description String?
+ projectId String
+ project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+ // Cohort definition (JSON structure)
+ /// [IPrismaCohortDefinition]
+ definition Json
+
+ // Type and metadata
+ isStatic Boolean @default(false) // Static vs Dynamic
+ profileCount Int @default(0) // Cached count
+ lastComputedAt DateTime? // When last computed
+ computeOnDemand Boolean @default(false) // Don't store, compute when needed
+
+ createdAt DateTime @default(now())
+ updatedAt DateTime @default(now()) @updatedAt
+
+ @@map("cohorts")
+ @@index([projectId])
+}
diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.ts
index cc8421004..27a8e807a 100644
--- a/packages/db/src/buffers/base-buffer.ts
+++ b/packages/db/src/buffers/base-buffer.ts
@@ -1,7 +1,7 @@
import { generateSecureId } from '@openpanel/common/server';
import { type ILogger, createLogger } from '@openpanel/logger';
import { cronQueue } from '@openpanel/queue';
-import { getRedisCache, runEvery } from '@openpanel/redis';
+import { type ExtendedRedis, getRedisCache, runEvery } from '@openpanel/redis';
export class BaseBuffer {
name: string;
@@ -12,11 +12,15 @@ export class BaseBuffer {
enableParallelProcessing: boolean;
protected bufferCounterKey: string;
+ protected redis: ExtendedRedis;
constructor(options: {
name: string;
onFlush: () => Promise;
enableParallelProcessing?: boolean;
+ // Pass a dedicated Redis client to isolate this buffer from the general cache.
+ // Falls back to getRedisCache() so existing buffers need no changes.
+ redis?: ExtendedRedis;
}) {
this.logger = createLogger({ name: options.name });
this.name = options.name;
@@ -24,6 +28,7 @@ export class BaseBuffer {
this.onFlush = options.onFlush;
this.bufferCounterKey = `${this.name}:buffer:count`;
this.enableParallelProcessing = options.enableParallelProcessing ?? false;
+ this.redis = options.redis ?? getRedisCache();
}
protected chunks(items: T[], size: number) {
@@ -48,14 +53,14 @@ export class BaseBuffer {
fn: async () => {
try {
const actual = await fallbackFn();
- await getRedisCache().set(this.bufferCounterKey, actual.toString());
+ await this.redis.set(this.bufferCounterKey, actual.toString());
} catch (error) {
this.logger.warn('Failed to resync buffer counter', { error });
}
},
}).catch(() => {});
- const counterValue = await getRedisCache().get(key);
+ const counterValue = await this.redis.get(key);
if (counterValue !== null) {
const parsed = Number.parseInt(counterValue, 10);
if (!Number.isNaN(parsed)) {
@@ -70,7 +75,7 @@ export class BaseBuffer {
// Initialize counter with current size
const count = await fallbackFn();
- await getRedisCache().set(key, count.toString());
+ await this.redis.set(key, count.toString());
return count;
} catch (error) {
this.logger.warn(
@@ -90,7 +95,7 @@ export class BaseBuffer {
return 0
end
`;
- await getRedisCache().eval(script, 1, this.lockKey, lockId);
+ await this.redis.eval(script, 1, this.lockKey, lockId);
}
async tryFlush() {
@@ -121,7 +126,7 @@ export class BaseBuffer {
// Sequential mode: Use lock to ensure only one worker processes at a time
const lockId = generateSecureId('lock');
- const acquired = await getRedisCache().set(
+ const acquired = await this.redis.set(
this.lockKey,
lockId,
'EX',
@@ -142,7 +147,7 @@ export class BaseBuffer {
// On error, we might want to reset counter to avoid drift
if (this.bufferCounterKey) {
this.logger.warn('Resetting buffer counter due to flush error');
- await getRedisCache().del(this.bufferCounterKey);
+ await this.redis.del(this.bufferCounterKey);
}
} finally {
await this.releaseLock(lockId);
diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.ts
index d305372aa..275b4e936 100644
--- a/packages/db/src/buffers/event-buffer.ts
+++ b/packages/db/src/buffers/event-buffer.ts
@@ -1,7 +1,7 @@
import { getSafeJson } from '@openpanel/json';
import {
type Redis,
- getRedisCache,
+ getRedisEvent,
getRedisPub,
publishEvent,
} from '@openpanel/redis';
@@ -25,8 +25,21 @@ import { BaseBuffer } from './base-buffer';
* - Retrieve the last screen_view (don't modify it)
* - Push both screen_view and session_end to buffer
* 4. Flush: Simply process all events from the list buffer
+ *
+ * Optimizations:
+ * - Micro-batching: Events are buffered locally and flushed every 10ms to reduce Redis round-trips
+ * - Batched publishes: All PUBLISH commands are included in the multi pipeline
+ * - Simplified active visitor tracking: Only uses ZADD (removed redundant heartbeat SET)
*/
+// Pending event for local buffer
+interface PendingEvent {
+ event: IClickhouseEvent;
+ eventJson: string;
+ eventWithTimestamp?: string;
+ type: 'regular' | 'screen_view' | 'session_end';
+}
+
export class EventBuffer extends BaseBuffer {
// Configurable limits
private batchSize = process.env.EVENT_BUFFER_BATCH_SIZE
@@ -36,13 +49,35 @@ export class EventBuffer extends BaseBuffer {
? Number.parseInt(process.env.EVENT_BUFFER_CHUNK_SIZE, 10)
: 1000;
+ // Micro-batching configuration
+ private microBatchIntervalMs = process.env.EVENT_BUFFER_MICRO_BATCH_MS
+ ? Number.parseInt(process.env.EVENT_BUFFER_MICRO_BATCH_MS, 10)
+ : 10; // Flush every 10ms by default
+ private microBatchMaxSize = process.env.EVENT_BUFFER_MICRO_BATCH_SIZE
+ ? Number.parseInt(process.env.EVENT_BUFFER_MICRO_BATCH_SIZE, 10)
+ : 100; // Or when we hit 100 events
+
+ // Local event buffer for micro-batching
+ private pendingEvents: PendingEvent[] = [];
+ private flushTimer: ReturnType | null = null;
+ private isFlushing = false;
+
+ // Throttled publish configuration
+ private publishThrottleMs = process.env.EVENT_BUFFER_PUBLISH_THROTTLE_MS
+ ? Number.parseInt(process.env.EVENT_BUFFER_PUBLISH_THROTTLE_MS, 10)
+ : 1000; // Publish at most once per second
+ private lastPublishTime = 0;
+ private pendingPublishEvent: IClickhouseEvent | null = null;
+ private publishTimer: ReturnType | null = null;
+
private activeVisitorsExpiration = 60 * 5; // 5 minutes
// LIST - Stores all events ready to be flushed
- private queueKey = 'event_buffer:queue';
+ // {event_buffer} hash tag ensures all EventBuffer keys hash to the same Redis Cluster slot
+ private queueKey = '{event_buffer}:queue';
// STRING - Tracks total buffer size incrementally
- protected bufferCounterKey = 'event_buffer:total_count';
+ protected bufferCounterKey = '{event_buffer}:total_count';
// Script SHAs for loaded Lua scripts
private scriptShas: {
@@ -52,12 +87,12 @@ export class EventBuffer extends BaseBuffer {
// Hash key for storing last screen_view per session
private getLastScreenViewKeyBySession(sessionId: string) {
- return `event_buffer:last_screen_view:session:${sessionId}`;
+ return `{event_buffer}:last_screen_view:session:${sessionId}`;
}
// Hash key for storing last screen_view per profile
private getLastScreenViewKeyByProfile(projectId: string, profileId: string) {
- return `event_buffer:last_screen_view:profile:${projectId}:${profileId}`;
+ return `{event_buffer}:last_screen_view:profile:${projectId}:${profileId}`;
}
/**
@@ -160,6 +195,8 @@ return added
onFlush: async () => {
await this.processBuffer();
},
+ enableParallelProcessing: true,
+ redis: getRedisEvent(),
});
// Load Lua scripts into Redis on startup
this.loadScripts();
@@ -171,7 +208,7 @@ return added
*/
private async loadScripts() {
try {
- const redis = getRedisCache();
+ const redis = this.redis;
const [screenViewSha, sessionEndSha] = await Promise.all([
redis.script('LOAD', this.addScreenViewScript),
redis.script('LOAD', this.addSessionEndScript),
@@ -190,98 +227,228 @@ return added
}
bulkAdd(events: IClickhouseEvent[]) {
- const redis = getRedisCache();
- const multi = redis.multi();
+ // Add all events to local buffer - they will be flushed together
for (const event of events) {
- this.add(event, multi);
+ this.add(event);
}
- return multi.exec();
}
/**
- * Add an event into Redis buffer.
+ * Add an event into the local buffer for micro-batching.
+ *
+ * Events are buffered locally and flushed to Redis every microBatchIntervalMs
+ * or when microBatchMaxSize is reached. This dramatically reduces Redis round-trips.
*
* Logic:
* - screen_view: Store as "last" for session, flush previous if exists
* - session_end: Flush last screen_view + session_end
* - Other events: Add directly to queue
*/
- async add(event: IClickhouseEvent, _multi?: ReturnType) {
+ add(event: IClickhouseEvent, _multi?: ReturnType) {
+ const eventJson = JSON.stringify(event);
+
+ // Determine event type and prepare data
+ let type: PendingEvent['type'] = 'regular';
+ let eventWithTimestamp: string | undefined;
+
+ if (event.session_id && event.name === 'screen_view') {
+ type = 'screen_view';
+ const timestamp = new Date(event.created_at || Date.now()).getTime();
+ eventWithTimestamp = JSON.stringify({
+ event: event,
+ ts: timestamp,
+ });
+ } else if (event.session_id && event.name === 'session_end') {
+ type = 'session_end';
+ }
+
+ const pendingEvent: PendingEvent = {
+ event,
+ eventJson,
+ eventWithTimestamp,
+ type,
+ };
+
+ // If a multi was provided (legacy bulkAdd pattern), add directly without batching
+ if (_multi) {
+ this.addToMulti(_multi, pendingEvent);
+ return;
+ }
+
+ // Add to local buffer for micro-batching
+ this.pendingEvents.push(pendingEvent);
+
+ // Check if we should flush immediately due to size
+ if (this.pendingEvents.length >= this.microBatchMaxSize) {
+ this.flushLocalBuffer();
+ return;
+ }
+
+ // Schedule flush if not already scheduled
+ if (!this.flushTimer) {
+ this.flushTimer = setTimeout(() => {
+ this.flushTimer = null;
+ this.flushLocalBuffer();
+ }, this.microBatchIntervalMs);
+ }
+ }
+
+ /**
+ * Add a single pending event to a multi pipeline.
+ * Used both for legacy _multi pattern and during batch flush.
+ */
+ private addToMulti(multi: ReturnType, pending: PendingEvent) {
+ const { event, eventJson, eventWithTimestamp, type } = pending;
+
+ if (type === 'screen_view' && event.session_id) {
+ const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
+ const profileKey = event.profile_id
+ ? this.getLastScreenViewKeyByProfile(event.project_id, event.profile_id)
+ : '';
+
+ this.evalScript(
+ multi,
+ 'addScreenView',
+ this.addScreenViewScript,
+ 4,
+ sessionKey,
+ profileKey,
+ this.queueKey,
+ this.bufferCounterKey,
+ eventWithTimestamp!,
+ '3600',
+ );
+ } else if (type === 'session_end' && event.session_id) {
+ const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
+ const profileKey = event.profile_id
+ ? this.getLastScreenViewKeyByProfile(event.project_id, event.profile_id)
+ : '';
+
+ this.evalScript(
+ multi,
+ 'addSessionEnd',
+ this.addSessionEndScript,
+ 4,
+ sessionKey,
+ profileKey,
+ this.queueKey,
+ this.bufferCounterKey,
+ eventJson,
+ );
+ } else {
+ // Regular events go directly to queue
+ multi.rpush(this.queueKey, eventJson).incr(this.bufferCounterKey);
+ }
+
+ // Active visitor tracking (simplified - only ZADD, no redundant SET)
+ if (event.profile_id) {
+ this.incrementActiveVisitorCount(
+ multi,
+ event.project_id,
+ event.profile_id,
+ );
+ }
+ }
+
+ /**
+ * Force flush all pending events from local buffer to Redis immediately.
+ * Useful for testing or when you need to ensure all events are persisted.
+ */
+ public async flush() {
+ // Clear any pending timer
+ if (this.flushTimer) {
+ clearTimeout(this.flushTimer);
+ this.flushTimer = null;
+ }
+ await this.flushLocalBuffer();
+ }
+
+ /**
+ * Flush all pending events from local buffer to Redis in a single pipeline.
+ * This is the core optimization - batching many events into one round-trip.
+ */
+ private async flushLocalBuffer() {
+ if (this.isFlushing || this.pendingEvents.length === 0) {
+ return;
+ }
+
+ this.isFlushing = true;
+
+ // Grab current pending events and clear buffer
+ const eventsToFlush = this.pendingEvents;
+ this.pendingEvents = [];
+
try {
- const redis = getRedisCache();
- const eventJson = JSON.stringify(event);
- const multi = _multi || redis.multi();
-
- if (event.session_id && event.name === 'screen_view') {
- // Handle screen_view
- const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
- const profileKey = event.profile_id
- ? this.getLastScreenViewKeyByProfile(
- event.project_id,
- event.profile_id,
- )
- : '';
- const timestamp = new Date(event.created_at || Date.now()).getTime();
-
- // Combine event and timestamp into single JSON for atomic operations
- const eventWithTimestamp = JSON.stringify({
- event: event,
- ts: timestamp,
- });
-
- this.evalScript(
- multi,
- 'addScreenView',
- this.addScreenViewScript,
- 4,
- sessionKey,
- profileKey,
- this.queueKey,
- this.bufferCounterKey,
- eventWithTimestamp,
- '3600', // 1 hour TTL
- );
- } else if (event.session_id && event.name === 'session_end') {
- // Handle session_end
- const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
- const profileKey = event.profile_id
- ? this.getLastScreenViewKeyByProfile(
- event.project_id,
- event.profile_id,
- )
- : '';
-
- this.evalScript(
- multi,
- 'addSessionEnd',
- this.addSessionEndScript,
- 4,
- sessionKey,
- profileKey,
- this.queueKey,
- this.bufferCounterKey,
- eventJson,
- );
- } else {
- // All other events go directly to queue
- multi.rpush(this.queueKey, eventJson).incr(this.bufferCounterKey);
- }
+ const redis = this.redis;
+ const multi = redis.multi();
- if (event.profile_id) {
- this.incrementActiveVisitorCount(
- multi,
- event.project_id,
- event.profile_id,
- );
+ // Add all events to the pipeline
+ for (const pending of eventsToFlush) {
+ this.addToMulti(multi, pending);
}
- if (!_multi) {
- await multi.exec();
- }
+ await multi.exec();
- await publishEvent('events', 'received', transformEvent(event));
+ // Throttled publish - just signal that events were received
+ // Store the last event for publishing (we only need one to signal activity)
+ const lastEvent = eventsToFlush[eventsToFlush.length - 1];
+ if (lastEvent) {
+ this.scheduleThrottledPublish(lastEvent.event);
+ }
} catch (error) {
- this.logger.error('Failed to add event to Redis buffer', { error });
+ this.logger.error('Failed to flush local buffer to Redis', {
+ error,
+ eventCount: eventsToFlush.length,
+ });
+ } finally {
+ this.isFlushing = false;
+ }
+ }
+
+ /**
+ * Throttled publish - publishes at most once per publishThrottleMs.
+ * Instead of publishing every event, we just signal that events were received.
+ * This reduces pub/sub load from 3000/s to 1/s.
+ */
+ private scheduleThrottledPublish(event: IClickhouseEvent) {
+ // Always keep the latest event
+ this.pendingPublishEvent = event;
+
+ const now = Date.now();
+ const timeSinceLastPublish = now - this.lastPublishTime;
+
+ // If enough time has passed, publish immediately
+ if (timeSinceLastPublish >= this.publishThrottleMs) {
+ this.executeThrottledPublish();
+ return;
+ }
+
+ // Otherwise, schedule a publish if not already scheduled
+ if (!this.publishTimer) {
+ const delay = this.publishThrottleMs - timeSinceLastPublish;
+ this.publishTimer = setTimeout(() => {
+ this.publishTimer = null;
+ this.executeThrottledPublish();
+ }, delay);
+ }
+ }
+
+ /**
+ * Execute the throttled publish with the latest pending event.
+ */
+ private executeThrottledPublish() {
+ if (!this.pendingPublishEvent) {
+ return;
+ }
+
+ const event = this.pendingPublishEvent;
+ this.pendingPublishEvent = null;
+ this.lastPublishTime = Date.now();
+
+ // Fire-and-forget publish (no multi = returns Promise)
+ const result = publishEvent('events', 'received', transformEvent(event));
+ if (result instanceof Promise) {
+ result.catch(() => {});
}
}
@@ -321,21 +488,20 @@ return added
* 5. Clean up processed events from queue
*/
async processBuffer() {
- const redis = getRedisCache();
+ const redis = this.redis;
try {
- // Fetch events from queue
- const queueEvents = await redis.lrange(
- this.queueKey,
- 0,
- this.batchSize - 1,
- );
+ // Atomically pop events off the queue — no two workers can get the same events
+ const queueEvents = await redis.lpop(this.queueKey, this.batchSize);
- if (queueEvents.length === 0) {
+ if (!queueEvents || queueEvents.length === 0) {
this.logger.debug('No events to process');
return;
}
+ // Decrement counter immediately after popping
+ await redis.decrby(this.bufferCounterKey, queueEvents.length);
+
// Parse events
const eventsToClickhouse: IClickhouseEvent[] = [];
for (const eventStr of queueEvents) {
@@ -357,19 +523,21 @@ return added
new Date(b.created_at || 0).getTime(),
);
- // Insert events into ClickHouse in chunks
+ // Insert events into ClickHouse in parallel chunks
this.logger.info('Inserting events into ClickHouse', {
totalEvents: eventsToClickhouse.length,
chunks: Math.ceil(eventsToClickhouse.length / this.chunkSize),
});
- for (const chunk of this.chunks(eventsToClickhouse, this.chunkSize)) {
- await ch.insert({
- table: 'events',
- values: chunk,
- format: 'JSONEachRow',
- });
- }
+ await Promise.all(
+ this.chunks(eventsToClickhouse, this.chunkSize).map((chunk) =>
+ ch.insert({
+ table: 'events',
+ values: chunk,
+ format: 'JSONEachRow',
+ })
+ )
+ );
// Publish "saved" events
const pubMulti = getRedisPub().multi();
@@ -378,13 +546,6 @@ return added
}
await pubMulti.exec();
- // Clean up processed events from queue
- await redis
- .multi()
- .ltrim(this.queueKey, queueEvents.length, -1)
- .decrby(this.bufferCounterKey, queueEvents.length)
- .exec();
-
this.logger.info('Processed events from Redis buffer', {
batchSize: this.batchSize,
eventsProcessed: eventsToClickhouse.length,
@@ -407,7 +568,7 @@ return added
profileId: string;
},
): Promise {
- const redis = getRedisCache();
+ const redis = this.redis;
let lastScreenViewKey: string;
if ('sessionId' in params) {
@@ -435,28 +596,32 @@ return added
public async getBufferSize() {
return this.getBufferSizeWithCounter(async () => {
- const redis = getRedisCache();
+ const redis = this.redis;
return await redis.llen(this.queueKey);
});
}
- private async incrementActiveVisitorCount(
+ /**
+ * Track active visitors using ZADD only.
+ *
+ * Optimization: Removed redundant heartbeat SET key.
+ * The ZADD score (timestamp) already tracks when a visitor was last seen.
+ * We use ZRANGEBYSCORE in getActiveVisitorCount to filter active visitors.
+ */
+ private incrementActiveVisitorCount(
multi: ReturnType,
projectId: string,
profileId: string,
) {
- // Track active visitors and emit expiry events when inactive for TTL
const now = Date.now();
- const zsetKey = `live:visitors:${projectId}`;
- const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
- return multi
- .zadd(zsetKey, now, profileId)
- .set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
+ const zsetKey = `{event_buffer}:live:visitors:${projectId}`;
+ // Only ZADD - the score is the timestamp, no need for separate heartbeat key
+ return multi.zadd(zsetKey, now, profileId);
}
public async getActiveVisitorCount(projectId: string): Promise {
- const redis = getRedisCache();
- const zsetKey = `live:visitors:${projectId}`;
+ const redis = this.redis;
+ const zsetKey = `{event_buffer}:live:visitors:${projectId}`;
const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
const multi = redis.multi();
diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.ts
index fc82937f1..823125a7a 100644
--- a/packages/db/src/buffers/profile-buffer.ts
+++ b/packages/db/src/buffers/profile-buffer.ts
@@ -6,6 +6,7 @@ import shallowEqual from 'fast-deep-equal';
import { omit } from 'ramda';
import { TABLE_NAMES, ch, chQuery } from '../clickhouse/client';
import type { IClickhouseProfile } from '../services/profile.service';
+import { createEvent } from '../services/event.service';
import { BaseBuffer } from './base-buffer';
export class ProfileBuffer extends BaseBuffer {
@@ -68,6 +69,20 @@ export class ProfileBuffer extends BaseBuffer {
const existingProfile = await this.fetchProfile(profile, logger);
+ // Create _firstSeen event for new profiles
+ if (!existingProfile) {
+ logger.info('New profile detected, creating _firstSeen event', {
+ profileId: profile.id,
+ projectId: profile.project_id,
+ });
+ await this.createFirstSeenEvent(profile).catch((error) => {
+ logger.error('Failed to create _firstSeen event', {
+ error,
+ profileId: profile.id,
+ });
+ });
+ }
+
// Delete any properties that are not server related if we have a non-server profile
if (
existingProfile?.properties.device !== 'server' &&
@@ -201,6 +216,8 @@ export class ProfileBuffer extends BaseBuffer {
}
ORDER BY created_at DESC
LIMIT 1`,
+ undefined,
+ true, // Bypass concurrency limit for profile queries
);
logger.debug('Clickhouse fetch result', {
@@ -254,4 +271,52 @@ export class ProfileBuffer extends BaseBuffer {
async getBufferSize() {
return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey));
}
+
+ /**
+ * Creates a _firstSeen event when a new profile is detected
+ * This event only fires once per user in their entire journey
+ * The underscore prefix marks it as a system-generated event
+ */
+ private async createFirstSeenEvent(profile: IClickhouseProfile) {
+ const createdAt = new Date(profile.created_at);
+
+ await createEvent({
+ name: '_firstSeen',
+ deviceId: profile.id,
+ profileId: profile.id,
+ projectId: profile.project_id,
+ sessionId: '', // No session for system events
+ properties: {
+ isExternal: profile.is_external,
+ source: 'system',
+ // Include initial profile properties
+ ...profile.properties,
+ },
+ createdAt,
+ path: profile.properties.path || '',
+ origin: profile.properties.origin || '',
+ country: profile.properties.country || '',
+ city: profile.properties.city || '',
+ region: profile.properties.region || '',
+ longitude: profile.properties.longitude
+ ? Number(profile.properties.longitude)
+ : undefined,
+ latitude: profile.properties.latitude
+ ? Number(profile.properties.latitude)
+ : undefined,
+ os: profile.properties.os || '',
+ osVersion: profile.properties.os_version || '',
+ browser: profile.properties.browser || '',
+ browserVersion: profile.properties.browser_version || '',
+ device: profile.properties.device || '',
+ brand: profile.properties.brand || '',
+ model: profile.properties.model || '',
+ referrer: profile.properties.referrer || '',
+ referrerName: profile.properties.referrer_name || '',
+ referrerType: profile.properties.referrer_type || '',
+ duration: 0,
+ sdkName: '',
+ sdkVersion: '',
+ });
+ }
}
diff --git a/packages/db/src/buffers/session-buffer.ts b/packages/db/src/buffers/session-buffer.ts
index fad5ee097..cea1eb671 100644
--- a/packages/db/src/buffers/session-buffer.ts
+++ b/packages/db/src/buffers/session-buffer.ts
@@ -1,4 +1,4 @@
-import { type Redis, getRedisCache } from '@openpanel/redis';
+import { getRedisSession } from '@openpanel/redis';
import { getSafeJson } from '@openpanel/json';
import { assocPath, clone } from 'ramda';
@@ -16,15 +16,15 @@ export class SessionBuffer extends BaseBuffer {
: 1000;
private readonly redisKey = 'session-buffer';
- private redis: Redis;
+
constructor() {
super({
name: 'session',
onFlush: async () => {
await this.processBuffer();
},
+ redis: getRedisSession(),
});
- this.redis = getRedisCache();
}
public async getExistingSession(
diff --git a/packages/db/src/cli/materialize.ts b/packages/db/src/cli/materialize.ts
new file mode 100644
index 000000000..14ef08d42
--- /dev/null
+++ b/packages/db/src/cli/materialize.ts
@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+import { materializeColumnsService } from '../services/materialize-columns.service';
+
+async function main() {
+ const args = process.argv.slice(2);
+
+ const dryRun = !args.includes('--execute');
+ const thresholdArg = args.find((arg) => arg.startsWith('--threshold='));
+ const threshold = thresholdArg
+ ? Number.parseInt(thresholdArg.split('=')[1]!)
+ : 150;
+
+ if (Number.isNaN(threshold) || threshold < 0) {
+ console.error('Error: Invalid threshold value');
+ process.exit(1);
+ }
+
+ console.log('Materialized Column Analyzer');
+ console.log(`Mode: ${dryRun ? 'DRY RUN' : 'EXECUTE'}`);
+ console.log(`Threshold: ${threshold}`);
+ console.log('');
+
+ try {
+ const result = await materializeColumnsService.analyze({
+ dryRun,
+ threshold,
+ });
+
+ console.log(result.report);
+
+ if (!dryRun && result.materialized.length > 0) {
+ console.log('\n✅ Successfully materialized:');
+ for (const prop of result.materialized) {
+ console.log(` - ${prop}`);
+ }
+ }
+
+ process.exit(0);
+ } catch (error) {
+ console.error('\n❌ Error during analysis:', error);
+ process.exit(1);
+ }
+}
+
+main();
diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.ts
index 50cfbb177..9f9615ff3 100644
--- a/packages/db/src/clickhouse/client.ts
+++ b/packages/db/src/clickhouse/client.ts
@@ -58,7 +58,11 @@ export const TABLE_NAMES = {
event_property_values_mv: 'event_property_values_mv',
cohort_events_mv: 'cohort_events_mv',
sessions: 'sessions',
- events_imports: 'events_imports',
+ events_imports: 'events_imports_v2',
+ cohort_members: 'cohort_members',
+ cohort_metadata: 'cohort_metadata',
+ profile_event_summary_mv: 'profile_event_summary_mv',
+ profile_event_property_summary_mv: 'profile_event_property_summary_mv',
};
/**
@@ -102,7 +106,7 @@ function getClickhouseSettings(): ClickHouseSettings {
export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {
max_open_connections: 30,
- request_timeout: 300000,
+ request_timeout: parseInt(process.env.CLICKHOUSE_REQUEST_TIMEOUT || '3600000', 10),
keep_alive: {
enabled: true,
idle_socket_ttl: 60000,
@@ -131,7 +135,7 @@ const cleanQuery = (query?: string) =>
export async function withRetry(
operation: () => Promise,
- maxRetries = 3,
+ maxRetries = 5,
baseDelay = 500,
): Promise {
let lastError: Error | undefined;
@@ -188,12 +192,25 @@ export const ch = new Proxy(originalCh, {
http_headers_progress_interval_ms: '50000',
// Ensure server holds the connection until the query is finished
wait_end_of_query: 1,
+ // Remove concurrent query limit for INSERT operations to prevent blocking
+ // when multiple buffers flush simultaneously
+ max_concurrent_queries_for_user: Number.parseInt(
+ process.env.CLICKHOUSE_INSERT_QUERY_LIMIT || '50',
+ 10
+ ),
...args[0].clickhouse_settings,
};
return value.apply(target, args);
});
}
+ if (property === 'query') {
+ return (...args: any[]) =>
+ withRetry(() => {
+ return value.apply(target, args);
+ });
+ }
+
if (property === 'command') {
return (...args: any[]) =>
withRetry(() => {
@@ -208,11 +225,25 @@ export const ch = new Proxy(originalCh, {
export async function chQueryWithMeta>(
query: string,
clickhouseSettings?: ClickHouseSettings,
+ bypassConcurrencyLimit = false,
): Promise> {
const start = Date.now();
+
+ // Merge settings, allowing higher concurrent query limit for critical operations
+ // to prevent profile queries from being blocked by dashboard query limits
+ const finalSettings = bypassConcurrencyLimit
+ ? {
+ ...clickhouseSettings,
+ max_concurrent_queries_for_user: Number.parseInt(
+ process.env.CLICKHOUSE_PROFILE_QUERY_LIMIT || '50',
+ 10
+ )
+ }
+ : clickhouseSettings;
+
const res = await ch.query({
query,
- clickhouse_settings: clickhouseSettings,
+ clickhouse_settings: finalSettings,
});
const json = await res.json();
const keys = Object.keys(json.data[0] || {});
@@ -246,24 +277,33 @@ export async function chQueryWithMeta>(
export async function chInsertCSV(tableName: string, rows: string[]) {
try {
const now = performance.now();
- // Create a readable stream in binary mode for CSV (similar to EventBuffer)
- const csvStream = Readable.from(rows.join('\n'), {
- objectMode: false,
- });
-
- await ch.insert({
- table: tableName,
- values: csvStream,
- format: 'CSV',
- clickhouse_settings: {
- format_csv_allow_double_quotes: 1,
- format_csv_allow_single_quotes: 0,
- },
- });
+ const chunkSize = Number.parseInt(
+ process.env.IMPORT_CSV_CHUNK_SIZE || '10000',
+ 10,
+ );
+
+ // Insert in chunks to reduce memory pressure
+ for (let i = 0; i < rows.length; i += chunkSize) {
+ const chunk = rows.slice(i, i + chunkSize);
+ const csvStream = Readable.from(chunk.join('\n'), {
+ objectMode: false,
+ });
+
+ await ch.insert({
+ table: tableName,
+ values: csvStream,
+ format: 'CSV',
+ clickhouse_settings: {
+ format_csv_allow_double_quotes: 1,
+ format_csv_allow_single_quotes: 0,
+ },
+ });
+ }
logger.info('CSV Insert successful', {
elapsed: performance.now() - now,
rows: rows.length,
+ chunks: Math.ceil(rows.length / chunkSize),
});
} catch (error) {
logger.error('CSV Insert failed:', error);
@@ -274,8 +314,9 @@ export async function chInsertCSV(tableName: string, rows: string[]) {
export async function chQuery>(
query: string,
clickhouseSettings?: ClickHouseSettings,
+ bypassConcurrencyLimit = false,
): Promise {
- return (await chQueryWithMeta(query, clickhouseSettings)).data;
+ return (await chQueryWithMeta(query, clickhouseSettings, bypassConcurrencyLimit)).data;
}
export function formatClickhouseDate(
diff --git a/packages/db/src/engine/fetch.ts b/packages/db/src/engine/fetch.ts
index 954c98e99..f11c694b8 100644
--- a/packages/db/src/engine/fetch.ts
+++ b/packages/db/src/engine/fetch.ts
@@ -4,6 +4,7 @@ import { alphabetIds } from '@openpanel/constants';
import type { IGetChartDataInput } from '@openpanel/validation';
import { chQuery } from '../clickhouse/client';
import { getChartSql } from '../services/chart.service';
+import { getCustomEventByName } from '../services/custom-event.service';
import type { ConcreteSeries, Plan } from './types';
/**
@@ -33,13 +34,20 @@ export async function fetch(plan: Plan): Promise {
continue;
}
- // Build query input
+ // Check if this is a custom event
+ const customEvent = await getCustomEventByName(
+ event.name,
+ plan.input.projectId,
+ );
+
+ // Build query input — merge global filters into per-event filters
+ const globalFilters = plan.input.globalFilters ?? [];
const queryInput: IGetChartDataInput = {
event: {
id: event.id,
name: event.name,
segment: event.segment,
- filters: event.filters,
+ filters: [...event.filters, ...globalFilters],
displayName: event.displayName,
property: event.property,
},
@@ -56,11 +64,24 @@ export async function fetch(plan: Plan): Promise {
criteria: plan.input.criteria,
funnelGroup: plan.input.funnelGroup,
funnelWindow: plan.input.funnelWindow,
+ cohortFilters: plan.input.cohortFilters ?? [],
+ globalFilters: plan.input.globalFilters ?? [],
+ holdProperties: plan.input.holdProperties ?? [],
+ measuring: plan.input.measuring ?? 'conversion_rate',
};
- // Execute query
+ // Execute query with custom event if applicable
let queryResult = await chQuery(
- getChartSql({ ...queryInput, timezone: plan.timezone }),
+ await getChartSql({
+ ...queryInput,
+ timezone: plan.timezone,
+ customEvent: customEvent
+ ? {
+ name: customEvent.name,
+ definition: customEvent.definition as any,
+ }
+ : undefined,
+ }),
{
session_timezone: plan.timezone,
},
@@ -69,10 +90,16 @@ export async function fetch(plan: Plan): Promise {
// Fallback: if no results with breakdowns, try without breakdowns
if (queryResult.length === 0 && plan.input.breakdowns.length > 0) {
queryResult = await chQuery(
- getChartSql({
+ await getChartSql({
...queryInput,
breakdowns: [],
timezone: plan.timezone,
+ customEvent: customEvent
+ ? {
+ name: customEvent.name,
+ definition: customEvent.definition as any,
+ }
+ : undefined,
}),
{
session_timezone: plan.timezone,
diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.ts
index f9e60337e..2d9a728e1 100644
--- a/packages/db/src/services/chart.service.ts
+++ b/packages/db/src/services/chart.service.ts
@@ -7,10 +7,204 @@ import type {
IChartInput,
IChartRange,
IGetChartDataInput,
+ ICustomEventDefinition,
+ CohortDefinition,
} from '@openpanel/validation';
import { TABLE_NAMES, formatClickhouseDate } from '../clickhouse/client';
import { createSqlBuilder } from '../sql-builder';
+import {
+ getCustomEventByName,
+ expandCustomEventToSQL,
+} from './custom-event.service';
+import { db } from '../../index';
+import { buildEventCriteriaQuery, buildPropertyBasedCohortQuery } from './cohort.service';
+
+// Cache for materialized columns mapping
+let materializedColumnsCache: Record | null = null;
+let cacheTimestamp = 0;
+const CACHE_TTL = 60 * 60 * 1000; // 1 hour
+
+function filterByTargetTable(
+ cache: Record,
+ targetTable?: 'events' | 'profiles',
+): Record {
+ if (!targetTable) return cache;
+ return Object.fromEntries(
+ Object.entries(cache).filter(([key]) =>
+ targetTable === 'profiles'
+ ? key.startsWith('profile.properties.')
+ : key.startsWith('properties.'),
+ ),
+ );
+}
+
+/**
+ * Get materialized columns from database with caching.
+ * Pass targetTable to get only columns for that table:
+ * 'events' → keys starting with "properties.*"
+ * 'profiles' → keys starting with "profile.properties.*"
+ * (omit) → all columns (used by getSelectPropertyKey)
+ */
+export async function getMaterializedColumns(targetTable?: 'events' | 'profiles'): Promise> {
+ const now = Date.now();
+
+ // Return cached value if still valid
+ if (materializedColumnsCache && (now - cacheTimestamp) < CACHE_TTL) {
+ return filterByTargetTable(materializedColumnsCache, targetTable);
+ }
+
+ try {
+ const columns = await db.materializedColumn.findMany({
+ where: { status: 'active' },
+ select: { propertyKey: true, columnName: true, targetTable: true },
+ });
+
+ const mapping: Record = {};
+ for (const col of columns) {
+ const quotedColName = `\`${col.columnName}\``;
+ if (col.targetTable === 'profiles') {
+ // e.g. "profile.properties.campaign" -> "profile.`campaign`"
+ mapping[`profile.properties.${col.propertyKey}`] = `profile.${quotedColName}`;
+ } else {
+ // e.g. "properties.utm_source" -> "`utm_source`"
+ mapping[`properties.${col.propertyKey}`] = quotedColName;
+ }
+ }
+
+ materializedColumnsCache = mapping;
+ cacheTimestamp = now;
+ return filterByTargetTable(mapping, targetTable);
+ } catch (error) {
+ // If database query fails, return empty mapping (fallback to properties['key'])
+ console.warn('Failed to load materialized columns:', error);
+ return {};
+ }
+}
+
+/**
+ * Initialize materialized columns cache (call at startup)
+ */
+export async function initMaterializedColumnsCache(): Promise {
+ await getMaterializedColumns();
+}
+
+/**
+ * Refresh materialized columns cache (call after adding new columns)
+ */
+export async function refreshMaterializedColumnsCache(): Promise {
+ materializedColumnsCache = null;
+ cacheTimestamp = 0;
+ await getMaterializedColumns();
+}
+
+// Initialize cache on module load (lazy)
+getMaterializedColumns().catch(() => {
+ // Ignore errors on initial load
+});
+
+// Cohort metadata type
+type CohortMetadata = {
+ id: string;
+ computeOnDemand: boolean;
+ definition: CohortDefinition;
+};
+
+/**
+ * Fetch metadata for multiple cohorts from Postgres (no cache - always fresh)
+ */
+export async function fetchCohortsMetadata(
+ cohortIds: string[],
+): Promise> {
+ if (cohortIds.length === 0) {
+ return new Map();
+ }
+
+ // Fetch all cohorts in one query
+ const cohorts = await db.cohort.findMany({
+ where: { id: { in: cohortIds } },
+ select: { id: true, computeOnDemand: true, definition: true },
+ });
+
+ return new Map(
+ cohorts.map((c) => [
+ c.id,
+ {
+ id: c.id,
+ computeOnDemand: c.computeOnDemand,
+ definition: c.definition as CohortDefinition,
+ },
+ ]),
+ );
+}
+
+/**
+ * Generate cohort membership query for CTE
+ * Returns the full SELECT query
+ */
+export function buildCohortMembershipQuery(
+ cohortId: string,
+ projectId: string,
+ cohortMeta?: CohortMetadata,
+): string {
+ // Pre-computed cohorts or missing metadata: read from stored membership
+ if (!cohortMeta || !cohortMeta.computeOnDemand) {
+ return `
+ SELECT profile_id
+ FROM ${TABLE_NAMES.cohort_members} FINAL
+ WHERE cohort_id = ${sqlstring.escape(cohortId)}
+ AND project_id = ${sqlstring.escape(projectId)}
+ `;
+ }
+
+ // Dynamic cohorts: build query from definition
+ const definition = cohortMeta.definition;
+
+ if (definition.type === 'event') {
+ const { events, operator } = definition.criteria;
+ const queries = events.map((eventCriteria) =>
+ buildEventCriteriaQuery(projectId, eventCriteria),
+ );
+
+ return operator === 'and'
+ ? queries.join(' INTERSECT ')
+ : queries.join(' UNION DISTINCT ');
+ } else if (definition.type === 'property') {
+ return buildPropertyBasedCohortQuery(projectId, definition);
+ }
+
+ throw new Error(`Unknown cohort type: ${(definition as any).type}`);
+}
+
+/**
+ * Get CTE name for a cohort (quoted with backticks)
+ */
+export function getCohortCteName(cohortId: string): string {
+ return `\`cohort-${cohortId}\``;
+}
+
+/**
+ * Get table alias for a cohort (used in JOINs)
+ */
+export function getCohortAlias(cohortId: string): string {
+ return `cohort_${cohortId.replace(/-/g, '_')}`;
+}
+
+/**
+ * Build inline cohort JOIN clause with subquery
+ * Used in CTEs where CTE references are not allowed by ClickHouse
+ */
+function buildInlineCohortJoin(
+ cohortId: string,
+ projectId: string,
+ tableAlias: string,
+ cohortMeta?: CohortMetadata,
+): string {
+ const cohortAlias = getCohortAlias(cohortId);
+ const cohortQuery = buildCohortMembershipQuery(cohortId, projectId, cohortMeta);
+ return `LEFT ANY JOIN (${cohortQuery}) AS ${cohortAlias} ON ${cohortAlias}.profile_id = ${tableAlias}.profile_id`;
+}
+
export function transformPropertyKey(property: string) {
const propertyPatterns = ['properties', 'profile.properties'];
@@ -33,9 +227,35 @@ export function transformPropertyKey(property: string) {
return `${match}['${property.replace(new RegExp(`^${match}.`), '')}']`;
}
-export function getSelectPropertyKey(property: string) {
+export function getSelectPropertyKey(
+ property: string,
+ projectId?: string,
+ cohortId?: string,
+) {
+ // Handle cohort breakdown
+ // Use cohortId parameter if provided, otherwise parse from property name (backwards compatibility)
+ const extractedCohortId = cohortId || (property.startsWith('cohort:') ? property.split(':')[1] : null);
+
+ if (extractedCohortId && projectId) {
+ // Use JOIN-based approach instead of IN subquery for better performance
+ const cohortAlias = getCohortAlias(extractedCohortId);
+ // Use notEmpty() to handle both join_use_nulls=0 (returns '') and join_use_nulls=1 (returns NULL)
+ // When join_use_nulls=0, ClickHouse returns empty string for unmatched LEFT JOIN rows instead of NULL
+ // notEmpty() correctly identifies both cases as "Not In Cohort"
+ return `if(
+ notEmpty(${cohortAlias}.profile_id),
+ 'In Cohort',
+ 'Not In Cohort'
+ )`;
+ }
+
if (property === 'has_profile') {
- return `if(profile_id != device_id, 'true', 'false')`;
+ return `if(e.profile_id != e.device_id, 'true', 'false')`;
+ }
+
+ // Handle profile.created_at - it's stored as created_at in the profiles table
+ if (property === 'profile.created_at') {
+ return 'profile.created_at';
}
const propertyPatterns = ['properties', 'profile.properties'];
@@ -45,6 +265,11 @@ export function getSelectPropertyKey(property: string) {
);
if (!match) return property;
+ // Use materialized columns from cache if available
+ if (materializedColumnsCache && materializedColumnsCache[property]) {
+ return materializedColumnsCache[property]!;
+ }
+
if (property.includes('*')) {
return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(${match}, ${sqlstring.escape(
transformPropertyKey(property),
@@ -54,7 +279,117 @@ export function getSelectPropertyKey(property: string) {
return `${match}['${property.replace(new RegExp(`^${match}.`), '')}']`;
}
-export function getChartSql({
+function getChartSqlFromMaterializedView({
+ event,
+ interval,
+ startDate,
+ endDate,
+ projectId,
+ timezone,
+}: {
+ event: IGetChartDataInput['event'];
+ interval: IGetChartDataInput['interval'];
+ startDate: string;
+ endDate: string;
+ projectId: string;
+ timezone: string;
+}): string {
+ const { sb, getSelect, getWhere, getGroupBy, getOrderBy, getFill } =
+ createSqlBuilder();
+
+ // Use materialized view table with alias to avoid column name conflicts
+ sb.from = 'events_daily_stats t';
+
+ // Base filters (use table alias)
+ sb.where.projectId = `t.project_id = ${sqlstring.escape(projectId)}`;
+ if (event.name !== '*') {
+ sb.where.eventName = `t.name = ${sqlstring.escape(event.name)}`;
+ }
+ sb.where.dateRange = `t.date >= toDate(${sqlstring.escape(startDate)}) AND t.date <= toDate(${sqlstring.escape(endDate)})`;
+
+ // Label
+ if (event.name !== '*') {
+ sb.select.label_0 = `${sqlstring.escape(event.name)} as label_0`;
+ } else {
+ sb.select.label_0 = `'*' as label_0`;
+ }
+
+ // Count based on segment (use table alias)
+ if (event.segment === 'user') {
+ sb.select.count = 'uniqMerge(t.unique_profiles_state) as count';
+ } else if (event.segment === 'session') {
+ sb.select.count = 'uniqMerge(t.unique_sessions_state) as count';
+ } else {
+ sb.select.count = 'countMerge(t.event_count) as count';
+ }
+
+ // Date aggregation based on interval (use table alias)
+ // Use DateTime format to match the regular query output (toStartOfDay returns DateTime).
+ // This prevents date-format mismatches when mixing MV series with non-MV series
+ // (e.g. one series has filters → regular query returns "2026-02-24 00:00:00",
+ // another has no filters → MV query previously returned bare "2026-02-24").
+ if (interval === 'day') {
+ sb.select.date = 'toStartOfDay(toDateTime(t.date)) as date';
+ sb.groupBy.date = 'toStartOfDay(toDateTime(t.date))';
+ sb.orderBy.date = 'toStartOfDay(toDateTime(t.date)) ASC';
+ } else if (interval === 'week') {
+ sb.select.date = 'toDateTime(toStartOfWeek(t.date, 1)) as date';
+ sb.groupBy.date = 'toDateTime(toStartOfWeek(t.date, 1))';
+ sb.orderBy.date = 'toDateTime(toStartOfWeek(t.date, 1)) ASC';
+ } else if (interval === 'month') {
+ sb.select.date = 'toDateTime(toStartOfMonth(t.date)) as date';
+ sb.groupBy.date = 'toDateTime(toStartOfMonth(t.date))';
+ sb.orderBy.date = 'toDateTime(toStartOfMonth(t.date)) ASC';
+ }
+
+ // Build WITH FILL for date gaps — use DateTime to match the regular query fill format
+ let fillClause = '';
+ if (interval === 'day') {
+ fillClause = `WITH FILL FROM toStartOfDay(toDateTime(${sqlstring.escape(startDate)})) TO toStartOfDay(toDateTime(${sqlstring.escape(endDate)})) STEP toIntervalDay(1)`;
+ } else if (interval === 'week') {
+ fillClause = `WITH FILL FROM toDateTime(toStartOfWeek(toDate(${sqlstring.escape(startDate)}), 1)) TO toDateTime(toStartOfWeek(toDate(${sqlstring.escape(endDate)}), 1)) STEP toIntervalWeek(1)`;
+ } else if (interval === 'month') {
+ fillClause = `WITH FILL FROM toDateTime(toStartOfMonth(toDate(${sqlstring.escape(startDate)}))) TO toDateTime(toStartOfMonth(toDate(${sqlstring.escape(endDate)}))) STEP toIntervalMonth(1)`;
+ }
+
+ const sql = `${getSelect()} FROM ${sb.from} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${fillClause}`;
+
+ console.log('-- Using Materialized View --');
+ console.log(sql.replaceAll(/[\n\r]/g, ' '));
+ console.log('-- End --');
+
+ return sql;
+}
+
+function canUseMaterializedView(
+ event: IGetChartDataInput['event'],
+ breakdowns: IGetChartDataInput['breakdowns'],
+ interval: IGetChartDataInput['interval'],
+): boolean {
+ // Can use MV if:
+ // 1. Interval is day or larger (not hour/minute)
+ // 2. No breakdowns OR single breakdown with no filters
+ // 3. Segment is 'user' or 'session' or 'event'
+ // 4. No complex property filters
+ // 5. No cohort breakdowns
+ const validIntervals = ['day', 'week', 'month'];
+ const validSegments = ['user', 'session', 'event'];
+
+ const hasCohortBreakdown = breakdowns.some(b => b.name.startsWith('cohort:'));
+ if (hasCohortBreakdown) {
+ return false;
+ }
+
+ return (
+ validIntervals.includes(interval) &&
+ validSegments.includes(event.segment ?? 'event') &&
+ breakdowns.length === 0 &&
+ (!event.filters || event.filters.length === 0) &&
+ event.segment !== 'one_event_per_user'
+ );
+}
+
+export async function getChartSql({
event,
breakdowns,
interval,
@@ -64,7 +399,48 @@ export function getChartSql({
limit,
timezone,
chartType,
-}: IGetChartDataInput & { timezone: string }) {
+ customEvent,
+}: IGetChartDataInput & {
+ timezone: string;
+ customEvent?: { name: string; definition: ICustomEventDefinition };
+}) {
+ // Pre-fetch cohort metadata for all cohorts used in this query (deduplicated)
+ const cohortIdsSet = new Set();
+
+ // Extract cohort IDs from breakdowns
+ breakdowns?.forEach((b) => {
+ if (b.cohortId) {
+ cohortIdsSet.add(b.cohortId);
+ } else if (b.name.startsWith('cohort:')) {
+ cohortIdsSet.add(b.name.split(':')[1]!);
+ }
+ });
+
+ // Extract cohort IDs from event filters
+ event.filters?.forEach((filter) => {
+ if (filter.cohortId) {
+ cohortIdsSet.add(filter.cohortId);
+ }
+ });
+
+ const cohortIds = Array.from(cohortIdsSet);
+
+ // Fetch cohort metadata from Postgres (always fresh, no cache)
+ const cohortMetadata = await fetchCohortsMetadata(cohortIds);
+
+ // Check if we can use materialized view for fast queries
+ // Custom events cannot use materialized views (for now)
+ if (!customEvent && canUseMaterializedView(event, breakdowns, interval)) {
+ return getChartSqlFromMaterializedView({
+ event,
+ interval,
+ startDate,
+ endDate,
+ projectId,
+ timezone,
+ });
+ }
+
const {
sb,
join,
@@ -79,14 +455,46 @@ export function getChartSql({
with: addCte,
} = createSqlBuilder();
- sb.where = getEventFiltersWhereClause(event.filters);
- sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
+ // Determine if we can use profile_event_summary_mv instead of raw events table
+ // for cohort-only breakdown queries (no property breakdowns/filters, day+ interval)
+ const hasCohortBreakdown = breakdowns.some((b) => b.name.startsWith('cohort:') || b.cohortId);
+ const allBreakdownsAreCohort = breakdowns.every((b) => b.name.startsWith('cohort:') || b.cohortId);
+ const hasPropertyFilters = event.filters?.some((f) => f.name.startsWith('properties.') || f.name.startsWith('profile.'));
+ const useCohortMV =
+ !customEvent &&
+ hasCohortBreakdown &&
+ allBreakdownsAreCohort &&
+ !hasPropertyFilters &&
+ ['day', 'week', 'month'].includes(interval) &&
+ ['user', 'event', undefined].includes(event.segment ?? 'event') &&
+ event.segment !== 'one_event_per_user';
+
+ if (useCohortMV) {
+ sb.from = `${TABLE_NAMES.profile_event_summary_mv} e`;
+ }
- if (event.name !== '*') {
- sb.select.label_0 = `${sqlstring.escape(event.name)} as label_0`;
+ // Create CTEs for all cohorts (used by main query only)
+ // NOTE: ClickHouse allows CTEs to be referenced in the main query's JOINs,
+ // but NOT in other CTEs' JOINs. For CTEs that need cohort data, we inline subqueries.
+ cohortIds.forEach((cohortId) => {
+ const cohortMeta = cohortMetadata.get(cohortId);
+ const cohortQuery = buildCohortMembershipQuery(cohortId, projectId, cohortMeta);
+ addCte(getCohortCteName(cohortId), cohortQuery);
+ });
+
+ // Common setup
+ sb.where = getEventFiltersWhereClause(event.filters, projectId);
+ sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
+ sb.select.label_0 =
+ event.name !== '*'
+ ? `${sqlstring.escape(event.name)} as label_0`
+ : `'*' as label_0`;
+
+ // Setup data source: custom event CTE or regular events table
+ if (customEvent) {
+ await setupCustomEventCTE(sb, addCte, customEvent, projectId, startDate, endDate);
+ } else if (event.name !== '*') {
sb.where.eventName = `name = ${sqlstring.escape(event.name)}`;
- } else {
- sb.select.label_0 = `'*' as label_0`;
}
const anyFilterOnProfile = event.filters.some((filter) =>
@@ -108,41 +516,41 @@ export function getChartSql({
// Collect all profile fields used in filters and breakdowns
// Extract top-level field names (e.g., 'properties' from 'profile.properties.os')
+ // When a profile.properties.* key has a materialized column, select that column
+ // directly instead of the whole properties map.
const getProfileFields = () => {
const fields = new Set();
// Always need id for the join
fields.add('id');
- // Collect from filters
- event.filters
- .filter((f) => f.name.startsWith('profile.'))
- .forEach((f) => {
- const fieldName = f.name.replace('profile.', '').split('.')[0];
- if (fieldName && fieldName === 'properties') {
+ const allProfileNames = [
+ ...event.filters.filter((f) => f.name.startsWith('profile.')).map((f) => f.name),
+ ...breakdowns.filter((b) => b.name.startsWith('profile.')).map((b) => b.name),
+ ];
+
+ for (const propName of allProfileNames) {
+ if (propName.startsWith('profile.properties.')) {
+ // Check if there's a materialized column in the cache
+ if (materializedColumnsCache && materializedColumnsCache[propName]) {
+ // e.g. cache value is "profile.campaign" -> select "campaign"
+ const colName = materializedColumnsCache[propName]!.replace('profile.', '');
+ fields.add(colName);
+ } else {
+ // No materialized column: select the whole properties map
fields.add('properties');
- } else if (
- fieldName &&
- ['email', 'first_name', 'last_name'].includes(fieldName)
- ) {
- fields.add(fieldName);
}
- });
-
- // Collect from breakdowns
- breakdowns
- .filter((b) => b.name.startsWith('profile.'))
- .forEach((b) => {
- const fieldName = b.name.replace('profile.', '').split('.')[0];
- if (fieldName && fieldName === 'properties') {
- fields.add('properties');
- } else if (
+ } else {
+ // Direct profile field (email, first_name, etc.)
+ const fieldName = propName.replace('profile.', '').split('.')[0];
+ if (
fieldName &&
- ['email', 'first_name', 'last_name'].includes(fieldName)
+ ['email', 'first_name', 'last_name', 'created_at'].includes(fieldName)
) {
fields.add(fieldName);
}
- });
+ }
+ }
return Array.from(fields);
};
@@ -151,27 +559,20 @@ export function getChartSql({
// Only select the fields that are actually used
const profilesJoinRef =
anyFilterOnProfile || anyBreakdownOnProfile
- ? 'LEFT ANY JOIN profile ON profile.id = profile_id'
+ ? 'LEFT ANY JOIN profile ON profile.id = e.profile_id'
+ : '';
+
+ // Profile JOIN for CTEs that don't use 'e' alias (use table name directly)
+ const profilesJoinRefForCTE =
+ anyFilterOnProfile || anyBreakdownOnProfile
+ ? `LEFT ANY JOIN profile ON profile.id = ${TABLE_NAMES.events}.profile_id`
: '';
if (anyFilterOnProfile || anyBreakdownOnProfile) {
const profileFields = getProfileFields();
const selectFields = profileFields.map((field) => {
- if (field === 'id') {
- return 'id as "profile.id"';
- }
- if (field === 'properties') {
- return 'properties as "profile.properties"';
- }
- if (field === 'email') {
- return 'email as "profile.email"';
- }
- if (field === 'first_name') {
- return 'first_name as "profile.first_name"';
- }
- if (field === 'last_name') {
- return 'last_name as "profile.last_name"';
- }
+ // Keep original column names without aliases
+ // so they can be accessed as profile.properties, profile.email, etc.
return field;
});
@@ -179,7 +580,7 @@ export function getChartSql({
addCte(
'profile',
`SELECT ${selectFields.join(', ')}
- FROM ${TABLE_NAMES.profiles} FINAL
+ FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`,
);
@@ -187,31 +588,43 @@ export function getChartSql({
sb.joins.profiles = profilesJoinRef;
}
- sb.select.count = 'count(*) as count';
+ // Add LEFT JOINs for all cohorts (much faster than IN subqueries)
+ cohortIds.forEach((cohortId) => {
+ const cohortAlias = getCohortAlias(cohortId);
+ const cohortCte = getCohortCteName(cohortId);
+ sb.joins[`cohort_${cohortId}`] = `LEFT ANY JOIN ${cohortCte} AS ${cohortAlias} ON ${cohortAlias}.profile_id = e.profile_id`;
+ });
+
+ // When using cohort MV, swap column names and aggregate functions
+ const countExpr = useCohortMV ? 'countMerge(e.event_count)' : 'count(*)';
+ const dateExpr = useCohortMV ? 'toDateTime(e.event_date)' : 'created_at';
+
+ sb.select.count = `${countExpr} as count`;
+
switch (interval) {
case 'minute': {
sb.fill = `FROM toStartOfMinute(toDateTime('${startDate}')) TO toStartOfMinute(toDateTime('${endDate}')) STEP toIntervalMinute(1)`;
- sb.select.date = 'toStartOfMinute(created_at) as date';
+ sb.select.date = `toStartOfMinute(${dateExpr}) as date`;
break;
}
case 'hour': {
sb.fill = `FROM toStartOfHour(toDateTime('${startDate}')) TO toStartOfHour(toDateTime('${endDate}')) STEP toIntervalHour(1)`;
- sb.select.date = 'toStartOfHour(created_at) as date';
+ sb.select.date = `toStartOfHour(${dateExpr}) as date`;
break;
}
case 'day': {
sb.fill = `FROM toStartOfDay(toDateTime('${startDate}')) TO toStartOfDay(toDateTime('${endDate}')) STEP toIntervalDay(1)`;
- sb.select.date = 'toStartOfDay(created_at) as date';
+ sb.select.date = `toStartOfDay(${dateExpr}) as date`;
break;
}
case 'week': {
sb.fill = `FROM toStartOfWeek(toDateTime('${startDate}'), 1, '${timezone}') TO toStartOfWeek(toDateTime('${endDate}'), 1, '${timezone}') STEP toIntervalWeek(1)`;
- sb.select.date = `toStartOfWeek(created_at, 1, '${timezone}') as date`;
+ sb.select.date = `toStartOfWeek(${dateExpr}, 1, '${timezone}') as date`;
break;
}
case 'month': {
sb.fill = `FROM toStartOfMonth(toDateTime('${startDate}'), '${timezone}') TO toStartOfMonth(toDateTime('${endDate}'), '${timezone}') STEP toIntervalMonth(1)`;
- sb.select.date = `toStartOfMonth(created_at, '${timezone}') as date`;
+ sb.select.date = `toStartOfMonth(${dateExpr}, '${timezone}') as date`;
break;
}
}
@@ -219,52 +632,69 @@ export function getChartSql({
sb.orderBy.date = 'date ASC';
if (startDate) {
- sb.where.startDate = `created_at >= toDateTime('${formatClickhouseDate(startDate)}')`;
+ sb.where.startDate = useCohortMV
+ ? `event_date >= toDate('${formatClickhouseDate(startDate)}')`
+ : `created_at >= toDateTime('${formatClickhouseDate(startDate)}')`;
}
if (endDate) {
- sb.where.endDate = `created_at <= toDateTime('${formatClickhouseDate(endDate)}')`;
+ sb.where.endDate = useCohortMV
+ ? `event_date <= toDate('${formatClickhouseDate(endDate)}')`
+ : `created_at <= toDateTime('${formatClickhouseDate(endDate)}')`;
}
// Use CTE to define top breakdown values once, then reference in WHERE clause
if (breakdowns.length > 0 && limit) {
const breakdownSelects = breakdowns
- .map((b) => getSelectPropertyKey(b.name))
+ .map((b) => getSelectPropertyKey(b.name, projectId, b.cohortId))
.join(', ');
+ // Build cohort JOINs for top_breakdowns CTE
+ // NOTE: ClickHouse CTEs cannot reference other CTEs in JOINs, so we inline the subquery
+ const cohortJoinsForTop = cohortIds.map((cohortId) => {
+ const cohortMeta = cohortMetadata.get(cohortId);
+ return buildInlineCohortJoin(cohortId, projectId, 'e', cohortMeta);
+ }).join(' ');
+
+ // Determine data source: use cohort MV, custom event CTE, or regular events table
+ const dataSource = useCohortMV
+ ? TABLE_NAMES.profile_event_summary_mv
+ : customEvent ? 'custom_event_data' : TABLE_NAMES.events;
+ const orderByCount = useCohortMV ? 'countMerge(e.event_count)' : 'count(*)';
+
// Add top_breakdowns CTE using the builder
addCte(
'top_breakdowns',
`SELECT ${breakdownSelects}
- FROM ${TABLE_NAMES.events} e
- ${profilesJoinRef ? `${profilesJoinRef} ` : ''}${getWhereWithoutBar()}
+ FROM ${dataSource} AS e
+ ${profilesJoinRef ? `${profilesJoinRef} ` : ''}${cohortJoinsForTop ? `${cohortJoinsForTop} ` : ''}${getWhereWithoutBar()}
GROUP BY ${breakdownSelects}
- ORDER BY count(*) DESC
+ ORDER BY ${orderByCount} DESC
LIMIT ${limit}`,
);
// Filter main query to only include top breakdown values
- sb.where.bar = `(${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}) IN (SELECT * FROM top_breakdowns)`;
+ sb.where.bar = `(${breakdowns.map((b) => getSelectPropertyKey(b.name, projectId, b.cohortId)).join(',')}) IN (SELECT * FROM top_breakdowns)`;
}
breakdowns.forEach((breakdown, index) => {
// Breakdowns start at label_1 (label_0 is reserved for event name)
const key = `label_${index + 1}`;
- sb.select[key] = `${getSelectPropertyKey(breakdown.name)} as ${key}`;
+ sb.select[key] = `${getSelectPropertyKey(breakdown.name, projectId, breakdown.cohortId)} as ${key}`;
sb.groupBy[key] = `${key}`;
});
if (event.segment === 'user') {
- sb.select.count = 'countDistinct(profile_id) as count';
+ sb.select.count = 'uniq(e.profile_id) as count';
}
if (event.segment === 'session') {
- sb.select.count = 'countDistinct(session_id) as count';
+ sb.select.count = 'uniq(e.session_id) as count';
}
if (event.segment === 'user_average') {
sb.select.count =
- 'COUNT(*)::float / COUNT(DISTINCT profile_id)::float as count';
+ 'COUNT(*)::float / COUNT(DISTINCT e.profile_id)::float as count';
}
const mathFunction = {
@@ -275,7 +705,7 @@ export function getChartSql({
}[event.segment as string];
if (mathFunction && event.property) {
- const propertyKey = getSelectPropertyKey(event.property);
+ const propertyKey = getSelectPropertyKey(event.property, projectId);
if (isNumericColumn(event.property)) {
sb.select.count = `${mathFunction}(${propertyKey}) as count`;
@@ -303,42 +733,74 @@ export function getChartSql({
return sql;
}
- // Note: The profile CTE (if it exists) is available in subqueries, so we can reference it directly
if (breakdowns.length > 0) {
- // Match breakdown properties in subquery with outer query's grouped values
- // Since outer query groups by label_X, we reference those in the correlation
- const breakdownMatches = breakdowns
+ const breakdownSelects = breakdowns
.map((b, index) => {
- const propertyKey = getSelectPropertyKey(b.name);
- // Correlate: match the property expression with outer query's label_X value
- // ClickHouse allows referencing outer query columns in correlated subqueries
- return `${propertyKey} = label_${index + 1}`;
+ const propertyKey = getSelectPropertyKey(b.name, projectId, b.cohortId);
+ return `${propertyKey} as breakdown_${index + 1}`;
+ })
+ .join(', ');
+
+ const breakdownGroupBy = breakdowns
+ .map((b, index) => `breakdown_${index + 1}`)
+ .join(', ');
+
+ const totalCountWhere = getWhereWithoutBar();
+
+ // Determine data source: use cohort MV, custom event CTE, or regular events table
+ const dataSourceForBreakdown = useCohortMV
+ ? TABLE_NAMES.profile_event_summary_mv
+ : customEvent ? 'custom_event_data' : TABLE_NAMES.events;
+
+ // Build cohort JOINs for breakdown_totals CTE
+ // NOTE: ClickHouse CTEs cannot reference other CTEs in JOINs, so we inline the subquery
+ const cohortJoinsForBreakdown = cohortIds.map((cohortId) => {
+ const cohortMeta = cohortMetadata.get(cohortId);
+ return buildInlineCohortJoin(cohortId, projectId, dataSourceForBreakdown, cohortMeta);
+ }).join(' ');
+
+ addCte(
+ 'breakdown_totals',
+ `SELECT
+ ${breakdownSelects},
+ uniq(${dataSourceForBreakdown}.profile_id) as total_count
+ FROM ${dataSourceForBreakdown}
+ ${profilesJoinRefForCTE ? `${profilesJoinRefForCTE} ` : ''}${cohortJoinsForBreakdown ? `${cohortJoinsForBreakdown} ` : ''}${totalCountWhere}
+ GROUP BY ${breakdownGroupBy}`,
+ );
+
+ const joinConditions = breakdowns
+ .map((b, index) => {
+ const propertyKey = getSelectPropertyKey(b.name, projectId, b.cohortId);
+ return `breakdown_totals.breakdown_${index + 1} = ${propertyKey}`;
})
.join(' AND ');
- // Build WHERE clause for subquery - replace table alias and keep profile CTE reference
- const subqueryWhere = getWhereWithoutBar()
- .replace(/\be\./g, 'e2.')
- .replace(/\bprofile\./g, 'profile.');
-
- sb.select.total_unique_count = `(
- SELECT uniq(profile_id)
- FROM ${TABLE_NAMES.events} e2
- ${profilesJoinRef ? `${profilesJoinRef} ` : ''}${subqueryWhere}
- AND ${breakdownMatches}
- ) as total_count`;
+ sb.joins.breakdown_totals = `LEFT JOIN breakdown_totals ON ${joinConditions}`;
+ sb.select.total_unique_count = `any(breakdown_totals.total_count) as total_count`;
} else {
- // No breakdowns: calculate unique count across all data
- // Build WHERE clause for subquery - replace table alias and keep profile CTE reference
- const subqueryWhere = getWhereWithoutBar()
- .replace(/\be\./g, 'e2.')
- .replace(/\bprofile\./g, 'profile.');
+ const totalCountWhere = getWhereWithoutBar();
+
+ // Determine data source: use cohort MV, custom event CTE, or regular events table
+ const dataSourceForTotal = useCohortMV
+ ? TABLE_NAMES.profile_event_summary_mv
+ : customEvent ? 'custom_event_data' : TABLE_NAMES.events;
+
+ // Build cohort JOINs for total_unique CTE
+ // NOTE: ClickHouse CTEs cannot reference other CTEs in JOINs, so we inline the subquery
+ const cohortJoinsForTotal = cohortIds.map((cohortId) => {
+ const cohortMeta = cohortMetadata.get(cohortId);
+ return buildInlineCohortJoin(cohortId, projectId, dataSourceForTotal, cohortMeta);
+ }).join(' ');
+
+ addCte(
+ 'total_unique',
+ `SELECT uniq(${dataSourceForTotal}.profile_id) as total_count
+ FROM ${dataSourceForTotal}
+ ${profilesJoinRefForCTE ? `${profilesJoinRefForCTE} ` : ''}${cohortJoinsForTotal ? `${cohortJoinsForTotal} ` : ''}${totalCountWhere}`,
+ );
- sb.select.total_unique_count = `(
- SELECT uniq(profile_id)
- FROM ${TABLE_NAMES.events} e2
- ${profilesJoinRef ? `${profilesJoinRef} ` : ''}${subqueryWhere}
- ) as total_count`;
+ sb.select.total_unique_count = `(SELECT total_count FROM total_unique) as total_count`;
}
const sql = `${getWith()}${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;
@@ -353,11 +815,56 @@ function isNumericColumn(columnName: string): boolean {
return numericColumns.includes(columnName);
}
-export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
+/**
+ * Setup CTE for custom event expansion
+ * Modifies the sql builder to use custom event data instead of events table
+ */
+async function setupCustomEventCTE(
+ sb: any,
+ addCte: (name: string, query: string) => void,
+ customEvent: { name: string; definition: ICustomEventDefinition },
+ projectId: string,
+ startDate: string,
+ endDate: string,
+) {
+ const baseWhere: string[] = [];
+ if (startDate) {
+ baseWhere.push(`created_at >= toDateTime('${formatClickhouseDate(startDate)}')`);
+ }
+ if (endDate) {
+ baseWhere.push(`created_at <= toDateTime('${formatClickhouseDate(endDate)}')`);
+ }
+
+ const customEventSQL = await expandCustomEventToSQL(
+ { ...customEvent, projectId },
+ baseWhere,
+ );
+
+ addCte('custom_event_data', customEventSQL);
+ sb.from = 'custom_event_data';
+}
+
+export function getEventFiltersWhereClause(
+ filters: IChartEventFilter[],
+ projectId?: string,
+) {
const where: Record = {};
filters.forEach((filter, index) => {
const id = `f${index}`;
- const { name, value, operator } = filter;
+ const { name, value, operator, cohortId } = filter;
+
+ // Handle cohort operators - use JOIN-based approach
+ if (operator === 'inCohort' && cohortId && projectId) {
+ const cohortAlias = getCohortAlias(cohortId);
+ where[id] = `notEmpty(${cohortAlias}.profile_id)`;
+ return;
+ }
+
+ if (operator === 'notInCohort' && cohortId && projectId) {
+ const cohortAlias = getCohortAlias(cohortId);
+ where[id] = `empty(${cohortAlias}.profile_id)`;
+ return;
+ }
if (
value.length === 0 &&
@@ -369,9 +876,9 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
if (name === 'has_profile') {
if (value.includes('true')) {
- where[id] = 'profile_id != device_id';
+ where[id] = 'e.profile_id != e.device_id';
} else {
- where[id] = 'profile_id = device_id';
+ where[id] = 'e.profile_id = e.device_id';
}
return;
}
@@ -380,9 +887,9 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
name.startsWith('properties.') ||
name.startsWith('profile.properties.')
) {
- const propertyKey = getSelectPropertyKey(name);
+ const propertyKey = getSelectPropertyKey(name, projectId);
const isWildcard = propertyKey.includes('%');
- const whereFrom = getSelectPropertyKey(name);
+ const whereFrom = getSelectPropertyKey(name, projectId);
switch (operator) {
case 'is': {
@@ -742,6 +1249,30 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
return where;
}
+/**
+ * Generate WHERE clause for global cohort filters
+ * These filters apply to the entire chart, not just specific events
+ */
+export function getGlobalCohortFiltersWhereClause(
+ cohortFilters: Array<{ cohortId: string; operator: 'inCohort' | 'notInCohort' }>,
+ projectId: string,
+): string {
+ if (!cohortFilters || cohortFilters.length === 0) {
+ return '';
+ }
+
+ const conditions = cohortFilters.map((filter) => {
+ const subquery = getCohortMembershipSubquery(filter.cohortId);
+
+ return filter.operator === 'inCohort'
+ ? `profile_id IN (${subquery})`
+ : `profile_id NOT IN (${subquery})`;
+ });
+
+ // AND logic between multiple cohort filters
+ return conditions.join(' AND ');
+}
+
export function getChartStartEndDate(
{
startDate,
diff --git a/packages/db/src/services/cohort.service.test.ts b/packages/db/src/services/cohort.service.test.ts
new file mode 100644
index 000000000..4da9a72ae
--- /dev/null
+++ b/packages/db/src/services/cohort.service.test.ts
@@ -0,0 +1,420 @@
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import type {
+ CohortDefinition,
+ EventBasedCohortDefinition,
+ PropertyBasedCohortDefinition,
+} from '@openpanel/validation';
+import { ch } from '../clickhouse/client';
+import { db } from '../prisma-client';
+import {
+ computeCohort,
+ computeEventBasedCohort,
+ computePropertyBasedCohort,
+ getCohortCount,
+ getCohortMembers,
+ storeCohortMembership,
+ updateCohortMembership,
+} from './cohort.service';
+
+// Mock the ch and db modules
+vi.mock('../clickhouse/client', () => ({
+ ch: vi.fn(),
+ chQuery: vi.fn(),
+ TABLE_NAMES: {
+ events: 'events',
+ profiles: 'profiles',
+ cohort_members: 'cohort_members',
+ cohort_metadata: 'cohort_metadata',
+ profile_event_summary_mv: 'profile_event_summary_mv',
+ },
+}));
+
+vi.mock('../prisma-client', () => ({
+ db: {
+ cohort: {
+ findUnique: vi.fn(),
+ update: vi.fn(),
+ },
+ },
+}));
+
+const mockChQuery = vi.fn();
+vi.mock('../clickhouse/client', async () => {
+ const actual = await vi.importActual('../clickhouse/client');
+ return {
+ ...actual,
+ chQuery: mockChQuery,
+ };
+});
+
+describe('Cohort Service', () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ describe('computeEventBasedCohort', () => {
+ it('should compute cohort with single event criteria (OR)', async () => {
+ const definition: EventBasedCohortDefinition = {
+ type: 'event',
+ criteria: {
+ events: [
+ {
+ name: 'page_view',
+ filters: [],
+ timeframe: { type: 'relative', value: '30d' },
+ frequency: { operator: 'gte', value: 1 },
+ },
+ ],
+ operator: 'or',
+ },
+ };
+
+ mockChQuery.mockResolvedValueOnce([
+ { profile_id: 'user1' },
+ { profile_id: 'user2' },
+ ]);
+
+ const result = await computeEventBasedCohort('project-123', definition);
+
+ expect(result).toEqual(['user1', 'user2']);
+ expect(mockChQuery).toHaveBeenCalledTimes(1);
+ });
+
+ it('should compute cohort with multiple event criteria (AND)', async () => {
+ const definition: EventBasedCohortDefinition = {
+ type: 'event',
+ criteria: {
+ events: [
+ {
+ name: 'signup',
+ filters: [],
+ timeframe: { type: 'relative', value: '30d' },
+ },
+ {
+ name: 'purchase',
+ filters: [],
+ timeframe: { type: 'relative', value: '7d' },
+ frequency: { operator: 'gte', value: 1 },
+ },
+ ],
+ operator: 'and',
+ },
+ };
+
+ mockChQuery.mockResolvedValueOnce([
+ { profile_id: 'user1' },
+ { profile_id: 'user2' },
+ ]);
+
+ const result = await computeEventBasedCohort('project-123', definition);
+
+ expect(result).toEqual(['user1', 'user2']);
+ expect(mockChQuery).toHaveBeenCalledTimes(1);
+ // Verify INTERSECT is used for AND
+ const query = mockChQuery.mock.calls[0][0];
+ expect(query).toContain('INTERSECT');
+ });
+
+ it('should handle event criteria with filters', async () => {
+ const definition: EventBasedCohortDefinition = {
+ type: 'event',
+ criteria: {
+ events: [
+ {
+ name: 'page_view',
+ filters: [
+ {
+ id: 'f1',
+ name: 'path',
+ operator: 'is',
+ value: ['/pricing'],
+ },
+ ],
+ timeframe: { type: 'relative', value: '7d' },
+ frequency: { operator: 'gte', value: 3 },
+ },
+ ],
+ operator: 'or',
+ },
+ };
+
+ mockChQuery.mockResolvedValueOnce([{ profile_id: 'user1' }]);
+
+ const result = await computeEventBasedCohort('project-123', definition);
+
+ expect(result).toEqual(['user1']);
+ expect(mockChQuery).toHaveBeenCalledTimes(1);
+ });
+
+ it('should handle absolute timeframe', async () => {
+ const definition: EventBasedCohortDefinition = {
+ type: 'event',
+ criteria: {
+ events: [
+ {
+ name: 'signup',
+ filters: [],
+ timeframe: { type: 'absolute', value: '2024-01-01' },
+ },
+ ],
+ operator: 'or',
+ },
+ };
+
+ mockChQuery.mockResolvedValueOnce([{ profile_id: 'user1' }]);
+
+ const result = await computeEventBasedCohort('project-123', definition);
+
+ expect(result).toEqual(['user1']);
+ const query = mockChQuery.mock.calls[0][0];
+ expect(query).toContain('2024-01-01');
+ });
+ });
+
+ describe('computePropertyBasedCohort', () => {
+ it('should compute cohort with single property (OR)', async () => {
+ const definition: PropertyBasedCohortDefinition = {
+ type: 'property',
+ criteria: {
+ properties: [
+ {
+ id: 'p1',
+ name: 'email',
+ operator: 'isSet',
+ value: [],
+ },
+ ],
+ operator: 'or',
+ },
+ };
+
+ mockChQuery.mockResolvedValueOnce([
+ { profile_id: 'user1' },
+ { profile_id: 'user2' },
+ ]);
+
+ const result = await computePropertyBasedCohort(
+ 'project-123',
+ definition,
+ );
+
+ expect(result).toEqual(['user1', 'user2']);
+ expect(mockChQuery).toHaveBeenCalledTimes(1);
+ });
+
+ it('should compute cohort with multiple properties (AND)', async () => {
+ const definition: PropertyBasedCohortDefinition = {
+ type: 'property',
+ criteria: {
+ properties: [
+ {
+ id: 'p1',
+ name: 'plan',
+ operator: 'is',
+ value: ['premium'],
+ },
+ {
+ id: 'p2',
+ name: 'country',
+ operator: 'is',
+ value: ['US'],
+ },
+ ],
+ operator: 'and',
+ },
+ };
+
+ mockChQuery.mockResolvedValueOnce([{ profile_id: 'user1' }]);
+
+ const result = await computePropertyBasedCohort(
+ 'project-123',
+ definition,
+ );
+
+ expect(result).toEqual(['user1']);
+ expect(mockChQuery).toHaveBeenCalledTimes(1);
+ });
+ });
+
+ describe('computeCohort', () => {
+ it('should route to event-based computation', async () => {
+ const definition: CohortDefinition = {
+ type: 'event',
+ criteria: {
+ events: [
+ {
+ name: 'login',
+ filters: [],
+ timeframe: { type: 'relative', value: '7d' },
+ },
+ ],
+ operator: 'or',
+ },
+ };
+
+ mockChQuery.mockResolvedValueOnce([{ profile_id: 'user1' }]);
+
+ const result = await computeCohort('project-123', definition);
+
+ expect(result).toEqual(['user1']);
+ });
+
+ it('should route to property-based computation', async () => {
+ const definition: CohortDefinition = {
+ type: 'property',
+ criteria: {
+ properties: [
+ {
+ id: 'p1',
+ name: 'email',
+ operator: 'isSet',
+ value: [],
+ },
+ ],
+ operator: 'or',
+ },
+ };
+
+ mockChQuery.mockResolvedValueOnce([{ profile_id: 'user1' }]);
+
+ const result = await computeCohort('project-123', definition);
+
+ expect(result).toEqual(['user1']);
+ });
+ });
+
+ describe('storeCohortMembership', () => {
+ it('should store cohort members in ClickHouse', async () => {
+ mockChQuery.mockResolvedValueOnce(undefined);
+
+ await storeCohortMembership(
+ 'cohort-123',
+ 'project-123',
+ ['user1', 'user2', 'user3'],
+ );
+
+ expect(mockChQuery).toHaveBeenCalledTimes(1);
+ const query = mockChQuery.mock.calls[0][0];
+ expect(query).toContain('cohort_members');
+ expect(query).toContain('cohort-123');
+ expect(query).toContain('project-123');
+ });
+
+ it('should handle empty profile list', async () => {
+ mockChQuery.mockResolvedValueOnce(undefined);
+
+ await storeCohortMembership('cohort-123', 'project-123', []);
+
+ // Should still execute to clear old members
+ expect(mockChQuery).toHaveBeenCalledTimes(1);
+ });
+ });
+
+ describe('getCohortMembers', () => {
+ it('should retrieve cohort members with pagination', async () => {
+ mockChQuery.mockResolvedValueOnce([
+ { profile_id: 'user1' },
+ { profile_id: 'user2' },
+ ]);
+ mockChQuery.mockResolvedValueOnce([{ count: 100 }]);
+
+ const result = await getCohortMembers('cohort-123', 'project-123', {
+ limit: 50,
+ offset: 0,
+ });
+
+ expect(result.profileIds).toEqual(['user1', 'user2']);
+ expect(result.total).toBe(100);
+ expect(mockChQuery).toHaveBeenCalledTimes(2);
+ });
+
+ it('should respect pagination offset', async () => {
+ mockChQuery.mockResolvedValueOnce([{ profile_id: 'user51' }]);
+ mockChQuery.mockResolvedValueOnce([{ count: 100 }]);
+
+ const result = await getCohortMembers('cohort-123', 'project-123', {
+ limit: 1,
+ offset: 50,
+ });
+
+ expect(result.profileIds).toEqual(['user51']);
+ expect(result.total).toBe(100);
+ });
+ });
+
+ describe('getCohortCount', () => {
+ it('should return cached count if available', async () => {
+ mockChQuery.mockResolvedValueOnce([
+ {
+ member_count: 150,
+ computed_at: new Date(),
+ },
+ ]);
+
+ const count = await getCohortCount('cohort-123', 'project-123');
+
+ expect(count).toBe(150);
+ expect(mockChQuery).toHaveBeenCalledTimes(1);
+ });
+
+ it('should return 0 if no metadata found', async () => {
+ mockChQuery.mockResolvedValueOnce([]);
+
+ const count = await getCohortCount('cohort-123', 'project-123');
+
+ expect(count).toBe(0);
+ });
+ });
+
+ describe('updateCohortMembership', () => {
+ it('should recompute and store cohort membership', async () => {
+ const mockCohort = {
+ id: 'cohort-123',
+ projectId: 'project-123',
+ definition: {
+ type: 'event',
+ criteria: {
+ events: [
+ {
+ name: 'login',
+ filters: [],
+ timeframe: { type: 'relative', value: '7d' },
+ },
+ ],
+ operator: 'or',
+ },
+ } as CohortDefinition,
+ };
+
+ vi.mocked(db.cohort.findUnique).mockResolvedValueOnce(mockCohort as any);
+ mockChQuery.mockResolvedValueOnce([
+ { profile_id: 'user1' },
+ { profile_id: 'user2' },
+ ]);
+ mockChQuery.mockResolvedValueOnce(undefined); // store
+ mockChQuery.mockResolvedValueOnce(undefined); // metadata
+ vi.mocked(db.cohort.update).mockResolvedValueOnce(mockCohort as any);
+
+ await updateCohortMembership('cohort-123');
+
+ expect(db.cohort.findUnique).toHaveBeenCalledWith({
+ where: { id: 'cohort-123' },
+ });
+ expect(db.cohort.update).toHaveBeenCalledWith({
+ where: { id: 'cohort-123' },
+ data: expect.objectContaining({
+ profileCount: 2,
+ lastComputedAt: expect.any(Date),
+ }),
+ });
+ });
+
+ it('should throw error if cohort not found', async () => {
+ vi.mocked(db.cohort.findUnique).mockResolvedValueOnce(null);
+
+ await expect(updateCohortMembership('nonexistent')).rejects.toThrow(
+ 'Cohort not found',
+ );
+ });
+ });
+});
diff --git a/packages/db/src/services/cohort.service.ts b/packages/db/src/services/cohort.service.ts
new file mode 100644
index 000000000..d8b4b853e
--- /dev/null
+++ b/packages/db/src/services/cohort.service.ts
@@ -0,0 +1,645 @@
+import sqlstring from 'sqlstring';
+import type {
+ CohortDefinition,
+ EventBasedCohortDefinition,
+ EventCriteria,
+ Frequency,
+ PropertyBasedCohortDefinition,
+ Timeframe,
+} from '@openpanel/validation';
+import type { IChartEventFilter } from '@openpanel/validation';
+
+import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';
+import { db } from '../prisma-client';
+import { getEventFiltersWhereClause } from './chart.service';
+
+/**
+ * Build time constraint SQL from timeframe
+ */
+function buildTimeConstraint(timeframe: Timeframe): string {
+ if (timeframe.type === 'relative') {
+ // Parse relative time like "30d", "90d"
+ const match = timeframe.value.match(/^(\d+)d$/);
+ if (!match) {
+ throw new Error(`Invalid relative timeframe: ${timeframe.value}`);
+ }
+ const days = Number.parseInt(match[1]!, 10);
+ return `created_at >= now() - INTERVAL ${days} DAY`;
+ } else {
+ // Absolute time: start date and optional end date
+ const start = timeframe.start;
+ const end = timeframe.end || 'now()';
+
+ if (timeframe.end) {
+ return `created_at BETWEEN toDate('${start}') AND toDate('${end}')`;
+ } else {
+ // "Since date" - no end date
+ return `created_at >= toDate('${start}')`;
+ }
+ }
+}
+
+/**
+ * Convert frequency operator to SQL comparison
+ */
+function getFrequencyOperator(frequency: Frequency): string {
+ switch (frequency.operator) {
+ case 'at_least':
+ return `>= ${frequency.count}`;
+ case 'exactly':
+ return `= ${frequency.count}`;
+ case 'at_most':
+ return `<= ${frequency.count}`;
+ default:
+ return `>= ${frequency.count}`;
+ }
+}
+
+/**
+ * Build ClickHouse query for a single event criteria
+ */
+export function buildEventCriteriaQuery(
+ projectId: string,
+ criteria: EventCriteria,
+): string {
+ const { name, filters, timeframe, frequency } = criteria;
+
+ // Build time constraint
+ const timeConstraint = buildTimeConstraint(timeframe);
+
+ // Build event filters
+ const filterWhere = filters.length > 0
+ ? getEventFiltersWhereClause(filters)
+ : {};
+ const filterClauses = Object.values(filterWhere);
+ const filterClause = filterClauses.length > 0
+ ? `AND ${filterClauses.join(' AND ')}`
+ : '';
+
+ // Check if there are event property filters
+ const hasEventPropertyFilters = filters.some(
+ (f) => f.name.startsWith('properties.') && !f.name.startsWith('profile.properties.')
+ );
+
+ // Use property-aware MV for event property filters
+ if (hasEventPropertyFilters) {
+ const propertyFilters = filters.filter(
+ (f) => f.name.startsWith('properties.')
+ );
+
+ // Build WHERE conditions for property filters
+ const propertyConditions = propertyFilters.map((filter) => {
+ const propertyKey = filter.name.replace('properties.', '');
+ const { value, operator } = filter;
+
+ switch (operator) {
+ case 'is':
+ if (value.length === 1) {
+ return `(property_key = ${sqlstring.escape(propertyKey)} AND property_value = ${sqlstring.escape(String(value[0]).trim())})`;
+ }
+ return `(property_key = ${sqlstring.escape(propertyKey)} AND property_value IN (${value.map((val) => sqlstring.escape(String(val).trim())).join(', ')}))`;
+ case 'isNot':
+ if (value.length === 1) {
+ return `(property_key = ${sqlstring.escape(propertyKey)} AND property_value != ${sqlstring.escape(String(value[0]).trim())})`;
+ }
+ return `(property_key = ${sqlstring.escape(propertyKey)} AND property_value NOT IN (${value.map((val) => sqlstring.escape(String(val).trim())).join(', ')}))`;
+ case 'contains':
+ return `(property_key = ${sqlstring.escape(propertyKey)} AND (${value.map((val) => `property_value LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`).join(' OR ')}))`;
+ case 'doesNotContain':
+ return `(property_key = ${sqlstring.escape(propertyKey)} AND (${value.map((val) => `property_value NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`).join(' AND ')}))`;
+ default:
+ return `(property_key = ${sqlstring.escape(propertyKey)} AND property_value IN (${value.map((val) => sqlstring.escape(String(val).trim())).join(', ')}))`;
+ }
+ }).join(' OR ');
+
+ if (frequency) {
+ const frequencyOp = getFrequencyOperator(frequency);
+ return `
+ SELECT profile_id
+ FROM ${TABLE_NAMES.profile_event_property_summary_mv}
+ WHERE project_id = ${sqlstring.escape(projectId)}
+ AND name = ${sqlstring.escape(name)}
+ AND ${timeConstraint.replace('created_at', 'event_date')}
+ AND (${propertyConditions})
+ GROUP BY profile_id
+ HAVING countMerge(event_count) ${frequencyOp}
+ `;
+ }
+
+ return `
+ SELECT DISTINCT profile_id
+ FROM ${TABLE_NAMES.profile_event_property_summary_mv}
+ WHERE project_id = ${sqlstring.escape(projectId)}
+ AND name = ${sqlstring.escape(name)}
+ AND ${timeConstraint.replace('created_at', 'event_date')}
+ AND (${propertyConditions})
+ `;
+ }
+
+ // Use standard MV for queries without property filters
+ if (frequency) {
+ const frequencyOp = getFrequencyOperator(frequency);
+
+ return `
+ SELECT profile_id
+ FROM ${TABLE_NAMES.profile_event_summary_mv}
+ WHERE project_id = ${sqlstring.escape(projectId)}
+ AND name = ${sqlstring.escape(name)}
+ AND ${timeConstraint.replace('created_at', 'event_date')}
+ GROUP BY profile_id
+ HAVING countMerge(event_count) ${frequencyOp}
+ `;
+ }
+
+ // For simple "did event" queries
+ return `
+ SELECT DISTINCT profile_id
+ FROM ${TABLE_NAMES.profile_event_summary_mv}
+ WHERE project_id = ${sqlstring.escape(projectId)}
+ AND name = ${sqlstring.escape(name)}
+ AND ${timeConstraint.replace('created_at', 'event_date')}
+ `;
+}
+
+/**
+ * Build ClickHouse query for property-based cohort
+ * Returns a SELECT query string that can be embedded as a subquery
+ */
+export function buildPropertyBasedCohortQuery(
+ projectId: string,
+ definition: PropertyBasedCohortDefinition,
+): string {
+ const { properties, operator } = definition.criteria;
+
+ // Build property filters
+ const filterWhere = getProfileFiltersWhereClause(properties);
+ const filterClauses = Object.values(filterWhere);
+
+ if (filterClauses.length === 0) {
+ return `SELECT id as profile_id FROM ${TABLE_NAMES.profiles} FINAL WHERE 1=0`; // Empty result
+ }
+
+ const filterClause = filterClauses.join(
+ operator === 'and' ? ' AND ' : ' OR ',
+ );
+
+ return `
+ SELECT id as profile_id
+ FROM ${TABLE_NAMES.profiles} FINAL
+ WHERE project_id = ${sqlstring.escape(projectId)}
+ AND (${filterClause})
+ `;
+}
+
+/**
+ * Compute event-based cohort membership
+ * Returns array of profile IDs that match the criteria
+ * @param limit - Optional limit on number of profiles (default: no limit)
+ */
+export async function computeEventBasedCohort(
+ projectId: string,
+ definition: EventBasedCohortDefinition,
+ limit?: number,
+): Promise {
+ const { events, operator } = definition.criteria;
+
+ const queries = events.map((eventCriteria) => {
+ return buildEventCriteriaQuery(projectId, eventCriteria);
+ });
+
+ // Combine queries based on AND/OR operator
+ const combinedQuery =
+ operator === 'and'
+ ? queries.join(' INTERSECT ')
+ : queries.join(' UNION DISTINCT ');
+
+ // Apply limit in SQL query itself to prevent loading too much into memory
+ const finalQuery = limit ? `${combinedQuery} LIMIT ${limit}` : combinedQuery;
+
+ const results = await chQuery<{ profile_id: string }>(finalQuery);
+ return results.map((r) => r.profile_id);
+}
+
+/**
+ * Count cohort members without loading into memory
+ * Used for dynamic cohorts and previews
+ */
+export async function countEventBasedCohort(
+ projectId: string,
+ definition: EventBasedCohortDefinition,
+): Promise {
+ const { events, operator } = definition.criteria;
+
+ const queries = events.map((eventCriteria) => {
+ return buildEventCriteriaQuery(projectId, eventCriteria);
+ });
+
+ // Combine queries based on AND/OR operator
+ const combinedQuery =
+ operator === 'and'
+ ? queries.join(' INTERSECT ')
+ : queries.join(' UNION DISTINCT ');
+
+ // Use COUNT instead of loading all IDs - prevents OOM
+ const countQuery = `SELECT count() as count FROM (${combinedQuery})`;
+ const results = await chQuery<{ count: number }>(countQuery);
+ return results[0]?.count ?? 0;
+}
+
+/**
+ * Build profile filter WHERE clause
+ * Similar to getEventFiltersWhereClause but for profiles table
+ */
+function getProfileFiltersWhereClause(
+ filters: IChartEventFilter[],
+): Record {
+ const where: Record = {};
+
+ filters.forEach((filter, index) => {
+ const id = `pf${index}`;
+ const { name, value, operator } = filter;
+
+ if (
+ value.length === 0 &&
+ operator !== 'isNull' &&
+ operator !== 'isNotNull'
+ ) {
+ return;
+ }
+
+ // Determine the column access pattern
+ // Replace profile. with profiles. since we're querying the profiles table directly
+ const normalizedName = name.replace(/^profile\./, 'profiles.');
+ let columnAccess: string;
+
+ if (normalizedName.startsWith('profiles.properties.')) {
+ const propKey = normalizedName.replace('profiles.properties.', '');
+ columnAccess = `profiles.properties['${propKey}']`;
+ } else {
+ // For profiles.email, profiles.first_name, etc. or any other column
+ columnAccess = normalizedName;
+ }
+
+ // Build WHERE clause based on operator
+ switch (operator) {
+ case 'is': {
+ if (value.length === 1) {
+ where[id] = `${columnAccess} = ${sqlstring.escape(String(value[0]).trim())}`;
+ } else {
+ where[id] = `${columnAccess} IN (${value
+ .map((val) => sqlstring.escape(String(val).trim()))
+ .join(', ')})`;
+ }
+ break;
+ }
+ case 'isNot': {
+ if (value.length === 1) {
+ where[id] = `${columnAccess} != ${sqlstring.escape(String(value[0]).trim())}`;
+ } else {
+ where[id] = `${columnAccess} NOT IN (${value
+ .map((val) => sqlstring.escape(String(val).trim()))
+ .join(', ')})`;
+ }
+ break;
+ }
+ case 'contains': {
+ where[id] = `(${value
+ .map((val) => `${columnAccess} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`)
+ .join(' OR ')})`;
+ break;
+ }
+ case 'doesNotContain': {
+ where[id] = `(${value
+ .map((val) => `${columnAccess} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`)
+ .join(' OR ')})`;
+ break;
+ }
+ case 'startsWith': {
+ where[id] = `(${value
+ .map((val) => `${columnAccess} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`)
+ .join(' OR ')})`;
+ break;
+ }
+ case 'endsWith': {
+ where[id] = `(${value
+ .map((val) => `${columnAccess} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`)
+ .join(' OR ')})`;
+ break;
+ }
+ case 'isNull': {
+ where[id] = `(${columnAccess} IS NULL OR ${columnAccess} = '')`;
+ break;
+ }
+ case 'isNotNull': {
+ where[id] = `(${columnAccess} IS NOT NULL AND ${columnAccess} != '')`;
+ break;
+ }
+ case 'gt': {
+ if (value[0] !== undefined) {
+ where[id] = `toFloat64OrNull(${columnAccess}) > ${Number(value[0])}`;
+ }
+ break;
+ }
+ case 'lt': {
+ if (value[0] !== undefined) {
+ where[id] = `toFloat64OrNull(${columnAccess}) < ${Number(value[0])}`;
+ }
+ break;
+ }
+ case 'gte': {
+ if (value[0] !== undefined) {
+ where[id] = `toFloat64OrNull(${columnAccess}) >= ${Number(value[0])}`;
+ }
+ break;
+ }
+ case 'lte': {
+ if (value[0] !== undefined) {
+ where[id] = `toFloat64OrNull(${columnAccess}) <= ${Number(value[0])}`;
+ }
+ break;
+ }
+ }
+ });
+
+ return where;
+}
+
+/**
+ * Compute property-based cohort membership
+ */
+export async function computePropertyBasedCohort(
+ projectId: string,
+ definition: PropertyBasedCohortDefinition,
+ limit?: number,
+): Promise {
+ const { properties, operator } = definition.criteria;
+
+ // Build property filters
+ const filterWhere = getProfileFiltersWhereClause(properties);
+ const filterClauses = Object.values(filterWhere);
+
+ if (filterClauses.length === 0) {
+ return [];
+ }
+
+ const filterClause = filterClauses.join(
+ operator === 'and' ? ' AND ' : ' OR ',
+ );
+
+ const query = `
+ SELECT id as profile_id
+ FROM ${TABLE_NAMES.profiles} FINAL
+ WHERE project_id = ${sqlstring.escape(projectId)}
+ AND (${filterClause})
+ ${limit ? `LIMIT ${limit}` : ''}
+ `;
+
+ const results = await chQuery<{ profile_id: string }>(query);
+ return results.map((r) => r.profile_id);
+}
+
+/**
+ * Count property-based cohort members without loading into memory
+ */
+export async function countPropertyBasedCohort(
+ projectId: string,
+ definition: PropertyBasedCohortDefinition,
+): Promise {
+ const { properties, operator } = definition.criteria;
+
+ // Build property filters
+ const filterWhere = getProfileFiltersWhereClause(properties);
+ const filterClauses = Object.values(filterWhere);
+
+ if (filterClauses.length === 0) {
+ return 0;
+ }
+
+ const filterClause = filterClauses.join(
+ operator === 'and' ? ' AND ' : ' OR ',
+ );
+
+ const query = `
+ SELECT count() as count
+ FROM ${TABLE_NAMES.profiles} FINAL
+ WHERE project_id = ${sqlstring.escape(projectId)}
+ AND (${filterClause})
+ `;
+
+ const results = await chQuery<{ count: number }>(query);
+ return results[0]?.count ?? 0;
+}
+
+/**
+ * Store cohort membership in ClickHouse
+ */
+export async function storeCohortMembership(
+ projectId: string,
+ cohortId: string,
+ profileIds: string[],
+ version: number,
+): Promise {
+ if (profileIds.length === 0) return;
+
+ // Use JSONEachRow format for better Map type handling
+ const now = new Date().toISOString().slice(0, 19).replace('T', ' ');
+ const data = profileIds.map((profileId) => ({
+ project_id: projectId,
+ cohort_id: cohortId,
+ profile_id: profileId,
+ matched_at: now,
+ matching_properties: {},
+ version: version,
+ }));
+
+ await ch.insert({
+ table: TABLE_NAMES.cohort_members,
+ values: data,
+ format: 'JSONEachRow',
+ });
+
+ // Update metadata
+ const sampleProfiles = profileIds.slice(0, 10);
+ await ch.insert({
+ table: TABLE_NAMES.cohort_metadata,
+ values: [{
+ project_id: projectId,
+ cohort_id: cohortId,
+ member_count: profileIds.length,
+ last_computed_at: new Date().toISOString().slice(0, 19).replace('T', ' '),
+ sample_profiles: sampleProfiles,
+ version: version,
+ }],
+ format: 'JSONEachRow',
+ });
+}
+
+/**
+ * Get cohort members with pagination
+ */
+export async function getCohortMembers(
+ cohortId: string,
+ projectId: string,
+ opts?: { limit?: number; offset?: number },
+): Promise<{ profileIds: string[]; total: number }> {
+ const cohort = await db.cohort.findUnique({ where: { id: cohortId } });
+
+ if (!cohort) {
+ throw new Error('Cohort not found');
+ }
+
+ // If stored in ClickHouse, query from cohort_members table
+ if (!cohort.computeOnDemand) {
+ const query = `
+ SELECT
+ profile_id,
+ count() OVER() as total
+ FROM ${TABLE_NAMES.cohort_members} FINAL
+ WHERE project_id = ${sqlstring.escape(projectId)}
+ AND cohort_id = ${sqlstring.escape(cohortId)}
+ ORDER BY matched_at DESC
+ ${opts?.limit ? `LIMIT ${opts.limit}` : ''}
+ ${opts?.offset ? `OFFSET ${opts.offset}` : ''}
+ `;
+
+ const results = await chQuery<{ profile_id: string; total: number }>(
+ query,
+ );
+ return {
+ profileIds: results.map((r) => r.profile_id),
+ total: results[0]?.total || 0,
+ };
+ }
+
+ // Otherwise, compute on-demand
+ const definition = cohort.definition as CohortDefinition;
+ const profileIds = await computeCohort(projectId, definition);
+
+ return {
+ profileIds: profileIds.slice(
+ opts?.offset || 0,
+ (opts?.offset || 0) + (opts?.limit || 50),
+ ),
+ total: profileIds.length,
+ };
+}
+
+/**
+ * Get cohort count
+ */
+export async function getCohortCount(
+ cohortId: string,
+ projectId: string,
+): Promise {
+ const cohort = await db.cohort.findUnique({ where: { id: cohortId } });
+
+ if (!cohort) {
+ throw new Error('Cohort not found');
+ }
+
+ // Check if cached in PostgreSQL
+ if (cohort.profileCount && cohort.lastComputedAt) {
+ const age = Date.now() - cohort.lastComputedAt.getTime();
+ if (age < 15 * 60 * 1000) {
+ // 15 minutes
+ return cohort.profileCount;
+ }
+ }
+
+ // Query from ClickHouse
+ if (!cohort.computeOnDemand) {
+ const result = await chQuery<{ count: number }>(`
+ SELECT count() as count
+ FROM ${TABLE_NAMES.cohort_members} FINAL
+ WHERE project_id = ${sqlstring.escape(projectId)}
+ AND cohort_id = ${sqlstring.escape(cohortId)}
+ `);
+ return result[0]?.count || 0;
+ }
+
+ // Compute on-demand using COUNT query - don't load all profile IDs into memory
+ const definition = cohort.definition as CohortDefinition;
+ return await countCohort(projectId, definition);
+}
+
+/**
+ * Main compute function - routes to correct implementation
+ */
+export async function computeCohort(
+ projectId: string,
+ definition: CohortDefinition,
+ limit?: number,
+): Promise {
+ if (definition.type === 'event') {
+ return computeEventBasedCohort(projectId, definition, limit);
+ } else if (definition.type === 'property') {
+ return computePropertyBasedCohort(projectId, definition, limit);
+ }
+ return [];
+}
+
+/**
+ * Count cohort members without loading into memory
+ * Works for both event and property-based cohorts
+ */
+export async function countCohort(
+ projectId: string,
+ definition: CohortDefinition,
+): Promise {
+ if (definition.type === 'event') {
+ return countEventBasedCohort(projectId, definition);
+ } else if (definition.type === 'property') {
+ return countPropertyBasedCohort(projectId, definition);
+ }
+ return 0;
+}
+
+/**
+ * Update cohort membership (for dynamic cohorts)
+ */
+export async function updateCohortMembership(
+ cohortId: string,
+): Promise {
+ const cohort = await db.cohort.findUnique({ where: { id: cohortId } });
+
+ if (!cohort || cohort.isStatic || cohort.computeOnDemand) {
+ return; // Skip static and on-demand cohorts
+ }
+
+ const definition = cohort.definition as CohortDefinition;
+ // Limit static cohorts to 10K profiles to prevent OOM
+ const profileIds = await computeCohort(cohort.projectId, definition, 10000);
+
+ // Increment version for ReplacingMergeTree
+ const version = Date.now();
+
+ // Store new membership
+ await storeCohortMembership(
+ cohort.projectId,
+ cohort.id,
+ profileIds,
+ version,
+ );
+
+ // Update cache in PostgreSQL
+ await db.cohort.update({
+ where: { id: cohortId },
+ data: {
+ profileCount: profileIds.length,
+ lastComputedAt: new Date(),
+ },
+ });
+}
+
+/**
+ * Get profiles in cohort as a Set (for filtering)
+ */
+export async function getProfilesInCohort(
+ cohortId: string,
+ projectId: string,
+): Promise> {
+ const { profileIds } = await getCohortMembers(cohortId, projectId, {
+ limit: 100000, // Large limit for filtering
+ });
+ return new Set(profileIds);
+}
diff --git a/packages/db/src/services/conversion.service.ts b/packages/db/src/services/conversion.service.ts
index f7e7b86b9..9a7fcc709 100644
--- a/packages/db/src/services/conversion.service.ts
+++ b/packages/db/src/services/conversion.service.ts
@@ -1,17 +1,225 @@
import { NOT_SET_VALUE } from '@openpanel/constants';
import type { IChartEvent, IChartInput } from '@openpanel/validation';
import { omit } from 'ramda';
-import { TABLE_NAMES, ch } from '../clickhouse/client';
+import { TABLE_NAMES, ch, formatClickhouseDate } from '../clickhouse/client';
import { clix } from '../clickhouse/query-builder';
import {
getEventFiltersWhereClause,
getSelectPropertyKey,
+ fetchCohortsMetadata,
+ getCohortCteName,
+ getCohortAlias,
+ buildCohortMembershipQuery,
+ getMaterializedColumns,
} from './chart.service';
import { onlyReportEvents } from './reports.service';
+import { getCustomEventByName, expandCustomEventToSQL } from './custom-event.service';
+
+const quoteCol = (col: string) => `\`${col.replace(/^`|`$/g, '')}\``;
export class ConversionService {
constructor(private client: typeof ch) {}
+ /**
+ * Helper to build breakdown column with table alias
+ * Handles property keys, profile fields, and cohort expressions
+ */
+ private getBreakdownColumnWithAlias(
+ breakdownName: string,
+ projectId: string,
+ cohortId: string | undefined,
+ tableAlias: string,
+ ): string {
+ const propertyKey = getSelectPropertyKey(breakdownName, projectId, cohortId);
+
+ // Cohort expressions already have their own aliases (e.g., cohort_abc123.profile_id)
+ if (propertyKey.includes('cohort_') || propertyKey.startsWith('if(')) {
+ return propertyKey;
+ }
+
+ // Profile fields are already qualified (e.g., profile.created_at)
+ if (propertyKey.startsWith('profile.')) {
+ return propertyKey;
+ }
+
+ // For property fields, prepend table alias
+ // e.g., properties['key'] -> se.properties['key']
+ if (propertyKey.startsWith('properties[') || propertyKey.includes('arrayMap')) {
+ return `${tableAlias}.${propertyKey}`;
+ }
+
+ // For simple fields (e.g., name, country), prepend alias
+ return `${tableAlias}.${propertyKey}`;
+ }
+
+ /**
+ * Build CTE for a single event (start or end of funnel)
+ * Handles both regular events and custom events
+ */
+ private async buildSingleEventCte(
+ event: IChartEvent,
+ cteName: string,
+ projectId: string,
+ startDate: string,
+ endDate: string,
+ extraColumns: string[] = [],
+ groupCol: string,
+ ): Promise {
+ // Check if this is a custom event
+ const customEvent = await getCustomEventByName(event.name, projectId);
+
+ if (customEvent) {
+ // Custom event - expand to SQL
+ const baseWhere = [
+ `created_at >= toDateTime('${formatClickhouseDate(startDate)}')`,
+ `created_at <= toDateTime('${formatClickhouseDate(endDate)}')`,
+ ];
+
+ const sql = await expandCustomEventToSQL(
+ {
+ name: customEvent.name,
+ projectId,
+ definition: customEvent.definition as any,
+ },
+ baseWhere,
+ );
+
+ // Wrap to only expose columns needed downstream — prevents broadcasting
+ // all materialized columns into FillingRightJoinSide × 32 copies
+ const baseColumns = ['profile_id', 'session_id', 'created_at'];
+ const neededColumns = [...new Set([...baseColumns, ...extraColumns])];
+ const selectList = neededColumns.map(quoteCol).join(', ');
+ return `${cteName} AS (SELECT ${selectList} FROM (${sql}) WHERE ${groupCol} != '')`;
+ } else {
+ // Regular event - apply filters if present
+ // Exclude cohort filters — they're handled via JOINs in the outer query
+ const nonCohortFilters = (event.filters ?? []).filter(
+ f => f.operator !== 'inCohort' && f.operator !== 'notInCohort'
+ );
+ const filterClauses = nonCohortFilters.length > 0
+ ? Object.values(getEventFiltersWhereClause(nonCohortFilters, projectId))
+ : [];
+ const filterWhere = filterClauses.length > 0
+ ? ' AND ' + filterClauses.join(' AND ')
+ : '';
+
+ // If any filter references profile.*, join the profiles table inside the CTE
+ const profileFilters = (event.filters || []).filter(f => f.name.startsWith('profile.'));
+ let profileJoinClause = '';
+ if (profileFilters.length > 0) {
+ const profileColumns = [...new Set(
+ profileFilters.map(f => f.name.replace('profile.', '').split('.')[0])
+ )];
+ profileJoinClause = `\n LEFT JOIN (SELECT id, ${profileColumns.join(', ')} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = '${projectId}') AS profile ON profile.id = profile_id`;
+ }
+
+ // Minimal SELECT: only the columns actually needed downstream
+ const baseColumns = ['profile_id', 'session_id', 'created_at'];
+ const selectColumns = [...new Set([...baseColumns, ...extraColumns])];
+ const selectList = selectColumns.map(quoteCol).join(', ');
+
+ return `${cteName} AS (
+ SELECT ${selectList}
+ FROM ${TABLE_NAMES.events}${profileJoinClause}
+ WHERE project_id = '${projectId}'
+ AND name = '${event.name}'
+ AND ${groupCol} != ''
+ AND created_at >= toDateTime('${formatClickhouseDate(startDate)}')
+ AND created_at <= toDateTime('${formatClickhouseDate(endDate)}')${filterWhere}
+ )`;
+ }
+ }
+
+ /**
+ * Build events source for conversion query
+ * Handles both regular events and custom events
+ * Supports N events (not just 2)
+ * @deprecated Use buildSingleEventCte instead for optimized self-join approach
+ */
+ private async buildEventsSource(
+ events: IChartEvent[],
+ projectId: string,
+ startDate: string,
+ endDate: string,
+ ): Promise<{
+ fromClause: string;
+ ctes: string[];
+ needsDateFilter: boolean;
+ }> {
+ // Check if any events are custom events
+ const customEvents = await Promise.all(
+ events.map(event => getCustomEventByName(event.name, projectId))
+ );
+
+ // If no custom events, use regular events table
+ if (customEvents.every(ce => !ce)) {
+ return {
+ fromClause: TABLE_NAMES.events,
+ ctes: [],
+ needsDateFilter: true,
+ };
+ }
+
+ // Get materialized columns to ensure UNION compatibility (events table only)
+ const materializedColumns = await getMaterializedColumns('events');
+ const materializedColumnNames = Object.values(materializedColumns);
+ const materializedColumnsSelect = materializedColumnNames.length > 0
+ ? `, ${materializedColumnNames.map(col => `\`${col}\``).join(', ')}`
+ : '';
+
+ // Build CTEs for custom events
+ const ctes: string[] = [];
+ const baseWhere = [
+ `created_at >= toDateTime('${formatClickhouseDate(startDate)}')`,
+ `created_at <= toDateTime('${formatClickhouseDate(endDate)}')`,
+ ];
+
+ // Build CTE for each custom event
+ const customEventQueries = await Promise.all(
+ customEvents.map(async (customEvent, index) => {
+ if (customEvent) {
+ const sql = await expandCustomEventToSQL(
+ {
+ name: customEvent.name,
+ projectId,
+ definition: customEvent.definition as any,
+ },
+ baseWhere,
+ );
+ return `custom_event_${index} AS (${sql})`;
+ }
+ return null;
+ })
+ );
+
+ ctes.push(...customEventQueries.filter((q): q is string => q !== null));
+
+ // Build union of custom and regular events
+ const unionParts: string[] = [];
+
+ events.forEach((event, index) => {
+ if (customEvents[index]) {
+ unionParts.push(`SELECT * FROM custom_event_${index}`);
+ } else {
+ // Regular event - include materialized columns to match custom events
+ unionParts.push(`
+ SELECT *${materializedColumnsSelect} FROM ${TABLE_NAMES.events}
+ WHERE project_id = '${projectId}'
+ AND name = '${event.name}'
+ AND created_at BETWEEN toDateTime('${startDate}') AND toDateTime('${endDate}')
+ `);
+ }
+ });
+
+ ctes.push(`combined_events AS (${unionParts.join(' UNION ALL ')})`);
+
+ return {
+ fromClause: 'combined_events',
+ ctes,
+ needsDateFilter: false, // Already filtered in CTEs
+ };
+ }
+
async getConversion({
projectId,
startDate,
@@ -20,115 +228,367 @@ export class ConversionService {
funnelWindow = 24,
series,
breakdowns = [],
+ holdProperties = [],
+ globalFilters = [],
+ measuring = 'conversion_rate',
limit,
interval,
timezone,
}: Omit & {
timezone: string;
}) {
- const group = funnelGroup === 'profile_id' ? 'profile_id' : 'session_id';
- const breakdownColumns = breakdowns.map(
- (b, index) => `${getSelectPropertyKey(b.name)} as b_${index}`,
+ // Merge global filters into each event's filters (same as fetch.ts does for regular charts)
+ const events = onlyReportEvents(series).map(event => ({
+ ...event,
+ filters: [...(event.filters ?? []), ...globalFilters],
+ }));
+
+ if (events.length < 2) {
+ throw new Error('events must be at least 2 events');
+ }
+
+ if (!startDate || !endDate) {
+ throw new Error('startDate and endDate are required');
+ }
+
+ // Extract cohort IDs from breakdowns and event filters (deduplicated)
+ const cohortIdsSet = new Set();
+ breakdowns?.forEach((b) => {
+ if (b.cohortId) {
+ cohortIdsSet.add(b.cohortId);
+ } else if (b.name.startsWith('cohort:')) {
+ cohortIdsSet.add(b.name.split(':')[1]!);
+ }
+ });
+ events.forEach((event) => {
+ event.filters?.forEach((filter) => {
+ if (filter.cohortId) {
+ cohortIdsSet.add(filter.cohortId);
+ }
+ });
+ });
+
+ const cohortIds = Array.from(cohortIdsSet);
+
+ // Fetch cohort metadata from Postgres (always fresh, no cache)
+ const cohortMetadata = await fetchCohortsMetadata(cohortIds);
+
+ const funnelWindowSeconds = funnelWindow * 3600;
+
+ // Use first and last events for conversion tracking
+ const firstEvent = events[0]!;
+ const lastEvent = events[events.length - 1]!;
+
+ // Calculate extended end date for conversion events (add funnel window)
+ const endDateObj = new Date(endDate);
+ const extendedEndDateObj = new Date(endDateObj.getTime() + funnelWindowSeconds * 1000);
+ const extendedEndDate = formatClickhouseDate(extendedEndDateObj);
+
+ // Ensure materialized columns cache is warm so getSelectPropertyKey works synchronously
+ await getMaterializedColumns('events');
+
+ // Determine which event-property columns are needed from start_events
+ // (profile.* and cohort breakdowns are handled separately via JOINs)
+ const breakdownExtraCols = breakdowns
+ .filter(b => !b.name.startsWith('profile.') && !b.cohortId && !b.name.startsWith('cohort:'))
+ .flatMap(b => {
+ const col = getSelectPropertyKey(b.name, projectId, undefined);
+ if (col.startsWith('profile.') || col.startsWith('if(')) return [];
+ // Map access (not materialized) — need the whole properties map
+ if (col.startsWith('properties[')) return ['properties'];
+ return [col];
+ });
+
+ // Hold property constant: columns needed in both CTEs for the JOIN condition
+ const holdExtraCols = holdProperties.flatMap(prop => {
+ const col = getSelectPropertyKey(prop, projectId, undefined);
+ if (col.startsWith('properties[')) return ['properties'];
+ return [col];
+ });
+
+ const startExtraCols = [...new Set([...breakdownExtraCols, ...holdExtraCols])];
+ const endExtraCols = [...new Set(holdExtraCols)];
+
+ // Define group column (profile_id or session_id) — needed by CTE builders below
+ const groupCol = funnelGroup === 'profile_id' ? 'profile_id' : 'session_id';
+
+ // Build CTEs for start and end events
+ const ctes: string[] = [];
+
+ // Start events CTE — named _raw so we can wrap it with deduplication below
+ const startEventCte = await this.buildSingleEventCte(
+ firstEvent,
+ 'start_events_raw',
+ projectId,
+ startDate,
+ endDate,
+ startExtraCols,
+ groupCol,
+ );
+ ctes.push(startEventCte);
+
+ // Deduplicate to one row per (groupCol, day) using the earliest created_at.
+ // A user who triggers the start event 50 times a day produces 50 rows — all
+ // collapse to the same uniqExact bucket anyway. Deduplicating here shrinks
+ // the ASOF JOIN left side proportionally, which matters a lot for 30d ranges.
+ const otherIdCol = groupCol === 'profile_id' ? 'session_id' : 'profile_id';
+ const safeGroupByCols = startExtraCols.filter(c => c !== 'properties');
+ const anyWrapCols = startExtraCols.filter(c => c === 'properties');
+ // GROUP BY uses _day (pre-computed in the subquery) instead of toDate(created_at)
+ // to avoid ClickHouse resolving 'created_at' in toDate(created_at) to the SELECT
+ // alias min(created_at) AS created_at — which would put an aggregate in GROUP BY.
+ const dedupeGroupBy = [quoteCol(groupCol), '_day', ...safeGroupByCols.map(quoteCol)].join(', ');
+ const dedupeSelect = [
+ quoteCol(groupCol),
+ `any(${quoteCol(otherIdCol)}) AS ${quoteCol(otherIdCol)}`,
+ 'min(created_at) AS created_at',
+ ...safeGroupByCols.map(quoteCol),
+ ...anyWrapCols.map(c => `any(${quoteCol(c)}) AS ${quoteCol(c)}`),
+ ].join(', ');
+ ctes.push(`start_events AS (
+ SELECT ${dedupeSelect}
+ FROM (SELECT *, toDate(created_at) AS _day FROM start_events_raw)
+ GROUP BY ${dedupeGroupBy}
+ )`);
+ // End events CTE — needs hold property columns for the JOIN condition
+ const endEventCte = await this.buildSingleEventCte(
+ lastEvent,
+ 'end_events',
+ projectId,
+ startDate,
+ extendedEndDate,
+ endExtraCols,
+ groupCol,
);
+ ctes.push(endEventCte);
+
+ // Add cohort CTEs (computed once per query, not per row)
+ cohortIds.forEach((cohortId) => {
+ const cohortMeta = cohortMetadata.get(cohortId);
+ const cohortQuery = buildCohortMembershipQuery(cohortId, projectId, cohortMeta);
+ ctes.push(`${getCohortCteName(cohortId)} AS (${cohortQuery})`);
+ });
+
+ // Build breakdown columns (from start_events with 'se' alias)
+ const breakdownColumns = breakdowns.map((b, index) => {
+ const columnWithAlias = this.getBreakdownColumnWithAlias(b.name, projectId, b.cohortId, 'se');
+ return `${columnWithAlias} as b_${index}`;
+ });
const breakdownGroupBy = breakdowns.map((b, index) => `b_${index}`);
- const events = onlyReportEvents(series);
+ // Build LEFT JOINs for cohorts (on start_events)
+ const cohortJoins = cohortIds.length > 0 ? '\n ' + cohortIds.map((cohortId) => {
+ const cohortAlias = getCohortAlias(cohortId);
+ const cohortCte = getCohortCteName(cohortId);
+ return `LEFT ANY JOIN ${cohortCte} AS ${cohortAlias} ON ${cohortAlias}.profile_id = se.profile_id`;
+ }).join('\n ') : '';
- if (events.length !== 2) {
- throw new Error('events must be an array of two events');
+ // Build LEFT JOIN for profile table if any breakdown uses profile.*
+ const profileBreakdowns = breakdowns.filter(b => b.name.startsWith('profile.'));
+ let profileJoin = '';
+ if (profileBreakdowns.length > 0) {
+ const matCols = await getMaterializedColumns('profiles');
+ const profileColumns = [...new Set(
+ profileBreakdowns.map(b => {
+ if (b.name.startsWith('profile.properties.')) {
+ const cached = matCols[b.name];
+ if (cached) {
+ // cached = "profile.campaign" -> select "campaign" (the materialized column)
+ return cached.replace('profile.', '');
+ }
+ }
+ // Fall back to the top-level field name (e.g., "properties", "email")
+ return b.name.replace('profile.', '').split('.')[0];
+ })
+ )];
+ profileJoin = `\n LEFT JOIN (SELECT id, ${profileColumns.join(', ')} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = '${projectId}') AS profile ON profile.id = se.profile_id`;
}
- if (!startDate || !endDate) {
- throw new Error('startDate and endDate are required');
+ // Grace period for events that fire very close together (like Mixpanel)
+ // Allows end event to happen up to 2 seconds before start event
+ const gracePeriodSeconds = 2;
+
+ // Hold property constant: require same property value in start and end events
+ const holdJoinConditions = holdProperties.map(prop => {
+ const col = getSelectPropertyKey(prop, projectId, undefined);
+ return `AND se.${col} = ee.${col}`;
+ }).join('\n ');
+
+ const toStartOf = clix.toStartOf('se.created_at', interval);
+ const breakdownGroupByStr = breakdownGroupBy.join(', ');
+
+ // Time diff column for time-to-convert measuring
+ const timeDiffCol = measuring === 'time_to_convert'
+ ? `,\n IF(ee.${groupCol} != '' AND ee.${groupCol} IS NOT NULL, dateDiff('second', se.created_at, ee.created_at), NULL) AS time_diff_seconds`
+ : '';
+
+ // Build WHERE clause for cohort global filters (inCohort / notInCohort)
+ const cohortFilterClauses = events.flatMap(event =>
+ (event.filters ?? [])
+ .filter(f => (f.operator === 'inCohort' || f.operator === 'notInCohort') && f.cohortId)
+ .map(f => {
+ const alias = getCohortAlias(f.cohortId!);
+ return f.operator === 'inCohort'
+ ? `notEmpty(${alias}.profile_id)`
+ : `empty(${alias}.profile_id)`;
+ })
+ );
+ // Deduplicate (same filter merged into multiple events)
+ const uniqueCohortFilterClauses = [...new Set(cohortFilterClauses)];
+
+ // Inner SELECT: raw per-event rows from the self-join (no aggregation yet)
+ // Build WHERE for inner query: ASOF upper bound + any cohort filters
+ const innerWhereConditions = [
+ `(ee.created_at IS NULL OR ee.created_at <= se.created_at + INTERVAL ${funnelWindowSeconds} SECOND)`,
+ ...uniqueCohortFilterClauses,
+ ];
+ const innerWhere = `\n WHERE ${innerWhereConditions.join('\n AND ')}`;
+
+ const innerSQL = `
+ SELECT
+ ${toStartOf} AS event_day,
+ se.${groupCol} AS ${groupCol},
+ ee.${groupCol} AS conversion_${groupCol}${breakdownColumns.length ? ',\n ' + breakdownColumns.join(',\n ') : ''}${timeDiffCol}
+ FROM start_events se${profileJoin}
+ LEFT ASOF JOIN end_events ee ON
+ ee.${groupCol} = se.${groupCol}
+ ${holdJoinConditions}
+ AND ee.created_at >= se.created_at - INTERVAL ${gracePeriodSeconds} SECOND
+ ${cohortJoins}${innerWhere}`;
+
+ // TTC aggregation columns (all computed in single scan, negligible extra cost)
+ const ttcAggColumns = measuring === 'time_to_convert'
+ ? `,
+ round(avgIf(time_diff_seconds, time_diff_seconds IS NOT NULL)) AS ttc_avg,
+ round(quantileIf(0.5)(time_diff_seconds, time_diff_seconds IS NOT NULL)) AS ttc_median,
+ minIf(time_diff_seconds, time_diff_seconds IS NOT NULL) AS ttc_min,
+ maxIf(time_diff_seconds, time_diff_seconds IS NOT NULL) AS ttc_max,
+ round(quantileIf(0.25)(time_diff_seconds, time_diff_seconds IS NOT NULL)) AS ttc_p25,
+ round(quantileIf(0.75)(time_diff_seconds, time_diff_seconds IS NOT NULL)) AS ttc_p75,
+ round(quantileIf(0.9)(time_diff_seconds, time_diff_seconds IS NOT NULL)) AS ttc_p90,
+ round(quantileIf(0.99)(time_diff_seconds, time_diff_seconds IS NOT NULL)) AS ttc_p99`
+ : '';
+
+ // agg CTE: aggregate inner rows into (event_day, breakdowns) buckets
+ const aggCte = `agg AS (
+ SELECT
+ event_day,
+ ${breakdownGroupBy.length ? breakdownGroupByStr + ',\n ' : ''}uniqExact(${groupCol}) AS total_first,
+ uniqExact(conversion_${groupCol}) AS conversions,
+ round(100.0 * uniqExact(conversion_${groupCol}) / uniqExact(${groupCol}), 2) AS conversion_rate_percentage${ttcAggColumns}
+ FROM (${innerSQL})
+ GROUP BY event_day${breakdownGroupBy.length ? ', ' + breakdownGroupByStr : ''}
+ )`;
+
+ // TTC columns for final SELECT
+ const ttcSelectColumns = measuring === 'time_to_convert'
+ ? ', ttc_avg, ttc_median, ttc_min, ttc_max, ttc_p25, ttc_p75, ttc_p90, ttc_p99'
+ : '';
+ const ttcSelectColumnsWithPrefix = measuring === 'time_to_convert'
+ ? ',\n agg.ttc_avg, agg.ttc_median, agg.ttc_min, agg.ttc_max, agg.ttc_p25, agg.ttc_p75, agg.ttc_p90, agg.ttc_p99'
+ : '';
+
+ let finalSql: string;
+
+ if (breakdownGroupBy.length > 0) {
+ // top_breakdowns CTE: rank breakdowns — by fastest TTC when measuring time, else by rate
+ const ttcTopBreakdownCol = measuring === 'time_to_convert'
+ ? ', avg(ttc_avg) AS avg_ttc'
+ : '';
+ const topBreakdownsOrderBy = measuring === 'time_to_convert'
+ ? 'avg_ttc ASC'
+ : 'avg_rate DESC';
+ const topBreakdownsCte = `top_breakdowns AS (
+ SELECT ${breakdownGroupByStr}, avg(conversion_rate_percentage) AS avg_rate${ttcTopBreakdownCol}
+ FROM agg
+ GROUP BY ${breakdownGroupByStr}
+ ORDER BY ${topBreakdownsOrderBy}
+ LIMIT ${limit ?? 50}
+ )`;
+
+ const joinConditions = breakdownGroupBy
+ .map(b => `agg.${b} = top_breakdowns.${b}`)
+ .join(' AND ');
+
+ const orderBy = measuring === 'time_to_convert'
+ ? 'top_breakdowns.avg_ttc ASC, agg.event_day ASC'
+ : 'top_breakdowns.avg_rate DESC, agg.event_day ASC';
+
+ finalSql = `
+ WITH ${[...ctes, aggCte, topBreakdownsCte].join(',\n')}
+ SELECT
+ agg.event_day,
+ ${breakdownGroupBy.map(b => `agg.${b}`).join(',\n ')},
+ agg.total_first,
+ agg.conversions,
+ agg.conversion_rate_percentage${ttcSelectColumnsWithPrefix}
+ FROM agg
+ INNER JOIN top_breakdowns ON ${joinConditions}
+ ORDER BY ${orderBy}`;
+ } else {
+ finalSql = `
+ WITH ${[...ctes, aggCte].join(',\n')}
+ SELECT event_day, total_first, conversions, conversion_rate_percentage${ttcSelectColumns}
+ FROM agg
+ ORDER BY event_day ASC`;
}
- const eventA = events[0]!;
- const eventB = events[1]!;
- const whereA = Object.values(
- getEventFiltersWhereClause(eventA.filters),
- ).join(' AND ');
- const whereB = Object.values(
- getEventFiltersWhereClause(eventB.filters),
- ).join(' AND ');
-
- const eventACte = clix(this.client, timezone)
- .select([
- `DISTINCT ${group}`,
- 'created_at AS a_time',
- `${clix.toStartOf('created_at', interval)} AS event_day`,
- ...breakdownColumns,
- ])
- .from(TABLE_NAMES.events)
- .where('project_id', '=', projectId)
- .where('name', '=', eventA.name)
- .rawWhere(whereA)
- .where('created_at', 'BETWEEN', [
- clix.datetime(startDate, 'toDateTime'),
- clix.datetime(endDate, 'toDateTime'),
- ]);
-
- const eventBCte = clix(this.client, timezone)
- .select([group, 'created_at AS b_time'])
- .from(TABLE_NAMES.events)
- .where('project_id', '=', projectId)
- .where('name', '=', eventB.name)
- .rawWhere(whereB)
- .where('created_at', 'BETWEEN', [
- clix.datetime(startDate, 'toDateTime'),
- clix.datetime(endDate, 'toDateTime'),
- ]);
-
- const query = clix(this.client, timezone)
- .with('event_a', eventACte)
- .with('event_b', eventBCte)
- .select<{
+ const rawResult = await this.client.query({
+ query: finalSql,
+ clickhouse_settings: { session_timezone: timezone },
+ });
+ const json = await rawResult.json() as {
+ data: {
event_day: string;
total_first: number;
conversions: number;
conversion_rate_percentage: number;
- [key: string]: string | number; // For breakdown columns
- }>([
- 'event_day',
- ...breakdownGroupBy,
- 'count(*) AS total_first',
- 'sum(if(conversion_time IS NOT NULL, 1, 0)) AS conversions',
- 'round(100.0 * sum(if(conversion_time IS NOT NULL, 1, 0)) / count(*), 2) AS conversion_rate_percentage',
- ])
- .from(
- clix.exp(`
- (SELECT
- a.${group},
- a.a_time,
- a.event_day,
- ${breakdownGroupBy.length ? `${breakdownGroupBy.join(', ')},` : ''}
- nullIf(min(b.b_time), '1970-01-01 00:00:00.000') AS conversion_time
- FROM event_a AS a
- LEFT JOIN event_b AS b ON a.${group} = b.${group}
- AND b.b_time BETWEEN a.a_time AND a.a_time + INTERVAL ${funnelWindow} HOUR
- GROUP BY a.${group}, a.a_time, a.event_day${breakdownGroupBy.length ? `, ${breakdownGroupBy.join(', ')}` : ''})
- `),
- )
- .groupBy(['event_day', ...breakdownGroupBy]);
-
- for (const order of ['event_day', ...breakdownGroupBy]) {
- query.orderBy(order);
- }
+ [key: string]: string | number;
+ }[];
+ };
+ const results = json.data;
- const results = await query.execute();
- return this.toSeries(results, breakdowns, limit).map(
- (serie, serieIndex) => {
- return {
- ...serie,
- data: serie.data.map((d, index) => ({
- ...d,
- timestamp: new Date(d.date).getTime(),
- serieIndex,
- index,
- serie: omit(['data'], serie),
- })),
- };
- },
- );
+ const resultSeries = this.toSeries(results, breakdowns);
+ const limitedSeries = resultSeries;
+
+ return limitedSeries.map((serie, serieIndex) => {
+ return {
+ ...serie,
+ data: serie.data.map((d, index) => ({
+ ...d,
+ timestamp: new Date(d.date).getTime(),
+ serieIndex,
+ index,
+ serie: omit(['data'], serie),
+ })),
+ };
+ });
+ }
+
+ private mapDataPoint(d: { [key: string]: string | number }) {
+ const base = {
+ date: d.event_day as string,
+ total: Number(d.total_first),
+ conversions: Number(d.conversions),
+ rate: Number(d.conversion_rate_percentage),
+ };
+ // Include TTC aggregations when present
+ if (d.ttc_avg != null) {
+ return {
+ ...base,
+ ttc: {
+ avg: Number(d.ttc_avg),
+ median: Number(d.ttc_median),
+ min: Number(d.ttc_min),
+ max: Number(d.ttc_max),
+ p25: Number(d.ttc_p25),
+ p75: Number(d.ttc_p75),
+ p90: Number(d.ttc_p90),
+ p99: Number(d.ttc_p99),
+ },
+ };
+ }
+ return base;
}
private toSeries(
@@ -140,19 +600,13 @@ export class ConversionService {
[key: string]: string | number;
}[],
breakdowns: { name: string }[] = [],
- limit: number | undefined = undefined,
) {
if (!breakdowns.length) {
return [
{
id: 'conversion',
breakdowns: [],
- data: data.map((d) => ({
- date: d.event_day,
- total: d.total_first,
- conversions: d.conversions,
- rate: d.conversion_rate_percentage,
- })),
+ data: data.map((d) => this.mapDataPoint(d)),
},
];
}
@@ -160,10 +614,6 @@ export class ConversionService {
// Group by breakdown values
const series = data.reduce(
(acc, d) => {
- if (limit && Object.keys(acc).length >= limit) {
- return acc;
- }
-
const key =
breakdowns.map((b, index) => d[`b_${index}`]).join('|') ||
NOT_SET_VALUE;
@@ -176,12 +626,7 @@ export class ConversionService {
data: [],
};
}
- acc[key]!.data.push({
- date: d.event_day,
- total: d.total_first,
- conversions: d.conversions,
- rate: d.conversion_rate_percentage,
- });
+ acc[key]!.data.push(this.mapDataPoint(d));
return acc;
},
{} as Record<
@@ -189,12 +634,7 @@ export class ConversionService {
{
id: string;
breakdowns: string[];
- data: {
- date: string;
- total: number;
- conversions: number;
- rate: number;
- }[];
+ data: any[];
}
>,
);
diff --git a/packages/db/src/services/custom-event.service.ts b/packages/db/src/services/custom-event.service.ts
new file mode 100644
index 000000000..4949bfede
--- /dev/null
+++ b/packages/db/src/services/custom-event.service.ts
@@ -0,0 +1,148 @@
+import sqlstring from 'sqlstring';
+import type { ICustomEventDefinition } from '@openpanel/validation';
+import { TABLE_NAMES } from '../clickhouse/client';
+import { db } from '../prisma-client';
+import { getEventFiltersWhereClause, getMaterializedColumns } from './chart.service';
+
+/**
+ * Check if an event name is a custom event
+ */
+export async function getCustomEventByName(
+ name: string,
+ projectId: string,
+) {
+ return db.customEvent.findUnique({
+ where: {
+ name_projectId: {
+ name,
+ projectId,
+ },
+ },
+ });
+}
+
+/**
+ * Get all custom events for a project
+ */
+export async function getCustomEventsByProject(projectId: string) {
+ return db.customEvent.findMany({
+ where: { projectId },
+ });
+}
+
+/**
+ * Build SQL fragment for a single source event in custom event
+ * Returns SELECT statement that can be used in UNION
+ */
+function buildCustomEventSourceQuery(
+ projectId: string,
+ customEventName: string,
+ sourceEvent: { name: string; filters?: any[] },
+ baseWhere: string[], // Additional WHERE conditions from outer query
+ materializedColumnsSelect: string, // Materialized columns to include
+): string {
+ const whereClauses = [
+ `project_id = ${sqlstring.escape(projectId)}`,
+ `name = ${sqlstring.escape(sourceEvent.name)}`,
+ ...baseWhere,
+ ];
+
+ // Add event property filters if present
+ if (sourceEvent.filters && sourceEvent.filters.length > 0) {
+ const filterWhere = getEventFiltersWhereClause(
+ sourceEvent.filters,
+ projectId,
+ );
+ whereClauses.push(...Object.values(filterWhere));
+ }
+
+ // Use ClickHouse REPLACE to replace the name column with custom event name
+ // Include materialized columns explicitly since SELECT * doesn't include them
+ // IMPORTANT: REPLACE must come immediately after *, then additional columns
+ return `
+ SELECT * REPLACE(${sqlstring.escape(customEventName)} AS name)${materializedColumnsSelect}
+ FROM ${TABLE_NAMES.events}
+ WHERE ${whereClauses.join(' AND ')}
+ `;
+}
+
+/**
+ * Expand a custom event into SQL query
+ *
+ * PERFORMANCE OPTIMIZATION:
+ * When all source events have no filters, we use a single SELECT with IN clause
+ * instead of UNION ALL. This reduces N table scans to 1 table scan.
+ *
+ * MATERIALIZED COLUMNS:
+ * SELECT * REPLACE(...) does not include materialized columns in CTEs.
+ * We explicitly include them so they're available for breakdowns/filters.
+ *
+ * Example:
+ * - Before: 20 separate SELECTs with UNION ALL (20 table scans)
+ * - After: 1 SELECT with IN ('event1', 'event2', ..., 'event20') (1 table scan)
+ *
+ * For events with filters, we fall back to UNION ALL to ensure correct filtering.
+ *
+ * @param customEvent - The custom event definition
+ * @param baseWhere - Base WHERE conditions to apply to all source events (date ranges, etc)
+ * @returns SQL query string that selects from all source events
+ */
+export async function expandCustomEventToSQL(
+ customEvent: {
+ name: string;
+ projectId: string;
+ definition: ICustomEventDefinition;
+ },
+ baseWhere: string[] = [],
+): Promise {
+ const definition = customEvent.definition;
+
+ // Get materialized column names to explicitly include in SELECT
+ // (SELECT * doesn't include materialized columns in CTEs)
+ // Use 'events' target to exclude profile.* materialized columns — those require a profiles JOIN
+ // and must not be referenced in a bare SELECT from the events table.
+ const materializedColumns = await getMaterializedColumns('events');
+ const materializedColumnNames = Object.values(materializedColumns);
+ const materializedColumnsSelect = materializedColumnNames.length > 0
+ ? `, ${materializedColumnNames.join(', ')}`
+ : '';
+
+ // Check if we can use the optimized path
+ // Optimization is only safe when all events have no filters
+ const canOptimize = definition.events.every(
+ (event) => !event.filters || event.filters.length === 0
+ );
+
+ if (canOptimize && definition.events.length > 0) {
+ // OPTIMIZED PATH: Single SELECT with IN clause
+ // This reduces N table scans to 1 table scan, dramatically improving performance
+ const eventNames = definition.events.map((e) => sqlstring.escape(e.name));
+
+ const whereClauses = [
+ `project_id = ${sqlstring.escape(customEvent.projectId)}`,
+ `name IN (${eventNames.join(', ')})`,
+ ...baseWhere,
+ ];
+
+ return `
+ SELECT * REPLACE(${sqlstring.escape(customEvent.name)} AS name)${materializedColumnsSelect}
+ FROM ${TABLE_NAMES.events}
+ WHERE ${whereClauses.join(' AND ')}
+ `;
+ }
+
+ // FALLBACK PATH: UNION ALL for events with filters
+ // When events have filters, each event may need different WHERE conditions,
+ // so we must use UNION ALL to apply filters correctly per event
+ const sourceQueries = definition.events.map((sourceEvent) =>
+ buildCustomEventSourceQuery(
+ customEvent.projectId,
+ customEvent.name,
+ sourceEvent,
+ baseWhere,
+ materializedColumnsSelect,
+ ),
+ );
+
+ return sourceQueries.join(' UNION ALL ');
+}
diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts
index 02033bddc..b7b5d02a0 100644
--- a/packages/db/src/services/event.service.ts
+++ b/packages/db/src/services/event.service.ts
@@ -596,7 +596,7 @@ export async function getEventList(options: GetEventListOptions) {
}
if (profileId) {
- sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND device_id != '' AND profile_id = ${sqlstring.escape(profileId)} group by did) OR profile_id = ${sqlstring.escape(profileId)})`;
+ sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`;
}
if (sessionId) {
diff --git a/packages/db/src/services/funnel.service.ts b/packages/db/src/services/funnel.service.ts
index 9ecc204d3..3ae85cdbe 100644
--- a/packages/db/src/services/funnel.service.ts
+++ b/packages/db/src/services/funnel.service.ts
@@ -6,29 +6,125 @@ import type {
} from '@openpanel/validation';
import { last, reverse, uniq } from 'ramda';
import sqlstring from 'sqlstring';
-import { ch } from '../clickhouse/client';
+import { ch, formatClickhouseDate } from '../clickhouse/client';
import { TABLE_NAMES } from '../clickhouse/client';
import { clix } from '../clickhouse/query-builder';
import { createSqlBuilder } from '../sql-builder';
import {
getEventFiltersWhereClause,
getSelectPropertyKey,
+ fetchCohortsMetadata,
+ getCohortCteName,
+ getCohortAlias,
+ buildCohortMembershipQuery,
+ getMaterializedColumns,
} from './chart.service';
import { onlyReportEvents } from './reports.service';
+import {
+ getCustomEventByName,
+ expandCustomEventToSQL,
+} from './custom-event.service';
export class FunnelService {
constructor(private client: typeof ch) {}
- getFunnelGroup(group?: string): [string, string] {
- return group === 'profile_id'
- ? [`COALESCE(nullIf(s.pid, ''), profile_id)`, 'profile_id']
- : ['session_id', 'session_id'];
+ /**
+ * Build events source for funnel query
+ * Handles both regular events and custom events
+ */
+ private async buildEventsSource(
+ events: IChartEvent[],
+ projectId: string,
+ startDate: string,
+ endDate: string,
+ ): Promise<{
+ fromClause: string;
+ withClauses: Array<{ name: string; query: any }>;
+ needsNameFilter: boolean;
+ }> {
+ // Check which events are custom events
+ const customEventsChecks = await Promise.all(
+ events.map((event) => getCustomEventByName(event.name, projectId)),
+ );
+
+ const hasCustomEvents = customEventsChecks.some((ce) => ce !== null);
+
+ // If no custom events, use regular events table
+ if (!hasCustomEvents) {
+ return {
+ fromClause: TABLE_NAMES.events,
+ withClauses: [],
+ needsNameFilter: true,
+ };
+ }
+
+ // Get materialized columns to ensure UNION compatibility (events table only)
+ const materializedColumns = await getMaterializedColumns('events');
+ const materializedColumnNames = Object.values(materializedColumns);
+ const materializedColumnsSelect = materializedColumnNames.length > 0
+ ? `, ${materializedColumnNames.map(col => `\`${col}\``).join(', ')}`
+ : '';
+
+ // Build CTEs for custom events
+ const withClauses: Array<{ name: string; query: any }> = [];
+ const baseWhere = [
+ `created_at >= toDateTime('${formatClickhouseDate(startDate)}')`,
+ `created_at <= toDateTime('${formatClickhouseDate(endDate)}')`,
+ ];
+
+ // Build union parts - one for each event (custom or regular)
+ const unionParts: string[] = [];
+
+ for (let i = 0; i < events.length; i++) {
+ const event = events[i]!;
+ const customEvent = customEventsChecks[i];
+
+ if (customEvent) {
+ // Custom event - create CTE and reference it
+ const cteName = `custom_event_${i}`;
+ const sql = await expandCustomEventToSQL(
+ {
+ name: customEvent.name,
+ projectId,
+ definition: customEvent.definition as any,
+ },
+ baseWhere,
+ );
+
+ withClauses.push({
+ name: cteName,
+ query: sql,
+ });
+
+ unionParts.push(`SELECT * FROM ${cteName}`);
+ } else {
+ // Regular event - include materialized columns to match custom events
+ unionParts.push(`
+ SELECT *${materializedColumnsSelect} FROM ${TABLE_NAMES.events}
+ WHERE project_id = '${projectId}'
+ AND name = '${event.name}'
+ AND created_at BETWEEN toDateTime('${startDate}') AND toDateTime('${endDate}')
+ `);
+ }
+ }
+
+ // Create combined_events CTE
+ withClauses.push({
+ name: 'combined_events',
+ query: unionParts.join(' UNION ALL '),
+ });
+
+ return {
+ fromClause: 'combined_events',
+ withClauses,
+ needsNameFilter: false, // Already filtered in CTEs
+ };
}
- getFunnelConditions(events: IChartEvent[] = []): string[] {
+ getFunnelConditions(events: IChartEvent[] = [], projectId?: string): string[] {
return events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
- sb.where = getEventFiltersWhereClause(event.filters);
+ sb.where = getEventFiltersWhereClause(event.filters, projectId);
sb.where.name = `name = ${sqlstring.escape(event.name)}`;
return getWhere().replace('WHERE ', '');
});
@@ -44,6 +140,8 @@ export class FunnelService {
timezone,
additionalSelects = [],
additionalGroupBy = [],
+ fromClause,
+ needsNameFilter,
}: {
projectId: string;
startDate: string;
@@ -54,27 +152,36 @@ export class FunnelService {
timezone: string;
additionalSelects?: string[];
additionalGroupBy?: string[];
+ fromClause: string;
+ needsNameFilter: boolean;
}) {
- const funnels = this.getFunnelConditions(eventSeries);
+ const funnels = this.getFunnelConditions(eventSeries, projectId);
- return clix(this.client, timezone)
+ const query = clix(this.client, timezone)
.select([
`${group[0]} AS ${group[1]}`,
...additionalSelects,
`windowFunnel(${funnelWindowMilliseconds}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), ${funnels.join(', ')}) AS level`,
])
- .from(TABLE_NAMES.events, false)
+ .from(fromClause, false)
.where('project_id', '=', projectId)
- .where('created_at', 'BETWEEN', [
- clix.datetime(startDate, 'toDateTime'),
- clix.datetime(endDate, 'toDateTime'),
- ])
- .where(
- 'name',
- 'IN',
- eventSeries.map((e) => e.name),
- )
.groupBy([group[1], ...additionalGroupBy]);
+
+ // Add date and name filters only for regular events
+ if (needsNameFilter) {
+ query
+ .where('created_at', 'BETWEEN', [
+ clix.datetime(startDate, 'toDateTime'),
+ clix.datetime(endDate, 'toDateTime'),
+ ])
+ .where(
+ 'name',
+ 'IN',
+ eventSeries.map((e) => e.name),
+ );
+ }
+
+ return query;
}
buildSessionsCte({
@@ -185,9 +292,13 @@ export class FunnelService {
startDate,
endDate,
series,
+ interval,
funnelWindow = 24,
funnelGroup,
breakdowns = [],
+ holdProperties = [],
+ globalFilters = [],
+ measuring = 'conversion_rate',
limit,
timezone = 'UTC',
}: IChartInput & { timezone: string; events?: IChartEvent[] }) {
@@ -195,27 +306,69 @@ export class FunnelService {
throw new Error('startDate and endDate are required');
}
- const eventSeries = onlyReportEvents(series);
+ // Merge global filters into each event's filters (same as fetch.ts does for regular charts)
+ const eventSeries = onlyReportEvents(series).map(event => ({
+ ...event,
+ filters: [...(event.filters ?? []), ...globalFilters],
+ }));
if (eventSeries.length === 0) {
throw new Error('events are required');
}
+ // Extract cohort IDs from breakdowns and event filters (deduplicated)
+ const cohortIdsSet = new Set();
+ breakdowns?.forEach((b) => {
+ if (b.cohortId) {
+ cohortIdsSet.add(b.cohortId);
+ } else if (b.name.startsWith('cohort:')) {
+ cohortIdsSet.add(b.name.split(':')[1]!);
+ }
+ });
+ eventSeries.forEach((event) => {
+ event.filters?.forEach((filter) => {
+ if (filter.cohortId) {
+ cohortIdsSet.add(filter.cohortId);
+ }
+ });
+ });
+
+ const cohortIds = Array.from(cohortIdsSet);
+
+ // Fetch cohort metadata from Postgres (always fresh, no cache)
+ const cohortMetadata = await fetchCohortsMetadata(cohortIds);
+
const funnelWindowSeconds = funnelWindow * 3600;
const funnelWindowMilliseconds = funnelWindowSeconds * 1000;
- const group = this.getFunnelGroup(funnelGroup);
const profileFilters = this.getProfileFilters(eventSeries);
const anyFilterOnProfile = profileFilters.length > 0;
const anyBreakdownOnProfile = breakdowns.some((b) =>
b.name.startsWith('profile.'),
);
+ // Get events source (handles custom events)
+ const { fromClause, withClauses, needsNameFilter } =
+ await this.buildEventsSource(eventSeries, projectId, startDate, endDate);
+
+ // Determine group column using the actual fromClause (not hardcoded table name)
+ const group = funnelGroup === 'profile_id'
+ ? [`COALESCE(nullIf(s.pid, ''), ${fromClause}.profile_id)`, 'profile_id'] as [string, string]
+ : [`${fromClause}.session_id`, 'session_id'] as [string, string];
+
// Create the funnel CTE
const breakdownSelects = breakdowns.map(
- (b, index) => `${getSelectPropertyKey(b.name)} as b_${index}`,
+ (b, index) => `${getSelectPropertyKey(b.name, projectId, b.cohortId)} as b_${index}`,
);
const breakdownGroupBy = breakdowns.map((b, index) => `b_${index}`);
+ // Hold property constant: add to inner CTE GROUP BY so windowFunnel()
+ // evaluates per (profile_id, property_value), but NOT to outer query
+ // so results aggregate into a single funnel.
+ const holdPropertySelects = holdProperties.map(
+ (prop, i) => `${getSelectPropertyKey(prop, projectId)} as hp_${i}`,
+ );
+ const holdPropertyGroupBy = holdProperties.map((_, i) => `hp_${i}`);
+
const funnelCte = this.buildFunnelCte({
projectId,
startDate,
@@ -224,18 +377,81 @@ export class FunnelService {
funnelWindowMilliseconds,
group,
timezone,
- additionalSelects: breakdownSelects,
- additionalGroupBy: breakdownGroupBy,
+ additionalSelects: [...breakdownSelects, ...holdPropertySelects],
+ additionalGroupBy: [...breakdownGroupBy, ...holdPropertyGroupBy],
+ fromClause,
+ needsNameFilter,
});
if (anyFilterOnProfile || anyBreakdownOnProfile) {
+ const matCols = await getMaterializedColumns('profiles');
+
+ // Collect fields needed from profile table (from both filters and breakdowns)
+ const profileFieldsSet = new Set(['id']);
+
+ // From event filters
+ for (const f of profileFilters) {
+ // f is like "properties.campaign" or "email" (already has "profile." stripped)
+ if (f.startsWith('properties.')) {
+ const fullKey = `profile.${f}`; // "profile.properties.campaign"
+ const cached = matCols[fullKey];
+ if (cached) {
+ profileFieldsSet.add(cached.replace('profile.', ''));
+ } else {
+ profileFieldsSet.add('properties');
+ }
+ } else {
+ profileFieldsSet.add(f.split('.')[0]!);
+ }
+ }
+
+ // From profile breakdowns
+ for (const b of breakdowns.filter((b) => b.name.startsWith('profile.'))) {
+ if (b.name.startsWith('profile.properties.')) {
+ const cached = matCols[b.name];
+ if (cached) {
+ profileFieldsSet.add(cached.replace('profile.', ''));
+ } else {
+ profileFieldsSet.add('properties');
+ }
+ } else {
+ const fieldName = b.name.replace('profile.', '').split('.')[0];
+ if (fieldName) {
+ profileFieldsSet.add(fieldName);
+ }
+ }
+ }
+
funnelCte.leftJoin(
- `(SELECT id, ${uniq(profileFilters.map((f) => f.split('.')[0]))} FROM ${TABLE_NAMES.profiles} FINAL
+ `(SELECT ${Array.from(profileFieldsSet).join(', ')} FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}) as profile`,
- 'profile.id = events.profile_id',
+ `profile.id = ${fromClause}.profile_id`,
);
}
+ // Add LEFT JOINs for all cohorts (much faster than IN subqueries)
+ cohortIds.forEach((cohortId) => {
+ const cohortAlias = getCohortAlias(cohortId);
+ const cohortCte = getCohortCteName(cohortId);
+ funnelCte.leftJoin(
+ `${cohortCte} AS ${cohortAlias}`,
+ `${cohortAlias}.profile_id = ${fromClause}.profile_id`,
+ );
+ });
+
+ // Apply cohort global filter WHERE clauses (inCohort / notInCohort)
+ eventSeries.forEach((event) => {
+ (event.filters ?? []).forEach((filter) => {
+ if (filter.operator === 'inCohort' && filter.cohortId) {
+ const alias = getCohortAlias(filter.cohortId);
+ funnelCte.where(`${alias}.profile_id`, '!=', '');
+ } else if (filter.operator === 'notInCohort' && filter.cohortId) {
+ const alias = getCohortAlias(filter.cohortId);
+ funnelCte.where(`${alias}.profile_id`, '=', '');
+ }
+ });
+ });
+
// Create the sessions CTE if needed
const sessionsCte =
group[0] !== 'session_id'
@@ -250,8 +466,20 @@ export class FunnelService {
// Base funnel query with CTEs
const funnelQuery = clix(this.client, timezone);
+ // Add custom event CTEs first (if any)
+ for (const withClause of withClauses) {
+ funnelQuery.with(withClause.name, withClause.query);
+ }
+
+ // Add cohort CTEs (computed once per query, not per row)
+ cohortIds.forEach((cohortId) => {
+ const cohortMeta = cohortMetadata.get(cohortId);
+ const cohortQuery = buildCohortMembershipQuery(cohortId, projectId, cohortMeta);
+ funnelQuery.with(getCohortCteName(cohortId), cohortQuery);
+ });
+
if (sessionsCte) {
- funnelCte.leftJoin('sessions s', 's.sid = events.session_id');
+ funnelCte.leftJoin('sessions s', `s.sid = ${fromClause}.session_id`);
funnelQuery.with('sessions', sessionsCte);
}
@@ -275,7 +503,7 @@ export class FunnelService {
const funnelData = await funnelQuery.execute();
const funnelSeries = this.toSeries(funnelData, breakdowns, limit);
- return funnelSeries
+ const funnelResult = funnelSeries
.map((data) => {
const maxLevel = eventSeries.length;
const filledFunnelRes = this.fillFunnel(
@@ -357,6 +585,111 @@ export class FunnelService {
const bTotal = b.steps.reduce((acc, step) => acc + step.count, 0);
return bTotal - aTotal;
});
+
+ // Compute time-to-convert if requested
+ if (measuring === 'time_to_convert' && eventSeries.length >= 2) {
+ const endDateObj = new Date(endDate);
+ const extendedEndDateObj = new Date(endDateObj.getTime() + funnelWindowSeconds * 1000);
+ const extendedEndDate = formatClickhouseDate(extendedEndDateObj);
+
+ const firstEvent = eventSeries[0]!;
+ const lastEventItem = eventSeries[eventSeries.length - 1]!;
+
+ const firstEventFilters = firstEvent.filters && firstEvent.filters.length > 0
+ ? ' AND ' + Object.values(getEventFiltersWhereClause(firstEvent.filters, projectId)).join(' AND ')
+ : '';
+ const lastEventFilters = lastEventItem.filters && lastEventItem.filters.length > 0
+ ? ' AND ' + Object.values(getEventFiltersWhereClause(lastEventItem.filters, projectId)).join(' AND ')
+ : '';
+
+ const toStartOf = clix.toStartOf('fs.first_ts', interval || 'day');
+
+ const ttcQuery = `
+ WITH
+ first_step_events AS (
+ SELECT profile_id, min(created_at) AS first_ts
+ FROM ${TABLE_NAMES.events}
+ WHERE project_id = ${sqlstring.escape(projectId)}
+ AND name = ${sqlstring.escape(firstEvent.name)}
+ AND created_at >= toDateTime('${formatClickhouseDate(startDate)}')
+ AND created_at <= toDateTime('${formatClickhouseDate(endDate)}')${firstEventFilters}
+ GROUP BY profile_id
+ ),
+ last_step_events AS (
+ SELECT profile_id, min(created_at) AS last_ts
+ FROM ${TABLE_NAMES.events}
+ WHERE project_id = ${sqlstring.escape(projectId)}
+ AND name = ${sqlstring.escape(lastEventItem.name)}
+ AND created_at >= toDateTime('${formatClickhouseDate(startDate)}')
+ AND created_at <= toDateTime('${extendedEndDate}')${lastEventFilters}
+ GROUP BY profile_id
+ ),
+ matched AS (
+ SELECT
+ ${toStartOf} AS event_day,
+ dateDiff('second', fs.first_ts, ls.last_ts) AS time_diff_seconds
+ FROM first_step_events fs
+ JOIN last_step_events ls ON ls.profile_id = fs.profile_id
+ AND ls.last_ts >= fs.first_ts
+ AND ls.last_ts <= fs.first_ts + INTERVAL ${funnelWindowSeconds} SECOND
+ )
+ SELECT
+ event_day,
+ count() AS completed_count,
+ round(avg(time_diff_seconds)) AS ttc_avg,
+ round(quantile(0.5)(time_diff_seconds)) AS ttc_median,
+ min(time_diff_seconds) AS ttc_min,
+ max(time_diff_seconds) AS ttc_max,
+ round(quantile(0.25)(time_diff_seconds)) AS ttc_p25,
+ round(quantile(0.75)(time_diff_seconds)) AS ttc_p75,
+ round(quantile(0.9)(time_diff_seconds)) AS ttc_p90,
+ round(quantile(0.99)(time_diff_seconds)) AS ttc_p99
+ FROM matched
+ GROUP BY event_day
+ ORDER BY event_day ASC
+ `;
+
+ const ttcResult = await this.client.query({
+ query: ttcQuery,
+ clickhouse_settings: { session_timezone: timezone },
+ });
+ const ttcJson = await ttcResult.json() as {
+ data: {
+ event_day: string;
+ completed_count: number;
+ ttc_avg: number;
+ ttc_median: number;
+ ttc_min: number;
+ ttc_max: number;
+ ttc_p25: number;
+ ttc_p75: number;
+ ttc_p90: number;
+ ttc_p99: number;
+ }[];
+ };
+
+ const timeToConvert = ttcJson.data.map(d => ({
+ date: d.event_day,
+ completedCount: Number(d.completed_count),
+ ttc: {
+ avg: Number(d.ttc_avg),
+ median: Number(d.ttc_median),
+ min: Number(d.ttc_min),
+ max: Number(d.ttc_max),
+ p25: Number(d.ttc_p25),
+ p75: Number(d.ttc_p75),
+ p90: Number(d.ttc_p90),
+ p99: Number(d.ttc_p99),
+ },
+ }));
+
+ return funnelResult.map(item => ({
+ ...item,
+ timeToConvert,
+ }));
+ }
+
+ return funnelResult;
}
}
diff --git a/packages/db/src/services/import.service.ts b/packages/db/src/services/import.service.ts
index 189d54d59..f718bb9dc 100644
--- a/packages/db/src/services/import.service.ts
+++ b/packages/db/src/services/import.service.ts
@@ -38,7 +38,7 @@ export async function insertImportBatch(
// Important to have same order as events_imports table
// CSV format: properly quotes fields that need it
- const csvRows = events.map((event) => {
+ let csvRows = events.map((event) => {
// Properties need to be converted to JSON for Map(String, String)
// All fields must be CSV-escaped when joining with commas
const fields = [
@@ -80,6 +80,9 @@ export async function insertImportBatch(
await chInsertCSV(TABLE_NAMES.events_imports, csvRows);
+ // Explicitly release memory
+ csvRows = null as any;
+
return {
importId,
totalEvents: events.length,
@@ -96,6 +99,17 @@ export async function generateSessionIds(
importId: string,
from: string,
): Promise {
+ console.log('ALTER TABLE session_id generation starting', {
+ importId,
+ from,
+ settings: {
+ mutations_sync: '1',
+ wait_end_of_query: 0,
+ },
+ });
+
+ const startTime = Date.now();
+
const rangeWhere = [
'import_id = {importId:String}',
"import_status = 'pending'",
@@ -120,16 +134,32 @@ export async function generateSessionIds(
WHERE ${rangeWhere}
`;
- await ch.command({
- query: updateQuery,
- query_params: { importId, from },
- clickhouse_settings: {
- wait_end_of_query: 1,
- mutations_sync: '2', // Wait for mutation to complete on all replicas (critical!)
- send_progress_in_http_headers: 1,
- http_headers_progress_interval_ms: '50000',
- },
- });
+ try {
+ await ch.command({
+ query: updateQuery,
+ query_params: { importId, from },
+ clickhouse_settings: {
+ wait_end_of_query: 0,
+ mutations_sync: '1',
+ send_progress_in_http_headers: 1,
+ http_headers_progress_interval_ms: '50000',
+ },
+ });
+
+ const elapsed = Date.now() - startTime;
+ console.log('Session ID generation submitted', {
+ importId,
+ from,
+ elapsedSec: Math.round(elapsed / 1000),
+ });
+ } catch (error) {
+ console.error('Session ID generation failed', {
+ importId,
+ from,
+ error: (error as Error).message,
+ });
+ throw error;
+ }
}
/**
@@ -142,6 +172,8 @@ export async function createSessionsStartEndEvents(
importId: string,
from: string,
): Promise {
+ console.log('[Phase 3] Starting createSessionsStartEndEvents', { importId, from });
+
// First, let's identify session boundaries and get first/last events for each session
const rangeWhere = [
'import_id = {importId:String}',
@@ -152,6 +184,8 @@ export async function createSessionsStartEndEvents(
.filter(Boolean)
.join(' AND ');
+ console.log('[Phase 3] WHERE clause:', rangeWhere);
+
// Use window functions to efficiently get first event (all fields) and last event (only changing fields)
// session_end only needs: properties, path, origin, created_at - the rest can be inherited from session_start
const sessionEventsQuery = `
@@ -170,10 +204,15 @@ export async function createSessionsStartEndEvents(
GROUP BY session_id, device_id, project_id, profile_id
`;
+ console.log('[Phase 3] Querying session boundaries...');
const sessionEventsResult = await ch.query({
query: sessionEventsQuery,
query_params: { importId, from },
format: 'JSONEachRow',
+ clickhouse_settings: {
+ max_memory_usage: 40000000000, // 40 GB - increased for large import GROUP BY queries
+ max_bytes_before_external_group_by: 20000000000, // 20 GB
+ },
});
const sessionData = (await sessionEventsResult.json()) as Array<{
@@ -218,136 +257,185 @@ export async function createSessionsStartEndEvents(
last_timestamp: string;
}>;
- // Create session_start and session_end events
- const sessionEvents: IClickhouseEvent[] = [];
-
- for (const session of sessionData) {
- // Destructure first event tuple (all fields)
- const [
- // firstId,
- // firstName,
- firstPath,
- firstOrigin,
- firstReferrer,
- firstReferrerName,
- firstReferrerType,
- // firstDuration,
- firstProperties,
- firstCreatedAt,
- firstCountry,
- firstCity,
- firstRegion,
- firstLongitude,
- firstLatitude,
- firstOs,
- firstOsVersion,
- firstBrowser,
- firstBrowserVersion,
- firstDevice,
- firstBrand,
- firstModel,
- // firstSdkName,
- // firstSdkVersion,
- // firstImportedAt,
- ] = session.first_event;
-
- // Destructure last event fields (only the changing ones)
- const [lastPath, lastOrigin, lastProperties, lastCreatedAt] =
- session.last_event_fields;
-
- // Calculate duration in milliseconds
- // Parse timestamps as Date objects to calculate duration
- const firstTime = new Date(session.first_timestamp).getTime();
- const lastTime = new Date(session.last_timestamp).getTime();
- const durationMs = Math.max(0, lastTime - firstTime); // Ensure non-negative duration
-
- // Helper function to adjust timestamp by milliseconds without timezone conversion
- const adjustTimestamp = (timestamp: string, offsetMs: number): string => {
- // Parse the timestamp, adjust it, and format back to ClickHouse format
- const date = convertClickhouseDateToJs(timestamp);
- date.setTime(date.getTime() + offsetMs);
- return formatClickhouseDate(date);
- };
+ console.log('[Phase 3] Found sessions:', sessionData.length);
+
+ // Process sessions in batches to avoid memory issues with large imports
+ const BATCH_SIZE = 50000; // Process 50K sessions at a time (creates 100K events per batch)
+ const totalBatches = Math.ceil(sessionData.length / BATCH_SIZE);
+ let totalEventsCreated = 0;
+
+ console.log('[Phase 3] Processing in batches:', {
+ totalSessions: sessionData.length,
+ batchSize: BATCH_SIZE,
+ totalBatches,
+ });
+
+ for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) {
+ const startIdx = batchIndex * BATCH_SIZE;
+ const endIdx = Math.min(startIdx + BATCH_SIZE, sessionData.length);
+ const batchSessions = sessionData.slice(startIdx, endIdx);
- // Create session_start event - inherit everything from first event but change name
- // Set created_at to 1 second before the first event
- sessionEvents.push({
- id: crypto.randomUUID(),
- name: 'session_start',
- device_id: session.device_id,
- profile_id: session.profile_id,
- project_id: session.project_id,
- session_id: session.session_id,
- path: firstPath,
- origin: firstOrigin,
- referrer: firstReferrer,
- referrer_name: firstReferrerName,
- referrer_type: firstReferrerType,
- duration: 0, // session_start always has 0 duration
- properties: firstProperties as Record<
- string,
- string | number | boolean | null | undefined
- >,
- created_at: adjustTimestamp(session.first_timestamp, -1000), // 1 second before first event
- country: firstCountry,
- city: firstCity,
- region: firstRegion,
- longitude: firstLongitude,
- latitude: firstLatitude,
- os: firstOs,
- os_version: firstOsVersion,
- browser: firstBrowser,
- browser_version: firstBrowserVersion,
- device: firstDevice,
- brand: firstBrand,
- model: firstModel,
- imported_at: new Date().toISOString(),
- sdk_name: 'import-session-reconstruction',
- sdk_version: '1.0.0',
+ console.log(`[Phase 3] Processing batch ${batchIndex + 1}/${totalBatches}`, {
+ sessions: batchSessions.length,
+ from: startIdx,
+ to: endIdx,
});
- // Create session_end event - inherit most from session_start, but use last event's path, origin, properties
- // Set created_at to 1 second after the last event
- sessionEvents.push({
- id: crypto.randomUUID(),
- name: 'session_end',
- device_id: session.device_id,
- profile_id: session.profile_id,
- project_id: session.project_id,
- session_id: session.session_id,
- path: lastPath, // From last event
- origin: lastOrigin, // From last event
- referrer: firstReferrer, // Same as session_start
- referrer_name: firstReferrerName, // Same as session_start
- referrer_type: firstReferrerType, // Same as session_start
- duration: durationMs,
- properties: lastProperties as Record<
- string,
- string | number | boolean | null | undefined
- >, // From last event
- created_at: adjustTimestamp(session.last_timestamp, 500), // 1 second after last event
- country: firstCountry, // Same as session_start
- city: firstCity, // Same as session_start
- region: firstRegion, // Same as session_start
- longitude: firstLongitude, // Same as session_start
- latitude: firstLatitude, // Same as session_start
- os: firstOs, // Same as session_start
- os_version: firstOsVersion, // Same as session_start
- browser: firstBrowser, // Same as session_start
- browser_version: firstBrowserVersion, // Same as session_start
- device: firstDevice, // Same as session_start
- brand: firstBrand, // Same as session_start
- model: firstModel, // Same as session_start
- imported_at: new Date().toISOString(),
- sdk_name: 'import-session-reconstruction',
- sdk_version: '1.0.0',
+ // Create session_start and session_end events for this batch
+ const sessionEvents: IClickhouseEvent[] = [];
+
+ for (const session of batchSessions) {
+ // Destructure first event tuple (all fields)
+ const [
+ // firstId,
+ // firstName,
+ firstPath,
+ firstOrigin,
+ firstReferrer,
+ firstReferrerName,
+ firstReferrerType,
+ // firstDuration,
+ firstProperties,
+ firstCreatedAt,
+ firstCountry,
+ firstCity,
+ firstRegion,
+ firstLongitude,
+ firstLatitude,
+ firstOs,
+ firstOsVersion,
+ firstBrowser,
+ firstBrowserVersion,
+ firstDevice,
+ firstBrand,
+ firstModel,
+ // firstSdkName,
+ // firstSdkVersion,
+ // firstImportedAt,
+ ] = session.first_event;
+
+ // Destructure last event fields (only the changing ones)
+ const [lastPath, lastOrigin, lastProperties, lastCreatedAt] =
+ session.last_event_fields;
+
+ // Calculate duration in milliseconds
+ // Parse timestamps as Date objects to calculate duration
+ const firstTime = new Date(session.first_timestamp).getTime();
+ const lastTime = new Date(session.last_timestamp).getTime();
+ const durationMs = Math.max(0, lastTime - firstTime); // Ensure non-negative duration
+
+ // Helper function to adjust timestamp by milliseconds without timezone conversion
+ const adjustTimestamp = (timestamp: string, offsetMs: number): string => {
+ // Parse the timestamp, adjust it, and format back to ClickHouse format
+ const date = convertClickhouseDateToJs(timestamp);
+ date.setTime(date.getTime() + offsetMs);
+ return formatClickhouseDate(date);
+ };
+
+ // Create session_start event - inherit everything from first event but change name
+ // Set created_at to 1 second before the first event
+ sessionEvents.push({
+ id: crypto.randomUUID(),
+ name: 'session_start',
+ device_id: session.device_id,
+ profile_id: session.profile_id,
+ project_id: session.project_id,
+ session_id: session.session_id,
+ path: firstPath,
+ origin: firstOrigin,
+ referrer: firstReferrer,
+ referrer_name: firstReferrerName,
+ referrer_type: firstReferrerType,
+ duration: 0, // session_start always has 0 duration
+ properties: firstProperties as Record<
+ string,
+ string | number | boolean | null | undefined
+ >,
+ created_at: adjustTimestamp(session.first_timestamp, -1000), // 1 second before first event
+ country: firstCountry,
+ city: firstCity,
+ region: firstRegion,
+ longitude: firstLongitude,
+ latitude: firstLatitude,
+ os: firstOs,
+ os_version: firstOsVersion,
+ browser: firstBrowser,
+ browser_version: firstBrowserVersion,
+ device: firstDevice,
+ brand: firstBrand,
+ model: firstModel,
+ imported_at: new Date().toISOString(),
+ sdk_name: 'import-session-reconstruction',
+ sdk_version: '1.0.0',
+ });
+
+ // Create session_end event - inherit most from session_start, but use last event's path, origin, properties
+ // Set created_at to 1 second after the last event
+ sessionEvents.push({
+ id: crypto.randomUUID(),
+ name: 'session_end',
+ device_id: session.device_id,
+ profile_id: session.profile_id,
+ project_id: session.project_id,
+ session_id: session.session_id,
+ path: lastPath, // From last event
+ origin: lastOrigin, // From last event
+ referrer: firstReferrer, // Same as session_start
+ referrer_name: firstReferrerName, // Same as session_start
+ referrer_type: firstReferrerType, // Same as session_start
+ duration: durationMs,
+ properties: lastProperties as Record<
+ string,
+ string | number | boolean | null | undefined
+ >, // From last event
+ created_at: adjustTimestamp(session.last_timestamp, 500), // 1 second after last event
+ country: firstCountry, // Same as session_start
+ city: firstCity, // Same as session_start
+ region: firstRegion, // Same as session_start
+ longitude: firstLongitude, // Same as session_start
+ latitude: firstLatitude, // Same as session_start
+ os: firstOs, // Same as session_start
+ os_version: firstOsVersion, // Same as session_start
+ browser: firstBrowser, // Same as session_start
+ browser_version: firstBrowserVersion, // Same as session_start
+ device: firstDevice, // Same as session_start
+ brand: firstBrand, // Same as session_start
+ model: firstModel, // Same as session_start
+ imported_at: new Date().toISOString(),
+ sdk_name: 'import-session-reconstruction',
+ sdk_version: '1.0.0',
+ });
+ }
+
+ console.log(`[Phase 3] Batch ${batchIndex + 1}/${totalBatches} created events:`, {
+ total: sessionEvents.length,
+ sessionStarts: sessionEvents.filter((e) => e.name === 'session_start')
+ .length,
+ sessionEnds: sessionEvents.filter((e) => e.name === 'session_end').length,
});
- }
- // Insert session events into imports table
- if (sessionEvents.length > 0) {
- await insertImportBatch(sessionEvents, importId);
+ // Insert this batch of session events
+ if (sessionEvents.length > 0) {
+ console.log(`[Phase 3] Inserting batch ${batchIndex + 1}/${totalBatches}...`);
+ await insertImportBatch(sessionEvents, importId);
+ totalEventsCreated += sessionEvents.length;
+ console.log(
+ `[Phase 3] Batch ${batchIndex + 1}/${totalBatches} inserted successfully`,
+ {
+ batchEvents: sessionEvents.length,
+ totalEventsCreated,
+ progress: `${((batchIndex + 1) / totalBatches * 100).toFixed(1)}%`,
+ }
+ );
+ }
}
+
+ console.log('[Phase 3] All session events inserted:', {
+ totalSessions: sessionData.length,
+ totalEventsCreated,
+ });
+
+ console.log('[Phase 3] Completed createSessionsStartEndEvents');
}
/**
@@ -358,6 +446,8 @@ export async function moveImportsToProduction(
importId: string,
from: string,
): Promise {
+ console.log('[Phase 4] Starting moveImportsToProduction', { importId, from });
+
// Build the WHERE clause for migration
// For session events (session_start/session_end), we don't filter by their created_at
// because they're created with adjusted timestamps (±1 second) that might fall outside
@@ -370,18 +460,20 @@ export async function moveImportsToProduction(
(
name IN ('session_start', 'session_end') AND
session_id IN (
- SELECT DISTINCT session_id
+ SELECT DISTINCT session_id
FROM ${TABLE_NAMES.events_imports}
WHERE import_id = {importId:String}
- AND toDate(created_at) = {from:String}
+ AND toDate(created_at) = {from:String}
AND name NOT IN ('session_start', 'session_end')
)
)
)`;
}
+ console.log('[Phase 4] WHERE clause:', whereClause);
+
const migrationQuery = `
- INSERT INTO ${TABLE_NAMES.events} (
+ INSERT INTO events (
id,
name,
sdk_name,
@@ -396,7 +488,6 @@ export async function moveImportsToProduction(
referrer_name,
referrer_type,
duration,
- properties,
created_at,
country,
city,
@@ -412,7 +503,7 @@ export async function moveImportsToProduction(
model,
imported_at
)
- SELECT
+ SELECT
id,
name,
sdk_name,
@@ -427,7 +518,6 @@ export async function moveImportsToProduction(
referrer_name,
referrer_type,
duration,
- properties,
created_at,
country,
city,
@@ -447,10 +537,15 @@ export async function moveImportsToProduction(
ORDER BY created_at ASC
`;
+ console.log('[Phase 4] Executing migration query...');
await ch.command({
query: migrationQuery,
query_params: { importId, from },
clickhouse_settings: {
+ // Increase memory limits for large INSERT with ORDER BY (257M events)
+ max_memory_usage: 40000000000, // 40 GB
+ max_bytes_before_external_sort: 20000000000, // 20 GB - force external sort if needed
+ max_bytes_before_external_group_by: 20000000000, // 20 GB
wait_end_of_query: 1,
// Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection.
send_progress_in_http_headers: 1,
@@ -458,12 +553,15 @@ export async function moveImportsToProduction(
http_headers_progress_interval_ms: '50000',
},
});
+ console.log('[Phase 4] Migration completed successfully');
}
export async function backfillSessionsToProduction(
importId: string,
from: string,
): Promise {
+ console.log('[Phase 5] Starting backfillSessionsToProduction', { importId, from });
+
// After migrating events, populate the sessions table based on the migrated sessions
// We detect all session_ids involved in this import from the imports table,
// then aggregate over the production events to construct session rows.
@@ -498,7 +596,6 @@ export async function backfillSessionsToProduction(
os_version,
sign,
version,
- properties,
utm_medium,
utm_source,
utm_campaign,
@@ -508,7 +605,7 @@ export async function backfillSessionsToProduction(
referrer_name,
referrer_type
)
- SELECT
+ SELECT
any(e.session_id) as id,
any(e.project_id) as project_id,
if(any(nullIf(e.profile_id, e.device_id)) IS NULL, any(e.profile_id), any(nullIf(e.profile_id, e.device_id))) as profile_id,
@@ -542,7 +639,6 @@ export async function backfillSessionsToProduction(
argMinIf(e.os_version, e.created_at, e.name = 'session_start') as os_version,
1 as sign,
1 as version,
- argMinIf(e.properties, e.created_at, e.name = 'session_start') as properties,
argMinIf(e.properties['__query.utm_medium'], e.created_at, e.name = 'session_start') as utm_medium,
argMinIf(e.properties['__query.utm_source'], e.created_at, e.name = 'session_start') as utm_source,
argMinIf(e.properties['__query.utm_campaign'], e.created_at, e.name = 'session_start') as utm_campaign,
@@ -571,9 +667,13 @@ export async function backfillSessionsToProduction(
GROUP BY e.session_id
`;
+ console.log('[Phase 5] Executing sessions backfill query...');
await ch.command({
query: sessionsInsertQuery,
clickhouse_settings: {
+ // Increase memory limits for large GROUP BY aggregation (120M events -> 3.9M sessions)
+ max_memory_usage: 40000000000, // 40 GB
+ max_bytes_before_external_group_by: 20000000000, // 20 GB - force external aggregation if needed
wait_end_of_query: 1,
// Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection.
send_progress_in_http_headers: 1,
@@ -581,13 +681,13 @@ export async function backfillSessionsToProduction(
http_headers_progress_interval_ms: '50000',
},
});
+ console.log('[Phase 5] Sessions backfill completed successfully');
}
/**
* Mark import as complete by updating status
*/
export async function markImportComplete(importId: string): Promise {
- // In clustered mode, we must use the replicated table for mutations
const mutationTableName = getReplicatedTableName(TABLE_NAMES.events_imports);
const updateQuery = `
ALTER TABLE ${mutationTableName}
@@ -599,11 +699,9 @@ export async function markImportComplete(importId: string): Promise {
query: updateQuery,
query_params: { importId },
clickhouse_settings: {
- wait_end_of_query: 1,
- mutations_sync: '2', // Wait for mutation to complete
- // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection.
+ wait_end_of_query: 0,
+ mutations_sync: '1',
send_progress_in_http_headers: 1,
- // The interval of sending these progress headers. Here it is less than 60s,
http_headers_progress_interval_ms: '50000',
},
});
diff --git a/packages/db/src/services/materialize-columns.service.ts b/packages/db/src/services/materialize-columns.service.ts
new file mode 100644
index 000000000..3677418b8
--- /dev/null
+++ b/packages/db/src/services/materialize-columns.service.ts
@@ -0,0 +1,633 @@
+import { type ILogger, createLogger } from '@openpanel/logger';
+import { ch } from '../clickhouse/client';
+import { chMigrationClient } from '../clickhouse/migration';
+import { db } from '../../index';
+import { refreshMaterializedColumnsCache } from './chart.service';
+
+interface PropertyUsageStats {
+ property: string; // Full path: "properties.utm_source" or "profile.properties.campaign"
+ propertyKey: string; // Key only: "utm_source" or "campaign"
+ targetTable: 'events' | 'profiles'; // Which ClickHouse table to materialize on
+ usageCount: number; // How many reports use it
+ queryFrequency: number; // Estimated queries per day
+ cardinality: number; // Number of unique values
+ estimatedSize: number; // Estimated storage cost in bytes
+ benefit: number; // Calculated benefit score
+}
+
+interface PropertyAnalysis extends PropertyUsageStats {
+ skipReason?: string; // Why it wasn't materialized (if skipped)
+}
+
+interface MaterializedColumnCandidate {
+ propertyKey: string;
+ columnName: string;
+ targetTable: 'events' | 'profiles';
+ reason: string;
+ stats: PropertyUsageStats;
+}
+
+export class MaterializeColumnsService {
+ private logger: ILogger;
+
+ // Thresholds for materialization decisions
+ private readonly MIN_USAGE_COUNT = 1; // Must be used in at least 1 report
+ private readonly MAX_CARDINALITY = 5000; // Don't materialize if >5000 unique values
+ private readonly MIN_BENEFIT_SCORE = 20; // Minimum benefit score to justify materialization
+ private readonly MAX_DAILY_MATERIALIZATIONS = 3; // Rate limit: max 3 new columns per day (total across both tables)
+
+ constructor() {
+ this.logger = createLogger({ name: 'materialize-columns' });
+ }
+
+ /**
+ * Main entry point with dry-run support
+ */
+ async analyze(options: {
+ dryRun: boolean;
+ threshold?: number;
+ }): Promise<{
+ candidates: MaterializedColumnCandidate[];
+ allProperties: PropertyAnalysis[];
+ report: string;
+ materialized: string[];
+ }> {
+ const { dryRun, threshold = this.MIN_BENEFIT_SCORE } = options;
+
+ this.logger.info('Starting materialized column analysis', {
+ dryRun,
+ threshold,
+ });
+
+ // Run both pipelines: events first, then profiles
+ const { candidates: eventsCandidates, allProperties: eventsProperties } =
+ await this.analyzeDashboardProperties('events', threshold);
+
+ const { candidates: profilesCandidates, allProperties: profilesProperties } =
+ await this.analyzeDashboardProperties('profiles', threshold);
+
+ const candidates = [...eventsCandidates, ...profilesCandidates];
+ const allProperties = [...eventsProperties, ...profilesProperties];
+
+ // Generate combined report
+ const report = this.generateReport(candidates, allProperties, dryRun);
+
+ // Execute if not dry-run
+ const materialized: string[] = [];
+ if (!dryRun && candidates.length > 0) {
+ // Rate limiting across both tables combined
+ const limited = candidates.slice(0, this.MAX_DAILY_MATERIALIZATIONS);
+ if (limited.length < candidates.length) {
+ this.logger.warn(
+ `Rate limiting: Only materializing ${limited.length} of ${candidates.length} candidates`,
+ );
+ }
+
+ for (const candidate of limited) {
+ try {
+ await this.materializeColumn(candidate);
+ materialized.push(`${candidate.targetTable}:${candidate.propertyKey}`);
+ } catch (error) {
+ this.logger.error(
+ `Failed to materialize ${candidate.targetTable}:${candidate.propertyKey}`,
+ { error },
+ );
+ }
+ }
+ }
+
+ return {
+ candidates,
+ allProperties,
+ report,
+ materialized,
+ };
+ }
+
+ /**
+ * Main analysis function for a specific target table
+ */
+ private async analyzeDashboardProperties(
+ targetTable: 'events' | 'profiles',
+ threshold: number,
+ ): Promise<{
+ candidates: MaterializedColumnCandidate[];
+ allProperties: PropertyAnalysis[];
+ }> {
+ const propertyUsage = targetTable === 'events'
+ ? await this.getEventsPropertyUsageFromReports()
+ : await this.getProfilePropertyUsageFromReports();
+
+ if (propertyUsage.length === 0) {
+ this.logger.info(`No ${targetTable} properties found in reports`);
+ return { candidates: [], allProperties: [] };
+ }
+
+ this.logger.info(`Found ${propertyUsage.length} unique ${targetTable} properties in reports`);
+
+ // Check already tracked in database
+ const existingColumns = await db.materializedColumn.findMany({
+ where: { status: 'active', targetTable },
+ select: { propertyKey: true },
+ });
+ const existingKeys = new Set(existingColumns.map((c) => c.propertyKey));
+
+ // Check columns already existing in ClickHouse
+ const clickhouseColumns = await this.getExistingClickHouseColumns(targetTable);
+ const clickhouseColumnNames = new Set(clickhouseColumns);
+
+ const alreadyTracked = propertyUsage.filter((p) => existingKeys.has(p.propertyKey));
+ const alreadyExistsInClickHouse = propertyUsage.filter(
+ (p) => !existingKeys.has(p.propertyKey) && clickhouseColumnNames.has(p.propertyKey),
+ );
+ const newProperties = propertyUsage.filter(
+ (p) => !existingKeys.has(p.propertyKey) && !clickhouseColumnNames.has(p.propertyKey),
+ );
+
+ const allProperties: PropertyAnalysis[] = [];
+
+ allProperties.push(
+ ...alreadyTracked.map((p) => ({
+ ...p,
+ cardinality: 0,
+ estimatedSize: 0,
+ benefit: 0,
+ skipReason: '✅ Already materialized (tracked)',
+ })),
+ );
+
+ allProperties.push(
+ ...alreadyExistsInClickHouse.map((p) => ({
+ ...p,
+ cardinality: 0,
+ estimatedSize: 0,
+ benefit: 0,
+ skipReason: `✅ Column already exists in ${targetTable} table`,
+ })),
+ );
+
+ if (newProperties.length === 0) {
+ this.logger.info(`No new ${targetTable} properties to analyze (all already materialized)`);
+ return { candidates: [], allProperties };
+ }
+
+ this.logger.info(`${newProperties.length} ${targetTable} properties not yet materialized`);
+
+ // Enrich with ClickHouse statistics
+ const enrichedStats = await Promise.all(
+ newProperties.map((usage) => this.enrichWithClickHouseStats(usage)),
+ );
+
+ // Calculate benefit scores
+ const statsWithBenefit = enrichedStats.map((stats) => this.calculateBenefitScore(stats));
+
+ // Determine skip reasons
+ const analyzed = statsWithBenefit.map((stats) => ({
+ ...stats,
+ skipReason: this.getSkipReason(stats, threshold),
+ }));
+
+ allProperties.push(...analyzed);
+
+ const candidates = analyzed
+ .filter((stat) => !stat.skipReason)
+ .map((stats) => this.createCandidate(stats))
+ .sort((a, b) => b.stats.benefit - a.stats.benefit);
+
+ this.logger.info(`Identified ${candidates.length} ${targetTable} candidates for materialization`);
+
+ return {
+ candidates,
+ allProperties: allProperties.sort((a, b) => b.benefit - a.benefit),
+ };
+ }
+
+ /**
+ * Get existing materialized column names from a ClickHouse table
+ */
+ private async getExistingClickHouseColumns(targetTable: 'events' | 'profiles'): Promise {
+ try {
+ const result = await ch.query({
+ query: `
+ SELECT name
+ FROM system.columns
+ WHERE database = 'default'
+ AND table = '${targetTable}'
+ AND default_kind = 'MATERIALIZED'
+ `,
+ format: 'JSONEachRow',
+ });
+
+ const data = await result.json<{ name: string }>();
+ return data.map((row) => row.name);
+ } catch (error) {
+ this.logger.warn(`Failed to get existing ClickHouse columns for ${targetTable}`, { error });
+ return [];
+ }
+ }
+
+ /**
+ * Determine why a property should be skipped
+ */
+ private getSkipReason(stats: PropertyUsageStats, threshold: number): string | undefined {
+ if (stats.usageCount < this.MIN_USAGE_COUNT) {
+ return `❌ Low usage (${stats.usageCount} reports, need ${this.MIN_USAGE_COUNT})`;
+ }
+
+ if (stats.cardinality === 0) {
+ return `❌ No data found in source table`;
+ }
+
+ if (stats.cardinality > this.MAX_CARDINALITY) {
+ return `❌ Too high cardinality (${stats.cardinality} values > ${this.MAX_CARDINALITY} limit)`;
+ }
+
+ if (stats.benefit < threshold) {
+ return `❌ Benefit too low (${stats.benefit.toFixed(0)} < ${threshold} threshold)`;
+ }
+
+ return undefined;
+ }
+
+ /**
+ * Extract event properties from reports (properties.*)
+ */
+ private async getEventsPropertyUsageFromReports(): Promise<
+ Array<{
+ property: string;
+ propertyKey: string;
+ targetTable: 'events';
+ usageCount: number;
+ queryFrequency: number;
+ }>
+ > {
+ const reports = await db.report.findMany({
+ select: { breakdowns: true, events: true },
+ });
+
+ const propertyMap = new Map();
+
+ for (const report of reports) {
+ const { eventsProperties } = this.extractPropertiesFromReport(report);
+ for (const prop of eventsProperties) {
+ propertyMap.set(prop, (propertyMap.get(prop) || 0) + 1);
+ }
+ }
+
+ const ESTIMATED_QUERIES_PER_DAY = 10;
+
+ return Array.from(propertyMap.entries()).map(([property, usageCount]) => ({
+ property,
+ propertyKey: property.replace('properties.', ''),
+ targetTable: 'events' as const,
+ usageCount,
+ queryFrequency: usageCount * ESTIMATED_QUERIES_PER_DAY,
+ }));
+ }
+
+ /**
+ * Extract profile properties from reports (profile.properties.*)
+ */
+ private async getProfilePropertyUsageFromReports(): Promise<
+ Array<{
+ property: string;
+ propertyKey: string;
+ targetTable: 'profiles';
+ usageCount: number;
+ queryFrequency: number;
+ }>
+ > {
+ const reports = await db.report.findMany({
+ select: { breakdowns: true, events: true },
+ });
+
+ const propertyMap = new Map();
+
+ for (const report of reports) {
+ const { profileProperties } = this.extractPropertiesFromReport(report);
+ for (const prop of profileProperties) {
+ propertyMap.set(prop, (propertyMap.get(prop) || 0) + 1);
+ }
+ }
+
+ const ESTIMATED_QUERIES_PER_DAY = 10;
+
+ return Array.from(propertyMap.entries()).map(([property, usageCount]) => ({
+ property,
+ // "profile.properties.campaign" -> "campaign"
+ propertyKey: property.replace('profile.properties.', ''),
+ targetTable: 'profiles' as const,
+ usageCount,
+ queryFrequency: usageCount * ESTIMATED_QUERIES_PER_DAY,
+ }));
+ }
+
+ /**
+ * Extract property names from report JSON fields
+ * Returns two sets: event properties and profile properties
+ */
+ private extractPropertiesFromReport(report: {
+ breakdowns: any;
+ events: any;
+ }): { eventsProperties: string[]; profileProperties: string[] } {
+ const eventsProperties = new Set();
+ const profileProperties = new Set();
+
+ const isValid = (name: string) =>
+ !name.includes('*') && !name.includes('(') && !name.includes('[');
+
+ // Parse breakdowns
+ try {
+ const breakdowns = Array.isArray(report.breakdowns) ? report.breakdowns : [];
+ for (const breakdown of breakdowns) {
+ if (breakdown?.name && typeof breakdown.name === 'string') {
+ if (breakdown.name.startsWith('properties.') && isValid(breakdown.name)) {
+ eventsProperties.add(breakdown.name);
+ } else if (breakdown.name.startsWith('profile.properties.') && isValid(breakdown.name)) {
+ profileProperties.add(breakdown.name);
+ }
+ }
+ }
+ } catch (e) {
+ this.logger.warn('Failed to parse breakdowns', { error: e });
+ }
+
+ // Parse events (filters)
+ try {
+ const events = Array.isArray(report.events) ? report.events : [];
+ for (const event of events) {
+ if (event?.filters && Array.isArray(event.filters)) {
+ for (const filter of event.filters) {
+ if (filter?.name && typeof filter.name === 'string') {
+ if (filter.name.startsWith('properties.') && isValid(filter.name)) {
+ eventsProperties.add(filter.name);
+ } else if (filter.name.startsWith('profile.properties.') && isValid(filter.name)) {
+ profileProperties.add(filter.name);
+ }
+ }
+ }
+ }
+ }
+ } catch (e) {
+ this.logger.warn('Failed to parse event filters', { error: e });
+ }
+
+ return {
+ eventsProperties: Array.from(eventsProperties),
+ profileProperties: Array.from(profileProperties),
+ };
+ }
+
+ /**
+ * Get cardinality and size stats from ClickHouse
+ * For events: uses event_property_values_mv
+ * For profiles: queries the profiles table directly
+ */
+ private async enrichWithClickHouseStats(usage: {
+ property: string;
+ propertyKey: string;
+ targetTable: 'events' | 'profiles';
+ usageCount: number;
+ queryFrequency: number;
+ }): Promise {
+ try {
+ let query: string;
+
+ if (usage.targetTable === 'profiles') {
+ // Query the profiles table directly for profile properties
+ query = `
+ SELECT
+ uniqExact(properties['${usage.propertyKey}']) AS cardinality,
+ avg(length(properties['${usage.propertyKey}'])) AS avg_length,
+ count() AS total_occurrences
+ FROM profiles
+ WHERE properties['${usage.propertyKey}'] != ''
+ `;
+ } else {
+ // Use the materialized view for event properties (much faster)
+ query = `
+ SELECT
+ uniqExact(property_value) AS cardinality,
+ avg(length(property_value)) AS avg_length,
+ count() AS total_occurrences
+ FROM event_property_values_mv
+ WHERE property_key = '${usage.propertyKey}'
+ AND property_value != ''
+ `;
+ }
+
+ const result = await ch.query({ query, format: 'JSONEachRow' });
+
+ const data = await result.json<{
+ cardinality: string;
+ avg_length: string;
+ total_occurrences: string;
+ }>();
+
+ const cardinality = Number(data[0]?.cardinality || 0);
+ const avgLength = Number(data[0]?.avg_length || 10);
+ const totalOccurrences = Number(data[0]?.total_occurrences || 0);
+ const estimatedSize = Math.ceil(avgLength * totalOccurrences);
+
+ return {
+ property: usage.property,
+ propertyKey: usage.propertyKey,
+ targetTable: usage.targetTable,
+ usageCount: usage.usageCount,
+ queryFrequency: usage.queryFrequency,
+ cardinality,
+ estimatedSize,
+ benefit: 0,
+ };
+ } catch (error) {
+ this.logger.warn(`Failed to get stats for ${usage.property}`, { error });
+ return {
+ property: usage.property,
+ propertyKey: usage.propertyKey,
+ targetTable: usage.targetTable,
+ usageCount: usage.usageCount,
+ queryFrequency: usage.queryFrequency,
+ cardinality: 0,
+ estimatedSize: 0,
+ benefit: 0,
+ };
+ }
+ }
+
+ /**
+ * Calculate benefit score
+ */
+ private calculateBenefitScore(stats: PropertyUsageStats): PropertyUsageStats {
+ const usageScore = stats.usageCount * 10;
+ const frequencyScore = Math.min(stats.queryFrequency, 1000);
+ const cardinalityPenalty = Math.max(0, stats.cardinality - 100) * 0.5;
+ const sizePenalty = stats.estimatedSize / 1_000_000;
+
+ const benefit = usageScore + frequencyScore - cardinalityPenalty - sizePenalty;
+
+ return { ...stats, benefit: Math.max(0, benefit) };
+ }
+
+ /**
+ * Create candidate object
+ */
+ private createCandidate(stats: PropertyUsageStats): MaterializedColumnCandidate {
+ const columnName = stats.propertyKey;
+
+ let reason = `Used in ${stats.usageCount} reports (~${stats.queryFrequency} queries/day). `;
+
+ if (stats.cardinality < 50) {
+ reason += 'Low cardinality (ideal). ';
+ } else if (stats.cardinality < 200) {
+ reason += 'Moderate cardinality. ';
+ }
+
+ if (stats.estimatedSize < 100_000_000) {
+ reason += 'Small storage cost. ';
+ }
+
+ reason += `Benefit: ${stats.benefit.toFixed(0)}.`;
+
+ return {
+ propertyKey: stats.propertyKey,
+ columnName,
+ targetTable: stats.targetTable,
+ reason,
+ stats,
+ };
+ }
+
+ /**
+ * Execute materialization on the appropriate table
+ */
+ private async materializeColumn(candidate: MaterializedColumnCandidate): Promise {
+ const table = candidate.targetTable;
+
+ this.logger.info(`Materializing column: ${table}.${candidate.columnName}`, {
+ reason: candidate.reason,
+ });
+
+ try {
+ // Execute ALTER TABLE on the target table
+ // Backtick-quote the column name to support keys with hyphens or other special chars
+ await chMigrationClient.command({
+ query: `
+ ALTER TABLE ${table}
+ ADD COLUMN IF NOT EXISTS \`${candidate.columnName}\` String
+ MATERIALIZED properties['${candidate.propertyKey}']
+ `,
+ });
+
+ // Record in database with targetTable
+ await db.materializedColumn.create({
+ data: {
+ targetTable: candidate.targetTable,
+ propertyKey: candidate.propertyKey,
+ columnName: candidate.columnName,
+ cardinality: candidate.stats.cardinality,
+ usageCount: candidate.stats.usageCount,
+ benefitScore: candidate.stats.benefit,
+ estimatedSize: BigInt(candidate.stats.estimatedSize),
+ status: 'active',
+ materializedAt: new Date(),
+ },
+ });
+
+ // Refresh chart service cache so new column is used immediately
+ await refreshMaterializedColumnsCache();
+
+ this.logger.info(`Successfully materialized: ${table}.${candidate.columnName}`);
+ } catch (error) {
+ try {
+ await db.materializedColumn.create({
+ data: {
+ targetTable: candidate.targetTable,
+ propertyKey: candidate.propertyKey,
+ columnName: candidate.columnName,
+ cardinality: candidate.stats.cardinality,
+ usageCount: candidate.stats.usageCount,
+ benefitScore: candidate.stats.benefit,
+ estimatedSize: BigInt(candidate.stats.estimatedSize),
+ status: 'failed',
+ },
+ });
+ } catch (dbError) {
+ this.logger.error('Failed to record failure in database', { dbError });
+ }
+
+ throw error;
+ }
+ }
+
+ /**
+ * Generate human-readable report
+ */
+ private generateReport(
+ candidates: MaterializedColumnCandidate[],
+ allProperties: PropertyAnalysis[],
+ dryRun: boolean,
+ ): string {
+ let report = '\n' + '='.repeat(80) + '\n';
+ report += dryRun
+ ? 'DRY RUN: Materialized Column Analysis\n'
+ : 'Materialized Column Analysis\n';
+ report += '='.repeat(80) + '\n\n';
+
+ report += `Total properties analyzed: ${allProperties.length}\n`;
+ report += `Candidates for materialization: ${candidates.length}\n\n`;
+
+ if (candidates.length > 0) {
+ report += '━'.repeat(80) + '\n';
+ report += '✅ RECOMMENDED FOR MATERIALIZATION\n';
+ report += '━'.repeat(80) + '\n\n';
+
+ for (let i = 0; i < candidates.length; i++) {
+ const candidate = candidates[i]!;
+ const prefix = candidate.targetTable === 'profiles' ? 'profile.properties' : 'properties';
+ report += `${i + 1}. ${prefix}.${candidate.propertyKey} [${candidate.targetTable}]\n`;
+ report += ` Usage: ${candidate.stats.usageCount} reports, ~${candidate.stats.queryFrequency} queries/day\n`;
+ report += ` Cardinality: ${candidate.stats.cardinality} unique values\n`;
+ report += ` Storage: ~${(candidate.stats.estimatedSize / 1_000_000).toFixed(2)} MB\n`;
+ report += ` Benefit Score: ${candidate.stats.benefit.toFixed(2)}\n`;
+ report += ` Reason: ${candidate.reason}\n\n`;
+ }
+ }
+
+ const skipped = allProperties.filter((p) => p.skipReason);
+ if (skipped.length > 0) {
+ report += '━'.repeat(80) + '\n';
+ report += 'ALL PROPERTIES ANALYZED\n';
+ report += '━'.repeat(80) + '\n\n';
+
+ for (const prop of skipped) {
+ const prefix = prop.targetTable === 'profiles' ? 'profile.properties' : 'properties';
+ report += `• ${prefix}.${prop.propertyKey} [${prop.targetTable}]\n`;
+ report += ` ${prop.skipReason}\n`;
+ report += ` Usage: ${prop.usageCount} reports, ~${prop.queryFrequency} queries/day`;
+ if (prop.cardinality > 0) {
+ report += `, Cardinality: ${prop.cardinality}, Benefit: ${prop.benefit.toFixed(0)}`;
+ }
+ report += '\n\n';
+ }
+ }
+
+ report += '━'.repeat(80) + '\n';
+ report += 'SUMMARY\n';
+ report += '━'.repeat(80) + '\n';
+
+ if (dryRun) {
+ report += '⚠️ DRY RUN MODE: No changes will be made.\n';
+ report += 'Run with --execute flag to materialize these columns.\n';
+ } else if (candidates.length > 0) {
+ report += `✅ Materializing top ${Math.min(candidates.length, this.MAX_DAILY_MATERIALIZATIONS)} columns...\n`;
+ } else {
+ report += 'No actions needed. All eligible properties are already materialized.\n';
+ }
+
+ report += '\n' + '='.repeat(80) + '\n';
+
+ return report;
+ }
+}
+
+export const materializeColumnsService = new MaterializeColumnsService();
diff --git a/packages/db/src/services/notification.service.ts b/packages/db/src/services/notification.service.ts
index d7d54eed4..604518898 100644
--- a/packages/db/src/services/notification.service.ts
+++ b/packages/db/src/services/notification.service.ts
@@ -13,7 +13,6 @@ import type {
IServiceCreateEventPayload,
IServiceEvent,
} from './event.service';
-import { getProfileById } from './profile.service';
import { getProjectByIdCached } from './project.service';
type ICreateNotification = Pick<
@@ -209,35 +208,6 @@ export function matchEvent(
return true;
}
-function notificationTemplateEvent({
- payload,
- rule,
-}: {
- payload: IServiceCreateEventPayload;
- rule: INotificationRuleCached;
-}) {
- if (!rule.template) return `You received a new "${payload.name}" event`;
- let template = rule.template
- .replaceAll('$EVENT_NAME', payload.name)
- .replaceAll('$RULE_NAME', rule.name)
- .replaceAll('{{rule_name}}', rule.name);
-
- // Replace all {{xxx}} placeholders with their values
- const placeholderMatches = template.match(/{{[^}]+}}/g) || [];
- for (const match of placeholderMatches) {
- const path = match.slice(2, -2); // Remove {{ and }}
- const value = pathOr('', path.split('.'), payload);
-
- if (value) {
- template = template.replaceAll(
- match,
- typeof value === 'object' ? JSON.stringify(value) : value,
- );
- }
- }
-
- return template;
-}
function notificationTemplateFunnel({
events,
@@ -252,81 +222,6 @@ function notificationTemplateFunnel({
.replaceAll('$RULE_NAME', rule.name);
}
-export async function checkNotificationRulesForEvent(
- payload: IServiceCreateEventPayload,
-) {
- const project = await getProjectByIdCached(payload.projectId);
- const rules = await getNotificationRulesByProjectId(payload.projectId);
-
- // If profile is present in the template, add it to the payload (event)
- // so we can use it in the template
- if (
- payload.profileId &&
- rules.some((rule) => rule.template?.match(/{{profile\.[^}]*}}/))
- ) {
- const profile = await getProfileById(payload.profileId, payload.projectId);
- if (profile) {
- (payload as any).profile = profile;
- }
- }
-
- await Promise.all(
- rules.flatMap((rule) => {
- if (rule.config.type === 'events') {
- const match = rule.config.events.find((event) => {
- return matchEvent(payload, event);
- });
-
- if (!match) {
- return [];
- }
-
- const notification = {
- title: notificationTemplateEvent({
- payload,
- rule,
- }),
- message: project?.name ? `Project: ${project?.name}` : '',
- projectId: payload.projectId,
- payload: {
- type: 'event',
- event: payload,
- },
- } as const;
-
- const promises = rule.integrations.map((integration) =>
- createNotification({
- ...notification,
- integrationId: integration.id,
- notificationRuleId: rule.id,
- }),
- );
-
- if (rule.sendToApp) {
- promises.push(
- createNotification({
- ...notification,
- integrationId: APP_NOTIFICATION_INTEGRATION_ID,
- notificationRuleId: rule.id,
- }),
- );
- }
-
- if (rule.sendToEmail) {
- promises.push(
- createNotification({
- ...notification,
- integrationId: EMAIL_NOTIFICATION_INTEGRATION_ID,
- notificationRuleId: rule.id,
- }),
- );
- }
-
- return promises;
- }
- }),
- );
-}
const isFunnelRule = (rule: INotificationRuleCached) =>
rule.config.type === 'funnel';
diff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.ts
index 41358791c..2d6f719ee 100644
--- a/packages/db/src/services/profile.service.ts
+++ b/packages/db/src/services/profile.service.ts
@@ -117,17 +117,19 @@ export async function getProfileById(id: string, projectId: string) {
}
const [profile] = await chQuery(
- `SELECT
- id,
+ `SELECT
+ id,
project_id,
- last_value(nullIf(first_name, '')) as first_name,
- last_value(nullIf(last_name, '')) as last_name,
- last_value(nullIf(email, '')) as email,
- last_value(nullIf(avatar, '')) as avatar,
- last_value(is_external) as is_external,
- last_value(properties) as properties,
+ last_value(nullIf(first_name, '')) as first_name,
+ last_value(nullIf(last_name, '')) as last_name,
+ last_value(nullIf(email, '')) as email,
+ last_value(nullIf(avatar, '')) as avatar,
+ last_value(is_external) as is_external,
+ last_value(properties) as properties,
last_value(created_at) as created_at
FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${sqlstring.escape(String(id))} AND project_id = ${sqlstring.escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`,
+ undefined,
+ true,
);
if (!profile) {
@@ -154,22 +156,24 @@ export async function getProfiles(ids: string[], projectId: string) {
}
const data = await chQuery(
- `SELECT
- id,
+ `SELECT
+ id,
project_id,
- any(nullIf(first_name, '')) as first_name,
- any(nullIf(last_name, '')) as last_name,
- any(nullIf(email, '')) as email,
- any(nullIf(avatar, '')) as avatar,
- last_value(is_external) as is_external,
- any(properties) as properties,
+ any(nullIf(first_name, '')) as first_name,
+ any(nullIf(last_name, '')) as last_name,
+ any(nullIf(email, '')) as email,
+ any(nullIf(avatar, '')) as avatar,
+ last_value(is_external) as is_external,
+ any(properties) as properties,
any(created_at) as created_at
FROM ${TABLE_NAMES.profiles}
- WHERE
+ WHERE
project_id = ${sqlstring.escape(projectId)} AND
id IN (${filteredIds.map((id) => sqlstring.escape(id)).join(',')})
GROUP BY id, project_id
`,
+ undefined,
+ true,
);
return data.map(transformProfile);
@@ -197,7 +201,7 @@ export async function getProfileList({
if (isExternal !== undefined) {
sb.where.external = `is_external = ${isExternal ? 'true' : 'false'}`;
}
- const data = await chQuery(getSql());
+ const data = await chQuery(getSql(), undefined, true);
return data.map(transformProfile);
}
@@ -217,7 +221,7 @@ export async function getProfileListCount({
if (isExternal !== undefined) {
sb.where.external = `is_external = ${isExternal ? 'true' : 'false'}`;
}
- const data = await chQuery<{ count: number }>(getSql());
+ const data = await chQuery<{ count: number }>(getSql(), undefined, true);
return data[0]?.count ?? 0;
}
diff --git a/packages/db/src/services/project.service.ts b/packages/db/src/services/project.service.ts
index 7e8a997a4..f2aa166e9 100644
--- a/packages/db/src/services/project.service.ts
+++ b/packages/db/src/services/project.service.ts
@@ -1,4 +1,4 @@
-import { cacheable } from '@openpanel/redis';
+import { cacheable, getRedisCache } from '@openpanel/redis';
import sqlstring from 'sqlstring';
import { TABLE_NAMES, chQuery } from '../clickhouse/client';
import type { Prisma, Project } from '../prisma-client';
@@ -102,9 +102,30 @@ export async function getProjects({
return projects;
}
-export const getProjectEventsCount = async (projectId: string) => {
- const res = await chQuery<{ count: number }>(
- `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`,
- );
- return res[0]?.count;
+const getProjectEventsCountUncached = async (projectId: string) => {
+ const lockKey = `lock:project-events-count:${projectId}`;
+ const redis = getRedisCache();
+
+ // Try to acquire lock to prevent thundering herd across workers
+ const acquired = await redis.set(lockKey, '1', 'EX', 30, 'NX');
+
+ if (!acquired) {
+ // Another worker is executing this query, skip - they'll update the count
+ return null;
+ }
+
+ try {
+ const res = await chQuery<{ count: number }>(
+ `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`,
+ );
+ return res[0]?.count;
+ } finally {
+ // Release lock
+ redis.del(lockKey).catch(() => {});
+ }
};
+
+export const getProjectEventsCount = cacheable(
+ getProjectEventsCountUncached,
+ 60 * 60, // 1 hour cache
+);
diff --git a/packages/db/src/services/reports.service.ts b/packages/db/src/services/reports.service.ts
index 7f8c79c41..593944e76 100644
--- a/packages/db/src/services/reports.service.ts
+++ b/packages/db/src/services/reports.service.ts
@@ -87,6 +87,11 @@ export function transformReport(
criteria: (report.criteria as ICriteria) ?? undefined,
funnelGroup: report.funnelGroup ?? undefined,
funnelWindow: report.funnelWindow ?? undefined,
+ measuring: (report.measuring as 'conversion_rate' | 'time_to_convert') ?? undefined,
+ globalFilters: ((report.globalFilters as IChartEventFilter[]) ?? []).map(transformFilter),
+ holdProperties: (report.holdProperties as string[]) ?? [],
+ limit: 50,
+ cohortFilters: [],
layout: report.layout ?? undefined,
};
}
diff --git a/packages/db/vitest.config.ts b/packages/db/vitest.config.ts
new file mode 100644
index 000000000..f87a2039f
--- /dev/null
+++ b/packages/db/vitest.config.ts
@@ -0,0 +1,3 @@
+import { getSharedVitestConfig } from '../../vitest.shared';
+
+export default getSharedVitestConfig({ __dirname });
diff --git a/packages/importer/src/providers/mixpanel.ts b/packages/importer/src/providers/mixpanel.ts
index 42e41c0aa..d7b38d371 100644
--- a/packages/importer/src/providers/mixpanel.ts
+++ b/packages/importer/src/providers/mixpanel.ts
@@ -197,8 +197,34 @@ export class MixpanelProvider extends BaseImportProvider {
}
validate(rawEvent: MixpanelRawEvent): boolean {
+ // Check event structure
const res = zMixpanelRawEvent.safeParse(rawEvent);
- return res.success;
+ if (!res.success) {
+ return false;
+ }
+
+ // list of events to exclude from import
+ const excludedEvents = [
+ 'reelForegroundWatchTime',
+ 'reelBackgroundWatchTime',
+ 'firebaseRemoteConfigLoad',
+ 'shorebirdInitialized',
+ 'shorebirdUpdateCheck',
+ 'logoScreenPaused',
+ 'otpLessEvent',
+ 'trialPromoVideoFinished',
+ 'installReferrer',
+ 'audioInterruption',
+ ];
+
+ // Check if event should be excluded
+ const eventName = rawEvent.event;
+ if (excludedEvents.includes(eventName)) {
+ this.logger?.debug('Event excluded by filter', { eventName });
+ return false;
+ }
+
+ return true;
}
transformEvent(_rawEvent: MixpanelRawEvent): IClickhouseEvent {
diff --git a/packages/logger/index.ts b/packages/logger/index.ts
index 753ceb62f..9e53be7de 100644
--- a/packages/logger/index.ts
+++ b/packages/logger/index.ts
@@ -93,7 +93,7 @@ export function createLogger({ name }: { name: string }): ILogger {
winston.format.printf((info) => {
const { level, message, service, ...meta } = info;
const metaStr =
- Object.keys(meta).length > 0 ? ` ${JSON.stringify(meta)}` : '';
+ Object.keys(meta).length > 0 ? ` ${JSON.stringify(meta, (_, v) => typeof v === 'bigint' ? v.toString() : v)}` : '';
return `${level} ${message}${metaStr}`;
}),
);
diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.ts
index d21bf4e78..080009534 100644
--- a/packages/queue/src/queues.ts
+++ b/packages/queue/src/queues.ts
@@ -111,13 +111,18 @@ export type CronQueuePayloadProject = {
type: 'deleteProjects';
payload: undefined;
};
+export type CronQueuePayloadCustomAlerts = {
+ type: 'customAlerts';
+ payload: undefined;
+};
export type CronQueuePayload =
| CronQueuePayloadSalt
| CronQueuePayloadFlushEvents
| CronQueuePayloadFlushSessions
| CronQueuePayloadFlushProfiles
| CronQueuePayloadPing
- | CronQueuePayloadProject;
+ | CronQueuePayloadProject
+ | CronQueuePayloadCustomAlerts;
export type MiscQueuePayloadTrialEndingSoon = {
type: 'trialEndingSoon';
@@ -151,8 +156,8 @@ export const eventsGroupQueues = Array.from({
list.length === 1 ? 'group_events' : `group_events_${index}`,
),
redis: getRedisGroupQueue(),
- keepCompleted: 1_000,
- keepFailed: 10_000,
+ keepCompleted: 0,
+ keepFailed: 0,
orderingDelayMs: orderingDelayMs,
autoBatch:
autoBatchMaxWaitMs && autoBatchSize
diff --git a/packages/redis/redis.ts b/packages/redis/redis.ts
index be23b99ec..cf873f1f1 100644
--- a/packages/redis/redis.ts
+++ b/packages/redis/redis.ts
@@ -10,6 +10,10 @@ export { Redis };
const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
+// Dedicated URLs for high-traffic buffers. Fall back to REDIS_URL if not set.
+const REDIS_SESSION_URL = process.env.REDIS_SESSION_URL || REDIS_URL;
+const REDIS_EVENT_URL = process.env.REDIS_EVENT_URL || REDIS_URL;
+
export interface ExtendedRedis extends Redis {
getJson: (key: string) => Promise;
setJson: (
@@ -107,8 +111,9 @@ export function getRedisQueue() {
let redisGroupQueue: ExtendedRedis;
export function getRedisGroupQueue() {
if (!redisGroupQueue) {
- // Dedicated Redis connection for GroupWorker to avoid blocking BullMQ
- redisGroupQueue = createRedisClient(REDIS_URL, {
+ // Dedicated Redis connection for GroupWorker to avoid blocking BullMQ.
+ // Uses REDIS_EVENT_URL so GroupMQ and EventBuffer share the same instance.
+ redisGroupQueue = createRedisClient(REDIS_EVENT_URL, {
...options,
enableReadyCheck: false,
maxRetriesPerRequest: null,
@@ -119,6 +124,22 @@ export function getRedisGroupQueue() {
return redisGroupQueue;
}
+let redisSession: ExtendedRedis;
+export function getRedisSession() {
+ if (!redisSession) {
+ redisSession = createRedisClient(REDIS_SESSION_URL, options);
+ }
+ return redisSession;
+}
+
+let redisEvent: ExtendedRedis;
+export function getRedisEvent() {
+ if (!redisEvent) {
+ redisEvent = createRedisClient(REDIS_EVENT_URL, options);
+ }
+ return redisEvent;
+}
+
export async function getLock(key: string, value: string, timeout: number) {
const lock = await getRedisCache().set(key, value, 'PX', timeout, 'NX');
return lock === 'OK';
diff --git a/packages/trpc/src/root.ts b/packages/trpc/src/root.ts
index d852c9c1c..814c2b658 100644
--- a/packages/trpc/src/root.ts
+++ b/packages/trpc/src/root.ts
@@ -2,6 +2,8 @@ import { authRouter } from './routers/auth';
import { chartRouter } from './routers/chart';
import { chatRouter } from './routers/chat';
import { clientRouter } from './routers/client';
+import { cohortRouter } from './routers/cohort';
+import { customEventRouter } from './routers/custom-event';
import { dashboardRouter } from './routers/dashboard';
import { eventRouter } from './routers/event';
import { importRouter } from './routers/import';
@@ -33,6 +35,8 @@ export const appRouter = createTRPCRouter({
user: userRouter,
project: projectRouter,
client: clientRouter,
+ cohort: cohortRouter,
+ customEvent: customEventRouter,
event: eventRouter,
profile: profileRouter,
session: sessionRouter,
diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts
index ccecf3d7c..b7bbd3a19 100644
--- a/packages/trpc/src/routers/chart.ts
+++ b/packages/trpc/src/routers/chart.ts
@@ -150,24 +150,43 @@ export const chartRouter = createTRPCRouter({
}),
)
.query(async ({ input: { projectId } }) => {
- const [events, meta] = await Promise.all([
+ const [events, meta, customEvents] = await Promise.all([
chQuery<{ name: string; count: number }>(
`SELECT name, count(name) as count FROM ${TABLE_NAMES.event_names_mv} WHERE project_id = ${sqlstring.escape(projectId)} GROUP BY name ORDER BY count DESC, name ASC`,
),
getEventMetasCached(projectId),
+ db.customEvent.findMany({
+ where: { projectId },
+ select: { name: true, conversion: true },
+ }),
]);
+ const regularEvents = events.map((event) => ({
+ name: event.name,
+ count: event.count,
+ meta: meta.find((m) => m.name === event.name),
+ isCustom: false,
+ }));
+
+ const customEventsList = customEvents.map((ce) => ({
+ name: ce.name,
+ count: 0, // Custom events don't have pre-computed counts
+ meta: {
+ name: ce.name,
+ conversion: ce.conversion,
+ },
+ isCustom: true,
+ }));
+
return [
{
name: '*',
count: events.reduce((acc, event) => acc + event.count, 0),
meta: undefined,
+ isCustom: false,
},
- ...events.map((event) => ({
- name: event.name,
- count: event.count,
- meta: meta.find((m) => m.name === event.name),
- })),
+ ...regularEvents,
+ ...customEventsList,
];
}),
@@ -293,7 +312,7 @@ export const chartRouter = createTRPCRouter({
} else {
const query = clix(ch)
.select<{ values: string[] }>([
- `distinct ${getSelectPropertyKey(property)} as values`,
+ `distinct ${getSelectPropertyKey(property, projectId)} as values`,
])
.from(TABLE_NAMES.events)
.where('project_id', '=', projectId)
@@ -526,12 +545,12 @@ export const chartRouter = createTRPCRouter({
GROUP BY cohort_interval
)
SELECT
- cohort_interval,
- cohort_sizes.total_first_event_count,
+ interval_users.cohort_interval,
+ cs.total_first_event_count,
${countsSelect}
FROM interval_users
- LEFT JOIN cohort_sizes AS cs ON cohort_interval = cs.cohort_interval
- ORDER BY cohort_interval ASC
+ LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval
+ ORDER BY interval_users.cohort_interval ASC
`;
const cohortData = await chQuery<{
@@ -603,7 +622,7 @@ export const chartRouter = createTRPCRouter({
if (input.breakdowns) {
Object.entries(input.breakdowns).forEach(([key, value]) => {
// Transform property keys (e.g., properties.method -> properties['method'])
- const propertyKey = getSelectPropertyKey(key);
+ const propertyKey = getSelectPropertyKey(key, projectId);
sb.where[`breakdown_${key}`] =
`${propertyKey} = ${sqlstring.escape(value)}`;
});
diff --git a/packages/trpc/src/routers/cohort.ts b/packages/trpc/src/routers/cohort.ts
new file mode 100644
index 000000000..b3083f105
--- /dev/null
+++ b/packages/trpc/src/routers/cohort.ts
@@ -0,0 +1,448 @@
+import { z } from 'zod';
+
+import {
+ chQuery,
+ computeCohort,
+ countCohort,
+ db,
+ getCohortCount,
+ getCohortMembers,
+ TABLE_NAMES,
+ updateCohortMembership,
+} from '@openpanel/db';
+import type { CohortDefinition } from '@openpanel/validation';
+import { zCohortDefinition, zCohortInput, zCohortUpdate } from '@openpanel/validation';
+import sqlstring from 'sqlstring';
+
+import { getProjectAccess } from '../access';
+import { TRPCAccessError, TRPCNotFoundError } from '../errors';
+import { createTRPCRouter, protectedProcedure } from '../trpc';
+
+export const cohortRouter = createTRPCRouter({
+ /**
+ * List all cohorts for a project
+ */
+ list: protectedProcedure
+ .input(
+ z.object({
+ projectId: z.string(),
+ includeCount: z.boolean().optional().default(false),
+ }),
+ )
+ .query(async ({ input, ctx }) => {
+ const access = await getProjectAccess({
+ projectId: input.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this project');
+ }
+
+ const cohorts = await db.cohort.findMany({
+ where: { projectId: input.projectId },
+ orderBy: { createdAt: 'desc' },
+ });
+
+ if (input.includeCount) {
+ // Return cached counts from Postgres - don't trigger expensive ClickHouse queries on list load
+ return cohorts.map((cohort) => ({
+ ...cohort,
+ currentCount: cohort.profileCount ?? 0,
+ }));
+ }
+
+ return cohorts;
+ }),
+
+ /**
+ * Get single cohort by ID
+ */
+ get: protectedProcedure
+ .input(z.object({ id: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const cohort = await db.cohort.findUnique({
+ where: { id: input.id },
+ });
+
+ if (!cohort) {
+ throw TRPCNotFoundError('Cohort not found');
+ }
+
+ const access = await getProjectAccess({
+ projectId: cohort.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this cohort');
+ }
+
+ return cohort;
+ }),
+
+ /**
+ * Create new cohort
+ */
+ create: protectedProcedure
+ .input(zCohortInput)
+ .mutation(async ({ input, ctx }) => {
+ const access = await getProjectAccess({
+ projectId: input.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this project');
+ }
+
+ // Create cohort
+ const cohort = await db.cohort.create({
+ data: {
+ name: input.name,
+ description: input.description,
+ projectId: input.projectId,
+ definition: input.definition as any, // Prisma Json type
+ isStatic: input.isStatic,
+ computeOnDemand: input.computeOnDemand,
+ },
+ });
+
+ // Compute initial count/membership in background
+ if (cohort.computeOnDemand) {
+ // Dynamic cohorts: just compute and cache the count
+ countCohort(cohort.projectId, input.definition as CohortDefinition)
+ .then((count) => {
+ return db.cohort.update({
+ where: { id: cohort.id },
+ data: {
+ profileCount: count,
+ lastComputedAt: new Date(),
+ },
+ });
+ })
+ .catch((err) => {
+ console.error('Failed to compute cohort count on creation:', err);
+ });
+ } else {
+ // Pre-computed cohorts: compute and store membership
+ updateCohortMembership(cohort.id).catch((err) => {
+ console.error('Failed to compute cohort membership on creation:', err);
+ });
+ }
+
+ return cohort;
+ }),
+
+ /**
+ * Update cohort
+ */
+ update: protectedProcedure
+ .input(zCohortUpdate)
+ .mutation(async ({ input, ctx }) => {
+ const { id, ...data } = input;
+
+ const existingCohort = await db.cohort.findUnique({
+ where: { id },
+ });
+
+ if (!existingCohort) {
+ throw TRPCNotFoundError('Cohort not found');
+ }
+
+ const access = await getProjectAccess({
+ projectId: existingCohort.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this cohort');
+ }
+
+ const cohort = await db.cohort.update({
+ where: { id },
+ data: {
+ ...(data.name && { name: data.name }),
+ ...(data.description !== undefined && { description: data.description }),
+ ...(data.definition && { definition: data.definition as any }),
+ ...(data.isStatic !== undefined && { isStatic: data.isStatic }),
+ ...(data.computeOnDemand !== undefined && {
+ computeOnDemand: data.computeOnDemand,
+ }),
+ },
+ });
+
+ // If definition changed, trigger recomputation
+ if (data.definition && !cohort.computeOnDemand) {
+ updateCohortMembership(cohort.id).catch((err) => {
+ console.error('Failed to recompute cohort after update:', err);
+ });
+ }
+
+ return cohort;
+ }),
+
+ /**
+ * Delete cohort
+ */
+ delete: protectedProcedure
+ .input(z.object({ id: z.string() }))
+ .mutation(async ({ input, ctx }) => {
+ const cohort = await db.cohort.findUnique({
+ where: { id: input.id },
+ });
+
+ if (!cohort) {
+ throw TRPCNotFoundError('Cohort not found');
+ }
+
+ const access = await getProjectAccess({
+ projectId: cohort.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this cohort');
+ }
+
+ await db.cohort.delete({
+ where: { id: input.id },
+ });
+
+ // Cleanup ClickHouse data in background
+ chQuery(`
+ DELETE FROM ${TABLE_NAMES.cohort_members}
+ WHERE cohort_id = ${sqlstring.escape(input.id)}
+ `).catch((err) => {
+ console.error('Failed to cleanup cohort_members:', err);
+ });
+
+ chQuery(`
+ DELETE FROM ${TABLE_NAMES.cohort_metadata}
+ WHERE cohort_id = ${sqlstring.escape(input.id)}
+ `).catch((err) => {
+ console.error('Failed to cleanup cohort_metadata:', err);
+ });
+
+ return { success: true };
+ }),
+
+ /**
+ * Get cohort members with pagination
+ */
+ getProfiles: protectedProcedure
+ .input(
+ z.object({
+ cohortId: z.string(),
+ limit: z.number().min(1).max(500).default(50),
+ offset: z.number().min(0).default(0),
+ }),
+ )
+ .query(async ({ input, ctx }) => {
+ const cohort = await db.cohort.findUnique({
+ where: { id: input.cohortId },
+ });
+
+ if (!cohort) {
+ throw TRPCNotFoundError('Cohort not found');
+ }
+
+ const access = await getProjectAccess({
+ projectId: cohort.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this cohort');
+ }
+
+ const result = await getCohortMembers(
+ input.cohortId,
+ cohort.projectId,
+ { limit: input.limit, offset: input.offset },
+ );
+
+ // Enrich with profile data
+ if (result.profileIds.length > 0) {
+ const profiles = await chQuery<{
+ id: string;
+ email: string;
+ first_name: string;
+ last_name: string;
+ properties: Record;
+ created_at: string;
+ }>(`
+ SELECT
+ id,
+ email,
+ first_name,
+ last_name,
+ properties,
+ created_at
+ FROM ${TABLE_NAMES.profiles} FINAL
+ WHERE project_id = ${sqlstring.escape(cohort.projectId)}
+ AND id IN (${result.profileIds.map((id) => sqlstring.escape(id)).join(',')})
+ `);
+
+ return {
+ profiles,
+ total: result.total,
+ };
+ }
+
+ return {
+ profiles: [],
+ total: result.total,
+ };
+ }),
+
+ /**
+ * Get cohort count
+ */
+ getCount: protectedProcedure
+ .input(z.object({ cohortId: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const cohort = await db.cohort.findUnique({
+ where: { id: input.cohortId },
+ });
+
+ if (!cohort) {
+ throw TRPCNotFoundError('Cohort not found');
+ }
+
+ const access = await getProjectAccess({
+ projectId: cohort.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this cohort');
+ }
+
+ const count = await getCohortCount(input.cohortId, cohort.projectId);
+ return { count };
+ }),
+
+ /**
+ * Preview cohort without saving
+ */
+ preview: protectedProcedure
+ .input(
+ z.object({
+ projectId: z.string(),
+ definition: zCohortDefinition,
+ }),
+ )
+ .query(async ({ input, ctx }) => {
+ const access = await getProjectAccess({
+ projectId: input.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this project');
+ }
+
+ const definition = input.definition as CohortDefinition;
+
+ // Use COUNT to get total without loading all profiles into memory
+ const count = await countCohort(input.projectId, definition);
+
+ // Get just 10 sample profiles
+ const sampleProfiles = await computeCohort(input.projectId, definition, 10);
+
+ return {
+ count,
+ sampleProfiles,
+ };
+ }),
+
+ /**
+ * Export cohort member IDs with pagination
+ */
+ exportProfiles: protectedProcedure
+ .input(
+ z.object({
+ cohortId: z.string(),
+ limit: z.number().min(1).max(10000).default(10000),
+ offset: z.number().min(0).default(0),
+ }),
+ )
+ .query(async ({ input, ctx }) => {
+ const cohort = await db.cohort.findUnique({
+ where: { id: input.cohortId },
+ });
+
+ if (!cohort) {
+ throw TRPCNotFoundError('Cohort not found');
+ }
+
+ const access = await getProjectAccess({
+ projectId: cohort.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this cohort');
+ }
+
+ const result = await getCohortMembers(input.cohortId, cohort.projectId, {
+ limit: input.limit,
+ offset: input.offset,
+ });
+
+ return {
+ profileIds: result.profileIds,
+ total: result.total,
+ cohortName: cohort.name,
+ };
+ }),
+
+ /**
+ * Manually trigger cohort refresh
+ */
+ refresh: protectedProcedure
+ .input(z.object({ cohortId: z.string() }))
+ .mutation(async ({ input, ctx }) => {
+ const cohort = await db.cohort.findUnique({
+ where: { id: input.cohortId },
+ });
+
+ if (!cohort) {
+ throw TRPCNotFoundError('Cohort not found');
+ }
+
+ const access = await getProjectAccess({
+ projectId: cohort.projectId,
+ userId: ctx.session.userId,
+ });
+
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this cohort');
+ }
+
+ // Static cohorts are snapshots - don't update
+ if (cohort.isStatic) {
+ throw new Error('Cannot refresh static cohorts - they are one-time snapshots');
+ }
+
+ // For dynamic cohorts, just update the cached count (don't store membership)
+ if (cohort.computeOnDemand) {
+ const definition = cohort.definition as CohortDefinition;
+ const count = await countCohort(cohort.projectId, definition);
+
+ await db.cohort.update({
+ where: { id: input.cohortId },
+ data: {
+ profileCount: count,
+ lastComputedAt: new Date(),
+ },
+ });
+ } else {
+ // Pre-computed cohorts: compute and store membership
+ await updateCohortMembership(input.cohortId);
+ }
+
+ return { success: true };
+ }),
+});
diff --git a/packages/trpc/src/routers/custom-event.ts b/packages/trpc/src/routers/custom-event.ts
new file mode 100644
index 000000000..667552fe8
--- /dev/null
+++ b/packages/trpc/src/routers/custom-event.ts
@@ -0,0 +1,184 @@
+import { z } from 'zod';
+import { db } from '@openpanel/db';
+import {
+ zCustomEventInput,
+ zCustomEventUpdate,
+} from '@openpanel/validation';
+import { getProjectAccess } from '../access';
+import { TRPCAccessError, TRPCNotFoundError } from '../errors';
+import { createTRPCRouter, protectedProcedure } from '../trpc';
+
+export const customEventRouter = createTRPCRouter({
+ /**
+ * List all custom events for a project
+ */
+ list: protectedProcedure
+ .input(z.object({ projectId: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const access = await getProjectAccess({
+ projectId: input.projectId,
+ userId: ctx.session.userId,
+ });
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this project');
+ }
+
+ return db.customEvent.findMany({
+ where: { projectId: input.projectId },
+ orderBy: { createdAt: 'desc' },
+ });
+ }),
+
+ /**
+ * Get single custom event by ID
+ */
+ get: protectedProcedure
+ .input(z.object({ id: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const customEvent = await db.customEvent.findUnique({
+ where: { id: input.id },
+ });
+
+ if (!customEvent) {
+ throw TRPCNotFoundError('Custom event not found');
+ }
+
+ const access = await getProjectAccess({
+ projectId: customEvent.projectId,
+ userId: ctx.session.userId,
+ });
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this custom event');
+ }
+
+ return customEvent;
+ }),
+
+ /**
+ * Create new custom event
+ */
+ create: protectedProcedure
+ .input(zCustomEventInput)
+ .mutation(async ({ input, ctx }) => {
+ const access = await getProjectAccess({
+ projectId: input.projectId,
+ userId: ctx.session.userId,
+ });
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this project');
+ }
+
+ // Check if name already exists (as custom event or regular event)
+ const [existingCustom, existingMeta] = await Promise.all([
+ db.customEvent.findUnique({
+ where: {
+ name_projectId: {
+ name: input.name,
+ projectId: input.projectId,
+ },
+ },
+ }),
+ db.eventMeta.findUnique({
+ where: {
+ name_projectId: {
+ name: input.name,
+ projectId: input.projectId,
+ },
+ },
+ }),
+ ]);
+
+ if (existingCustom || existingMeta) {
+ throw new Error('An event with this name already exists');
+ }
+
+ return db.customEvent.create({
+ data: {
+ name: input.name,
+ description: input.description,
+ projectId: input.projectId,
+ definition: input.definition as any,
+ conversion: input.conversion,
+ },
+ });
+ }),
+
+ /**
+ * Update custom event
+ */
+ update: protectedProcedure
+ .input(zCustomEventUpdate)
+ .mutation(async ({ input, ctx }) => {
+ const { id, ...data } = input;
+
+ const existing = await db.customEvent.findUnique({ where: { id } });
+ if (!existing) {
+ throw TRPCNotFoundError('Custom event not found');
+ }
+
+ const access = await getProjectAccess({
+ projectId: existing.projectId,
+ userId: ctx.session.userId,
+ });
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this custom event');
+ }
+
+ // If name is being changed, check for conflicts
+ if (data.name && data.name !== existing.name) {
+ const [existingCustom, existingMeta] = await Promise.all([
+ db.customEvent.findUnique({
+ where: {
+ name_projectId: {
+ name: data.name,
+ projectId: existing.projectId,
+ },
+ },
+ }),
+ db.eventMeta.findUnique({
+ where: {
+ name_projectId: {
+ name: data.name,
+ projectId: existing.projectId,
+ },
+ },
+ }),
+ ]);
+
+ if (existingCustom || existingMeta) {
+ throw new Error('An event with this name already exists');
+ }
+ }
+
+ return db.customEvent.update({
+ where: { id },
+ data: data as any,
+ });
+ }),
+
+ /**
+ * Delete custom event
+ */
+ delete: protectedProcedure
+ .input(z.object({ id: z.string() }))
+ .mutation(async ({ input, ctx }) => {
+ const existing = await db.customEvent.findUnique({
+ where: { id: input.id },
+ });
+ if (!existing) {
+ throw TRPCNotFoundError('Custom event not found');
+ }
+
+ const access = await getProjectAccess({
+ projectId: existing.projectId,
+ userId: ctx.session.userId,
+ });
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this custom event');
+ }
+
+ return db.customEvent.delete({
+ where: { id: input.id },
+ });
+ }),
+});
diff --git a/packages/trpc/src/routers/profile.ts b/packages/trpc/src/routers/profile.ts
index 64159f493..b9c2dd627 100644
--- a/packages/trpc/src/routers/profile.ts
+++ b/packages/trpc/src/routers/profile.ts
@@ -65,7 +65,7 @@ export const profileRouter = createTRPCRouter({
.map((item) => item.replace(/\.([0-9]+)/g, '[*]'))
.map((item) => `properties.${item}`);
- properties.push('id', 'first_name', 'last_name', 'email');
+ properties.push('id', 'first_name', 'last_name', 'email', 'created_at');
return pipe(
sort((a, b) => a.length - b.length),
diff --git a/packages/trpc/src/routers/report.ts b/packages/trpc/src/routers/report.ts
index b17acb8ca..a7c37e4c2 100644
--- a/packages/trpc/src/routers/report.ts
+++ b/packages/trpc/src/routers/report.ts
@@ -18,6 +18,26 @@ export const reportRouter = createTRPCRouter({
.query(async ({ input: { dashboardId, projectId }, ctx }) => {
return getReportsByDashboardId(dashboardId);
}),
+ listByProject: protectedProcedure
+ .input(
+ z.object({
+ projectId: z.string(),
+ }),
+ )
+ .query(async ({ input: { projectId }, ctx }) => {
+ const access = await getProjectAccess({
+ userId: ctx.session.userId,
+ projectId,
+ });
+ if (!access) {
+ throw TRPCAccessError('You do not have access to this project');
+ }
+ return db.report.findMany({
+ where: { projectId },
+ select: { id: true, name: true, chartType: true },
+ orderBy: { name: 'asc' },
+ });
+ }),
create: protectedProcedure
.input(
z.object({
@@ -57,8 +77,11 @@ export const reportRouter = createTRPCRouter({
unit: report.unit,
criteria: report.criteria,
metric: report.metric === 'count' ? 'sum' : report.metric,
- funnelGroup: report.funnelGroup,
- funnelWindow: report.funnelWindow,
+ funnelGroup: report.funnelGroup ?? null,
+ funnelWindow: report.funnelWindow ?? null,
+ measuring: report.measuring ?? null,
+ globalFilters: report.globalFilters ?? [],
+ holdProperties: report.holdProperties ?? [],
},
});
}),
@@ -102,8 +125,11 @@ export const reportRouter = createTRPCRouter({
unit: report.unit,
criteria: report.criteria,
metric: report.metric === 'count' ? 'sum' : report.metric,
- funnelGroup: report.funnelGroup,
- funnelWindow: report.funnelWindow,
+ funnelGroup: report.funnelGroup ?? null,
+ funnelWindow: report.funnelWindow ?? null,
+ measuring: report.measuring ?? null,
+ globalFilters: report.globalFilters ?? [],
+ holdProperties: report.holdProperties ?? [],
},
});
}),
@@ -175,6 +201,8 @@ export const reportRouter = createTRPCRouter({
metric: report.metric,
funnelGroup: report.funnelGroup,
funnelWindow: report.funnelWindow,
+ globalFilters: report.globalFilters ?? [],
+ holdProperties: report.holdProperties ?? [],
},
});
}),
diff --git a/packages/validation/src/cohort.validation.test.ts b/packages/validation/src/cohort.validation.test.ts
new file mode 100644
index 000000000..f40680fea
--- /dev/null
+++ b/packages/validation/src/cohort.validation.test.ts
@@ -0,0 +1,371 @@
+import { describe, expect, it } from 'vitest';
+import {
+ zCohortDefinition,
+ zCohortInput,
+ zCohortUpdate,
+ zEventCriteria,
+ zFrequency,
+ zTimeframe,
+} from './cohort.validation';
+
+describe('Cohort Validation', () => {
+ describe('zTimeframe', () => {
+ it('should validate relative timeframe', () => {
+ const result = zTimeframe.safeParse({
+ type: 'relative',
+ value: '30d',
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should validate absolute timeframe', () => {
+ const result = zTimeframe.safeParse({
+ type: 'absolute',
+ value: '2024-01-01',
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should reject invalid timeframe type', () => {
+ const result = zTimeframe.safeParse({
+ type: 'invalid',
+ value: '30d',
+ });
+ expect(result.success).toBe(false);
+ });
+ });
+
+ describe('zFrequency', () => {
+ it('should validate gte operator', () => {
+ const result = zFrequency.safeParse({
+ operator: 'gte',
+ value: 5,
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should validate eq operator', () => {
+ const result = zFrequency.safeParse({
+ operator: 'eq',
+ value: 1,
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should validate lte operator', () => {
+ const result = zFrequency.safeParse({
+ operator: 'lte',
+ value: 10,
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should reject negative values', () => {
+ const result = zFrequency.safeParse({
+ operator: 'gte',
+ value: -1,
+ });
+ expect(result.success).toBe(false);
+ });
+
+ it('should reject invalid operator', () => {
+ const result = zFrequency.safeParse({
+ operator: 'invalid',
+ value: 5,
+ });
+ expect(result.success).toBe(false);
+ });
+ });
+
+ describe('zEventCriteria', () => {
+ it('should validate basic event criteria', () => {
+ const result = zEventCriteria.safeParse({
+ name: 'page_view',
+ filters: [],
+ timeframe: { type: 'relative', value: '30d' },
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should validate event criteria with frequency', () => {
+ const result = zEventCriteria.safeParse({
+ name: 'purchase',
+ filters: [],
+ timeframe: { type: 'relative', value: '7d' },
+ frequency: { operator: 'gte', value: 3 },
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should validate event criteria with filters', () => {
+ const result = zEventCriteria.safeParse({
+ name: 'click',
+ filters: [
+ {
+ id: 'filter1',
+ name: 'button_name',
+ operator: 'is',
+ value: ['submit'],
+ },
+ ],
+ timeframe: { type: 'absolute', value: '2024-01-01' },
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should reject empty event name', () => {
+ const result = zEventCriteria.safeParse({
+ name: '',
+ filters: [],
+ timeframe: { type: 'relative', value: '30d' },
+ });
+ expect(result.success).toBe(false);
+ });
+ });
+
+ describe('zCohortDefinition - Event-based', () => {
+ it('should validate event-based cohort with OR operator', () => {
+ const result = zCohortDefinition.safeParse({
+ type: 'event',
+ criteria: {
+ events: [
+ {
+ name: 'signup',
+ filters: [],
+ timeframe: { type: 'relative', value: '30d' },
+ frequency: { operator: 'gte', value: 1 },
+ },
+ ],
+ operator: 'or',
+ },
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should validate event-based cohort with AND operator', () => {
+ const result = zCohortDefinition.safeParse({
+ type: 'event',
+ criteria: {
+ events: [
+ {
+ name: 'signup',
+ filters: [],
+ timeframe: { type: 'relative', value: '30d' },
+ },
+ {
+ name: 'purchase',
+ filters: [],
+ timeframe: { type: 'relative', value: '7d' },
+ frequency: { operator: 'gte', value: 1 },
+ },
+ ],
+ operator: 'and',
+ },
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should validate event-based cohort with multiple events and filters', () => {
+ const result = zCohortDefinition.safeParse({
+ type: 'event',
+ criteria: {
+ events: [
+ {
+ name: 'page_view',
+ filters: [
+ {
+ id: 'f1',
+ name: 'path',
+ operator: 'is',
+ value: ['/pricing'],
+ },
+ ],
+ timeframe: { type: 'relative', value: '7d' },
+ frequency: { operator: 'gte', value: 5 },
+ },
+ {
+ name: 'click',
+ filters: [
+ {
+ id: 'f2',
+ name: 'element',
+ operator: 'is',
+ value: ['button'],
+ },
+ ],
+ timeframe: { type: 'relative', value: '7d' },
+ },
+ ],
+ operator: 'or',
+ },
+ });
+ expect(result.success).toBe(true);
+ });
+ });
+
+ describe('zCohortDefinition - Property-based', () => {
+ it('should validate property-based cohort with single property', () => {
+ const result = zCohortDefinition.safeParse({
+ type: 'property',
+ criteria: {
+ properties: [
+ {
+ id: 'p1',
+ name: 'email',
+ operator: 'isSet',
+ value: [],
+ },
+ ],
+ operator: 'or',
+ },
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should validate property-based cohort with multiple properties', () => {
+ const result = zCohortDefinition.safeParse({
+ type: 'property',
+ criteria: {
+ properties: [
+ {
+ id: 'p1',
+ name: 'plan',
+ operator: 'is',
+ value: ['premium'],
+ },
+ {
+ id: 'p2',
+ name: 'country',
+ operator: 'is',
+ value: ['US'],
+ },
+ ],
+ operator: 'and',
+ },
+ });
+ expect(result.success).toBe(true);
+ });
+ });
+
+ describe('zCohortInput', () => {
+ it('should validate complete cohort input', () => {
+ const result = zCohortInput.safeParse({
+ name: 'Active Users',
+ description: 'Users who logged in recently',
+ projectId: 'proj-123',
+ definition: {
+ type: 'event',
+ criteria: {
+ events: [
+ {
+ name: 'login',
+ filters: [],
+ timeframe: { type: 'relative', value: '7d' },
+ frequency: { operator: 'gte', value: 1 },
+ },
+ ],
+ operator: 'or',
+ },
+ },
+ isStatic: false,
+ computeOnDemand: false,
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should validate cohort input with defaults', () => {
+ const result = zCohortInput.safeParse({
+ name: 'Test Cohort',
+ projectId: 'proj-123',
+ definition: {
+ type: 'property',
+ criteria: {
+ properties: [
+ {
+ id: 'p1',
+ name: 'email',
+ operator: 'isSet',
+ value: [],
+ },
+ ],
+ operator: 'or',
+ },
+ },
+ });
+ expect(result.success).toBe(true);
+ if (result.success) {
+ expect(result.data.isStatic).toBe(false);
+ expect(result.data.computeOnDemand).toBe(false);
+ }
+ });
+
+ it('should reject cohort without name', () => {
+ const result = zCohortInput.safeParse({
+ projectId: 'proj-123',
+ definition: {
+ type: 'event',
+ criteria: {
+ events: [],
+ operator: 'or',
+ },
+ },
+ });
+ expect(result.success).toBe(false);
+ });
+
+ it('should reject cohort without projectId', () => {
+ const result = zCohortInput.safeParse({
+ name: 'Test',
+ definition: {
+ type: 'event',
+ criteria: {
+ events: [],
+ operator: 'or',
+ },
+ },
+ });
+ expect(result.success).toBe(false);
+ });
+ });
+
+ describe('zCohortUpdate', () => {
+ it('should validate cohort update with all fields', () => {
+ const result = zCohortUpdate.safeParse({
+ id: 'cohort-123',
+ name: 'Updated Name',
+ description: 'Updated description',
+ definition: {
+ type: 'event',
+ criteria: {
+ events: [
+ {
+ name: 'signup',
+ filters: [],
+ timeframe: { type: 'relative', value: '30d' },
+ },
+ ],
+ operator: 'or',
+ },
+ },
+ isStatic: true,
+ computeOnDemand: false,
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should validate cohort update with partial fields', () => {
+ const result = zCohortUpdate.safeParse({
+ id: 'cohort-123',
+ name: 'New Name',
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('should reject update without id', () => {
+ const result = zCohortUpdate.safeParse({
+ name: 'New Name',
+ });
+ expect(result.success).toBe(false);
+ });
+ });
+});
diff --git a/packages/validation/src/cohort.validation.ts b/packages/validation/src/cohort.validation.ts
new file mode 100644
index 000000000..b6a153185
--- /dev/null
+++ b/packages/validation/src/cohort.validation.ts
@@ -0,0 +1,159 @@
+import { z } from 'zod';
+import { operators } from '@openpanel/constants';
+
+// Helper function for enum conversion
+function objectToZodEnums(
+ obj: Record,
+): [K, ...K[]] {
+ const [firstKey, ...otherKeys] = Object.keys(obj) as K[];
+ return [firstKey!, ...otherKeys];
+}
+
+// Define zChartEventFilter locally to avoid circular dependency
+const zChartEventFilter = z.object({
+ id: z.string().optional().describe('Unique identifier for the filter'),
+ name: z.string().describe('The property name to filter on'),
+ operator: z
+ .enum(objectToZodEnums(operators))
+ .describe('The operator to use for the filter'),
+ value: z
+ .array(z.string().or(z.number()).or(z.boolean()).or(z.null()))
+ .describe('The values to filter on'),
+ cohortId: z
+ .string()
+ .optional()
+ .describe('Cohort ID when using inCohort/notInCohort operators'),
+});
+
+// Timeframe schemas
+export const zRelativeTimeframe = z.object({
+ type: z.literal('relative'),
+ value: z.enum(['7d', '30d', '90d', '180d', '365d']),
+});
+
+export const zAbsoluteTimeframe = z.object({
+ type: z.literal('absolute'),
+ start: z.string(), // ISO date string
+ end: z.string().optional(), // ISO date string, optional for "since date"
+});
+
+export const zTimeframe = z.discriminatedUnion('type', [
+ zRelativeTimeframe,
+ zAbsoluteTimeframe,
+]);
+
+export type Timeframe = z.infer;
+
+// Frequency schema
+export const zFrequency = z.object({
+ operator: z.enum(['at_least', 'exactly', 'at_most']),
+ count: z.number().int().min(1),
+});
+
+export type Frequency = z.infer;
+
+// Event criteria schema
+export const zEventCriteria = z.object({
+ name: z.string().min(1).describe('The event name to match'),
+ filters: z
+ .array(zChartEventFilter)
+ .default([])
+ .describe('Filters applied to event properties'),
+ timeframe: zTimeframe.describe('When the event should have occurred'),
+ frequency: zFrequency
+ .optional()
+ .describe('How many times the event should have occurred'),
+});
+
+export type EventCriteria = z.infer;
+
+// Event-based cohort definition
+export const zEventBasedCohortDefinition = z.object({
+ type: z.literal('event'),
+ criteria: z.object({
+ operator: z.enum(['and', 'or']).describe('How to combine multiple events'),
+ events: z
+ .array(zEventCriteria)
+ .min(1)
+ .describe('Array of event criteria to match'),
+ }),
+});
+
+export type EventBasedCohortDefinition = z.infer<
+ typeof zEventBasedCohortDefinition
+>;
+
+// Property-based cohort definition
+export const zPropertyBasedCohortDefinition = z.object({
+ type: z.literal('property'),
+ criteria: z.object({
+ operator: z
+ .enum(['and', 'or'])
+ .describe('How to combine multiple properties'),
+ properties: z
+ .array(zChartEventFilter)
+ .min(1)
+ .describe('Array of profile property filters'),
+ }),
+});
+
+export type PropertyBasedCohortDefinition = z.infer<
+ typeof zPropertyBasedCohortDefinition
+>;
+
+// Combined cohort definition
+export const zCohortDefinition = z.discriminatedUnion('type', [
+ zEventBasedCohortDefinition,
+ zPropertyBasedCohortDefinition,
+]);
+
+export type CohortDefinition = z.infer;
+
+// Cohort input for creation
+export const zCohortInput = z.object({
+ name: z
+ .string()
+ .min(1)
+ .max(255)
+ .describe('User-friendly name for the cohort'),
+ description: z
+ .string()
+ .max(1000)
+ .optional()
+ .describe('Optional description of the cohort'),
+ projectId: z.string().describe('The project this cohort belongs to'),
+ definition: zCohortDefinition.describe('The cohort criteria definition'),
+ isStatic: z
+ .boolean()
+ .default(false)
+ .describe('Whether this is a static cohort (one-time snapshot)'),
+ computeOnDemand: z
+ .boolean()
+ .default(true)
+ .describe('Whether to compute on-demand instead of storing'),
+});
+
+export type CohortInput = z.infer;
+
+// Cohort update schema
+export const zCohortUpdate = z.object({
+ id: z.string().describe('The cohort ID to update'),
+ name: z.string().min(1).max(255).optional(),
+ description: z.string().max(1000).optional().nullable(),
+ definition: zCohortDefinition.optional(),
+ isStatic: z.boolean().optional(),
+ computeOnDemand: z.boolean().optional(),
+});
+
+export type CohortUpdate = z.infer;
+
+// Cohort filter schema (for using cohorts as filters in charts)
+export const zCohortFilter = z.object({
+ cohortId: z.string().describe('The cohort ID to filter by'),
+ operator: z
+ .enum(['inCohort', 'notInCohort'])
+ .default('inCohort')
+ .describe('Whether to include or exclude cohort members'),
+});
+
+export type CohortFilter = z.infer;
diff --git a/packages/validation/src/custom-event.validation.ts b/packages/validation/src/custom-event.validation.ts
new file mode 100644
index 000000000..25236038e
--- /dev/null
+++ b/packages/validation/src/custom-event.validation.ts
@@ -0,0 +1,70 @@
+import { z } from 'zod';
+import { operators } from '@openpanel/constants';
+
+// Helper function for enum conversion
+function objectToZodEnums(
+ obj: Record,
+): [K, ...K[]] {
+ const [firstKey, ...otherKeys] = Object.keys(obj) as K[];
+ return [firstKey!, ...otherKeys];
+}
+
+// Define zChartEventFilter locally to avoid circular dependency
+const zChartEventFilter = z.object({
+ id: z.string().optional().describe('Unique identifier for the filter'),
+ name: z.string().describe('The property name to filter on'),
+ operator: z
+ .enum(objectToZodEnums(operators))
+ .describe('The operator to use for the filter'),
+ value: z
+ .array(z.string().or(z.number()).or(z.boolean()).or(z.null()))
+ .describe('The values to filter on'),
+ cohortId: z
+ .string()
+ .optional()
+ .describe('Cohort ID when using inCohort/notInCohort operators'),
+});
+
+// Event criteria for custom events (simplified - no timeframe/frequency)
+export const zCustomEventCriteria = z.object({
+ name: z.string().min(1).describe('The source event name'),
+ filters: z
+ .array(zChartEventFilter)
+ .default([])
+ .describe('Filters applied to this event'),
+});
+
+export type ICustomEventCriteria = z.infer;
+
+// Custom event definition (OR-only logic)
+export const zCustomEventDefinition = z.object({
+ operator: z.literal('or'),
+ events: z
+ .array(zCustomEventCriteria)
+ .min(1)
+ .max(20)
+ .describe('Source events to combine (max 20)'),
+});
+
+export type ICustomEventDefinition = z.infer;
+
+// Create/update schemas
+export const zCustomEventInput = z.object({
+ name: z.string().min(1).max(100),
+ description: z.string().max(1000).optional(),
+ projectId: z.string(),
+ definition: zCustomEventDefinition,
+ conversion: z.boolean().default(false),
+});
+
+export type ICustomEventInput = z.infer;
+
+export const zCustomEventUpdate = z.object({
+ id: z.string(),
+ name: z.string().min(1).max(100).optional(),
+ description: z.string().max(1000).optional().nullable(),
+ definition: zCustomEventDefinition.optional(),
+ conversion: z.boolean().optional(),
+});
+
+export type ICustomEventUpdate = z.infer;
diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts
index bc8777ee2..d5811a17f 100644
--- a/packages/validation/src/index.ts
+++ b/packages/validation/src/index.ts
@@ -28,6 +28,10 @@ export const zChartEventFilter = z.object({
value: z
.array(z.string().or(z.number()).or(z.boolean()).or(z.null()))
.describe('The values to filter on'),
+ cohortId: z
+ .string()
+ .optional()
+ .describe('Cohort ID when using inCohort/notInCohort operators'),
});
export const zChartEventSegment = z
@@ -84,6 +88,7 @@ export const zChartEventItem = z.discriminatedUnion('type', [
export const zChartBreakdown = z.object({
id: z.string().optional(),
name: z.string(),
+ cohortId: z.string().optional(),
});
// Support both old format (array of events without type) and new format (array of event/formula items)
@@ -182,6 +187,7 @@ export const zChartInputBase = z.object({
limit: z
.number()
.optional()
+ .default(50)
.describe('Limit how many series should be returned'),
offset: z
.number()
@@ -200,6 +206,31 @@ export const zChartInputBase = z.object({
.number()
.optional()
.describe('Time window in hours for funnel analysis'),
+ cohortFilters: z
+ .array(
+ z.object({
+ cohortId: z.string(),
+ operator: z.enum(['inCohort', 'notInCohort']).default('inCohort'),
+ }),
+ )
+ .optional()
+ .default([])
+ .describe('Global cohort filters applied to the entire chart'),
+ globalFilters: z
+ .array(zChartEventFilter)
+ .optional()
+ .default([])
+ .describe('Global filters applied to all events in the chart'),
+ holdProperties: z
+ .array(z.string())
+ .optional()
+ .default([])
+ .describe('Properties to hold constant across all funnel steps'),
+ measuring: z
+ .enum(['conversion_rate', 'time_to_convert'])
+ .optional()
+ .default('conversion_rate')
+ .describe('What to measure: conversion rate or time to convert'),
});
export const zChartInput = z.preprocess((val) => {
@@ -402,9 +433,37 @@ export type INotificationRuleFunnelConfig = z.infer<
typeof zNotificationRuleFunnelConfig
>;
+export const zAlertFrequency = z.enum(['hour', 'day', 'week', 'month']);
+export type IAlertFrequency = z.infer;
+
+export const zNotificationRuleThresholdConfig = z.object({
+ type: z.literal('threshold'),
+ reportId: z.string().uuid(),
+ operator: z.enum(['above', 'below']),
+ value: z.number(),
+ frequency: zAlertFrequency,
+});
+
+export type INotificationRuleThresholdConfig = z.infer<
+ typeof zNotificationRuleThresholdConfig
+>;
+
+export const zNotificationRuleAnomalyConfig = z.object({
+ type: z.literal('anomaly'),
+ reportId: z.string().uuid(),
+ confidence: z.enum(['95', '98', '99']),
+ frequency: zAlertFrequency,
+});
+
+export type INotificationRuleAnomalyConfig = z.infer<
+ typeof zNotificationRuleAnomalyConfig
+>;
+
export const zNotificationRuleConfig = z.discriminatedUnion('type', [
zNotificationRuleEventConfig,
zNotificationRuleFunnelConfig,
+ zNotificationRuleThresholdConfig,
+ zNotificationRuleAnomalyConfig,
]);
export type INotificationRuleConfig = z.infer;
@@ -553,3 +612,9 @@ export const zCreateImport = z.object({
});
export type ICreateImport = z.infer;
+
+// Export cohort validation schemas
+export * from './cohort.validation';
+
+// Export custom event validation schemas
+export * from './custom-event.validation';
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index c45412d52..ab4a7e68c 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -861,6 +861,9 @@ importers:
'@openpanel/common':
specifier: workspace:*
version: link:../../packages/common
+ '@openpanel/constants':
+ specifier: workspace:*
+ version: link:../../packages/constants
'@openpanel/db':
specifier: workspace:*
version: link:../../packages/db
@@ -11261,7 +11264,7 @@ packages:
glob@6.0.4:
resolution: {integrity: sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==}
- deprecated: Glob versions prior to v9 are no longer supported
+ deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me
glob@7.1.6:
resolution: {integrity: sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==}
@@ -15343,6 +15346,7 @@ packages:
tar@6.2.0:
resolution: {integrity: sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==}
engines: {node: '>=10'}
+ deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me
tar@7.4.3:
resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==}