diff --git a/apps/api/src/routes/queries.ts b/apps/api/src/routes/queries.ts index 072b13cdc..e9562bb26 100644 --- a/apps/api/src/routes/queries.ts +++ b/apps/api/src/routes/queries.ts @@ -3,12 +3,22 @@ import * as Integrations from "@maple/query-engine-integrations" import { defineQuery } from "@maple/query-engine/registry" import { Queries as Core } from "@maple/query-engine/registry" import type { + CloudflareInfraZoneDetailRequest, + ServicePlanetScaleStatsRequest, + CloudflareInfraPlatformResourcesRequest, CloudflareInfraWorkerTimeseriesRequest, + CloudflareInfraWorkersRequest, + CloudflareInfraZoneDnsRequest, + CloudflareInfraZoneHostsRequest, + CloudflareInfraZoneSecurityRequest, CloudflareInfraZoneTimeseriesRequest, + CloudflareInfraZonesRequest, FleetUtilizationTimeseriesRequest, GetLogRequest, NodeInfraTimeseriesRequest, + PlanetScaleInfraTimeseriesRequest, PodInfraTimeseriesRequest, + ServiceCloudflareStatsRequest, SpanDetailRequest, WorkloadInfraTimeseriesRequest, } from "@maple/domain/http" @@ -38,6 +48,407 @@ import { traceCacheTtlSeconds } from "@/services/warehouse/trace-detail-cache" * Handlers import `Queries` from here, so the split is invisible at the call * site and an entry can move between the two halves without touching handlers. */ +// --- Cloudflare / PlanetScale integration queries ------------------------- +// +// These live app-side rather than in the core registry because +// `@maple/query-engine-integrations` depends on `@maple/query-engine`; +// declaring them there would invert that edge. +// +// Each entry inlines the small payload-derived prologue (`params`, `filters`, +// `base`) that used to sit in the handler. Handlers that report +// `ignoredFilters` keep their own `toCloudflareFilters` call — it is a pure +// function of the payload, so computing it in both places cannot drift, and +// which filters a metric family could not honor is presentation, not query +// construction. + +const cloudflareInfraZoneCounters = defineQuery({ + id: "cloudflareInfraZoneCounters", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraZonesRequest, orgId: string) => { + const params = { + orgId: orgId, + startTime: payload.startTime, + endTime: payload.endTime, + } + // Counters (metrics_sum) + percentiles (metrics_gauge) run + // concurrently, then merge by ServiceName — same shape as + // serviceCloudflareStats above. + const filters = toCloudflareFilters(payload) + return CH.compile(Integrations.cloudflareZoneCountersSQL(filters), params, { + rowSchema: Integrations.cloudflareZoneCountersRowSchema, + }) + }, +}) + +const cloudflareInfraZoneLatency = defineQuery({ + id: "cloudflareInfraZoneLatency", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraZonesRequest, orgId: string) => { + const params = { + orgId: orgId, + startTime: payload.startTime, + endTime: payload.endTime, + } + // Counters (metrics_sum) + percentiles (metrics_gauge) run + // concurrently, then merge by ServiceName — same shape as + // serviceCloudflareStats above. + const filters = toCloudflareFilters(payload) + return CH.compile(Integrations.cloudflareZoneLatencySQL(), params, { + rowSchema: Integrations.cloudflareZoneLatencyRowSchema, + }) + }, +}) + +const cloudflareInfraZoneHostTotals = defineQuery({ + id: "cloudflareInfraZoneHostTotals", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraZoneHostsRequest, orgId: string) => { + const params = { + orgId: orgId, + serviceName: payload.serviceName, + startTime: payload.startTime, + endTime: payload.endTime, + } + const filters = toCloudflareFilters(payload) + return CH.compile(Integrations.cloudflareZoneHostBreakdownSQL(filters), params, { + rowSchema: Integrations.cloudflareZoneHostBreakdownRowSchema, + }) + }, +}) + +const cloudflareInfraZoneHostTimeseries = defineQuery({ + id: "cloudflareInfraZoneHostTimeseries", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraZoneHostsRequest, orgId: string) => { + const params = { + orgId: orgId, + serviceName: payload.serviceName, + startTime: payload.startTime, + endTime: payload.endTime, + } + const filters = toCloudflareFilters(payload) + return CH.compile( + Integrations.cloudflareZoneHostTimeseriesSQL(filters), + { ...params, bucketSeconds: payload.bucketSeconds }, + { rowSchema: Integrations.cloudflareZoneHostTimeseriesRowSchema }, + ) + }, +}) + +const cloudflareInfraZoneFirewallTimeseries = defineQuery({ + id: "cloudflareInfraZoneFirewallTimeseries", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraZoneSecurityRequest, orgId: string) => { + const params = { + orgId: orgId, + serviceName: payload.serviceName, + startTime: payload.startTime, + endTime: payload.endTime, + } + const filters = toCloudflareFilters(payload) + return CH.compile( + Integrations.cloudflareZoneFirewallTimeseriesSQL(filters), + { ...params, bucketSeconds: payload.bucketSeconds }, + { rowSchema: Integrations.cloudflareZoneFirewallTimeseriesRowSchema }, + ) + }, +}) + +const cloudflareInfraZoneFirewallTop = defineQuery({ + id: "cloudflareInfraZoneFirewallTop", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraZoneSecurityRequest, orgId: string) => { + const params = { + orgId: orgId, + serviceName: payload.serviceName, + startTime: payload.startTime, + endTime: payload.endTime, + } + const filters = toCloudflareFilters(payload) + return CH.compile(Integrations.cloudflareZoneFirewallTopSQL(filters), params, { + rowSchema: Integrations.cloudflareZoneFirewallTopRowSchema, + }) + }, +}) + +const cloudflareInfraZoneDnsTimeseries = defineQuery({ + id: "cloudflareInfraZoneDnsTimeseries", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraZoneDnsRequest, orgId: string) => { + const params = { + orgId: orgId, + serviceName: payload.serviceName, + startTime: payload.startTime, + endTime: payload.endTime, + } + const filters = toCloudflareFilters(payload) + return CH.compile( + Integrations.cloudflareZoneDnsTimeseriesSQL(filters), + { ...params, bucketSeconds: payload.bucketSeconds }, + { rowSchema: Integrations.cloudflareZoneDnsTimeseriesRowSchema }, + ) + }, +}) + +const cloudflareInfraZoneDnsBreakdown = defineQuery({ + id: "cloudflareInfraZoneDnsBreakdown", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraZoneDnsRequest, orgId: string) => { + const params = { + orgId: orgId, + serviceName: payload.serviceName, + startTime: payload.startTime, + endTime: payload.endTime, + } + const filters = toCloudflareFilters(payload) + return CH.compile(Integrations.cloudflareZoneDnsBreakdownSQL(filters), params, { + rowSchema: Integrations.cloudflareZoneDnsBreakdownRowSchema, + }) + }, +}) + +const cloudflareInfraWorkerCounters = defineQuery({ + id: "cloudflareInfraWorkerCounters", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraWorkersRequest, orgId: string) => { + const params = { + orgId: orgId, + startTime: payload.startTime, + endTime: payload.endTime, + } + return CH.compile(Integrations.cloudflareWorkerCountersSQL(), params, { + rowSchema: Integrations.cloudflareWorkerCountersRowSchema, + }) + }, +}) + +const cloudflareInfraWorkerLatency = defineQuery({ + id: "cloudflareInfraWorkerLatency", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraWorkersRequest, orgId: string) => { + const params = { + orgId: orgId, + startTime: payload.startTime, + endTime: payload.endTime, + } + return CH.compile(Integrations.cloudflareWorkerLatencySQL(), params, { + rowSchema: Integrations.cloudflareWorkerLatencyRowSchema, + }) + }, +}) + +const cloudflareInfraQueueGauges = defineQuery({ + id: "cloudflareInfraQueueGauges", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraPlatformResourcesRequest, orgId: string) => { + const params = { + orgId: orgId, + startTime: payload.startTime, + endTime: payload.endTime, + } + return CH.compile(Integrations.cloudflareQueueGaugesSQL(), params, { + rowSchema: Integrations.cloudflareQueueGaugesRowSchema, + }) + }, +}) + +const cloudflareInfraDurableObjects = defineQuery({ + id: "cloudflareInfraDurableObjects", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraPlatformResourcesRequest, orgId: string) => { + const params = { + orgId: orgId, + startTime: payload.startTime, + endTime: payload.endTime, + } + return CH.compile(Integrations.cloudflareDurableObjectCountersSQL(), params, { + rowSchema: Integrations.cloudflareDurableObjectCountersRowSchema, + }) + }, +}) + +const cloudflareServiceCounters = defineQuery({ + id: "cloudflareServiceCounters", + profile: "aggregation", + cache: undefined, + compile: (payload: ServiceCloudflareStatsRequest, orgId: string) => { + const params = { + orgId: orgId, + startTime: payload.startTime, + endTime: payload.endTime, + } + // Counters (metrics_sum) + percentiles (metrics_gauge) run + // concurrently, then merge by ServiceName. Routed through the org's + // configured warehouse exactly like the metric explorer reads these + // same `cloudflare.*` metrics — no special ingest pin needed. + return CH.compile(Integrations.cloudflareServiceCountersSQL(), params, { + rowSchema: Integrations.cloudflareServiceCountersRowSchema, + }) + }, +}) + +const cloudflareServiceLatency = defineQuery({ + id: "cloudflareServiceLatency", + profile: "aggregation", + cache: undefined, + compile: (payload: ServiceCloudflareStatsRequest, orgId: string) => { + const params = { + orgId: orgId, + startTime: payload.startTime, + endTime: payload.endTime, + } + // Counters (metrics_sum) + percentiles (metrics_gauge) run + // concurrently, then merge by ServiceName. Routed through the org's + // configured warehouse exactly like the metric explorer reads these + // same `cloudflare.*` metrics — no special ingest pin needed. + return CH.compile(Integrations.cloudflareServiceLatencySQL(), params, { + rowSchema: Integrations.cloudflareServiceLatencyRowSchema, + }) + }, +}) + +const planetscaleInfraTimeseries = defineQuery({ + id: "planetscaleInfraTimeseries", + profile: "aggregation", + cache: undefined, + compile: (payload: PlanetScaleInfraTimeseriesRequest, orgId: string) => { + const base = { + orgId: orgId, + startTime: payload.startTime, + endTime: payload.endTime, + bucketSeconds: Math.max(60, Math.floor(payload.bucketSeconds)), + database: payload.database, + } + return payload.branch === undefined + ? CH.compile(Integrations.planetscaleInfraTimeseriesSQL(), base, { + rowSchema: Integrations.planetscaleInfraTimeseriesRowSchema, + }) + : CH.compile( + Integrations.planetscaleBranchInfraTimeseriesSQL(), + { ...base, branch: payload.branch }, + { rowSchema: Integrations.planetscaleInfraTimeseriesRowSchema }, + ) + }, +}) + +// --- cloudflareInfraZoneDetail sub-queries -------------------------------- + +const zoneDetailParams = (payload: CloudflareInfraZoneDetailRequest, orgId: string) => ({ + orgId, + serviceName: payload.serviceName, + startTime: payload.startTime, + endTime: payload.endTime, + bucketSeconds: payload.bucketSeconds, +}) + +const cloudflareInfraZoneDetailStatus = defineQuery({ + id: "cloudflareInfraZoneDetailStatus", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraZoneDetailRequest, orgId: string) => + CH.compile( + Integrations.cloudflareZoneStatusTimeseriesSQL(toCloudflareFilters(payload)), + zoneDetailParams(payload, orgId), + { rowSchema: Integrations.cloudflareZoneStatusTimeseriesRowSchema }, + ), +}) + +const cloudflareInfraZoneDetailCache = defineQuery({ + id: "cloudflareInfraZoneDetailCache", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraZoneDetailRequest, orgId: string) => + CH.compile( + Integrations.cloudflareZoneCacheTimeseriesSQL(toCloudflareFilters(payload)), + zoneDetailParams(payload, orgId), + { rowSchema: Integrations.cloudflareZoneCacheTimeseriesRowSchema }, + ), +}) + +/** Latency comes from a gauge family that honors no request filters — hence the no-arg SQL. */ +const cloudflareInfraZoneDetailLatency = defineQuery({ + id: "cloudflareInfraZoneDetailLatency", + profile: "aggregation", + cache: undefined, + compile: (payload: CloudflareInfraZoneDetailRequest, orgId: string) => + CH.compile(Integrations.cloudflareZoneLatencyTimeseriesSQL(), zoneDetailParams(payload, orgId), { + rowSchema: Integrations.cloudflareZoneLatencyTimeseriesRowSchema, + }), +}) + +// --- servicePlanetScaleStats sub-queries ---------------------------------- +// +// Each reads either the database-level or the branch-level rollup depending on +// whether a database was requested. The branch lives inside `compile` so the id, +// profile and row schema stay one decision per sub-query rather than two. + +const planetscaleStatsParams = (payload: ServicePlanetScaleStatsRequest, orgId: string) => ({ + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + ...(payload.database !== undefined ? { database: payload.database } : {}), +}) + +const planetscaleServiceGauges = defineQuery({ + id: "planetscaleServiceGauges", + profile: "aggregation", + cache: undefined, + compile: (payload: ServicePlanetScaleStatsRequest, orgId: string) => { + const params = planetscaleStatsParams(payload, orgId) + return payload.database !== undefined + ? CH.compile(Integrations.planetscaleBranchGaugesSQL(), params, { + rowSchema: Integrations.planetscaleBranchStatsRowSchema, + }) + : CH.compile(Integrations.planetscaleGaugesSQL(), params, { + rowSchema: Integrations.planetscaleDatabaseStatsRowSchema, + }) + }, +}) + +const planetscaleServiceConnections = defineQuery({ + id: "planetscaleServiceConnections", + profile: "aggregation", + cache: undefined, + compile: (payload: ServicePlanetScaleStatsRequest, orgId: string) => { + const params = planetscaleStatsParams(payload, orgId) + return payload.database !== undefined + ? CH.compile(Integrations.planetscaleBranchConnectionsSQL(), params, { + rowSchema: Integrations.planetscaleBranchConnectionsRowSchema, + }) + : CH.compile(Integrations.planetscaleConnectionsSQL(), params, { + rowSchema: Integrations.planetscaleConnectionsRowSchema, + }) + }, +}) + +const planetscaleServiceStorage = defineQuery({ + id: "planetscaleServiceStorage", + profile: "aggregation", + cache: undefined, + compile: (payload: ServicePlanetScaleStatsRequest, orgId: string) => { + const params = planetscaleStatsParams(payload, orgId) + return payload.database !== undefined + ? CH.compile(Integrations.planetscaleBranchStorageSQL(), params, { + rowSchema: Integrations.planetscaleBranchStorageRowSchema, + }) + : CH.compile(Integrations.planetscaleStorageSQL(), params, { + rowSchema: Integrations.planetscaleStorageRowSchema, + }) + }, +}) + export const Queries = { ...Core, @@ -194,4 +605,29 @@ export const Queries = { { rowSchema: Integrations.cloudflareWorkerTimeseriesRowSchema }, ), }), + + // Integration queries, declared above. + cloudflareInfraZoneCounters, + cloudflareInfraZoneLatency, + cloudflareInfraZoneHostTotals, + cloudflareInfraZoneHostTimeseries, + cloudflareInfraZoneFirewallTimeseries, + cloudflareInfraZoneFirewallTop, + cloudflareInfraZoneDnsTimeseries, + cloudflareInfraZoneDnsBreakdown, + cloudflareInfraWorkerCounters, + cloudflareInfraWorkerLatency, + cloudflareInfraQueueGauges, + cloudflareInfraDurableObjects, + cloudflareServiceCounters, + cloudflareServiceLatency, + planetscaleInfraTimeseries, + + // ZoneDetail / PlanetScaleStats sub-queries, declared above. + cloudflareInfraZoneDetailStatus, + cloudflareInfraZoneDetailCache, + cloudflareInfraZoneDetailLatency, + planetscaleServiceGauges, + planetscaleServiceConnections, + planetscaleServiceStorage, } as const diff --git a/apps/api/src/routes/v1/query-engine.http.ts b/apps/api/src/routes/v1/query-engine.http.ts index 6c430d6aa..b61b5975e 100644 --- a/apps/api/src/routes/v1/query-engine.http.ts +++ b/apps/api/src/routes/v1/query-engine.http.ts @@ -429,37 +429,14 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("serviceCloudflareStats", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const params = { - orgId: tenant.orgId, - startTime: payload.startTime, - endTime: payload.endTime, - } // Counters (metrics_sum) + percentiles (metrics_gauge) run // concurrently, then merge by ServiceName. Routed through the org's // configured warehouse exactly like the metric explorer reads these // same `cloudflare.*` metrics — no special ingest pin needed. - const countersCompiled = CH.compile(Integrations.cloudflareServiceCountersSQL(), params, { - rowSchema: Integrations.cloudflareServiceCountersRowSchema, - }) - const latencyCompiled = CH.compile(Integrations.cloudflareServiceLatencySQL(), params, { - rowSchema: Integrations.cloudflareServiceLatencyRowSchema, - }) const [counterRows, latencyRows] = yield* Effect.all( [ - mapExecError( - warehouse.compiledQuery(tenant, countersCompiled, { - profile: "aggregation", - context: "cloudflareServiceCounters", - }), - "cloudflareServiceCounters query failed", - ), - mapExecError( - warehouse.compiledQuery(tenant, latencyCompiled, { - profile: "aggregation", - context: "cloudflareServiceLatency", - }), - "cloudflareServiceLatency query failed", - ), + runQuery(Queries.cloudflareServiceCounters, tenant, payload), + runQuery(Queries.cloudflareServiceLatency, tenant, payload), ], { concurrency: 2 }, ) @@ -481,60 +458,15 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const byBranch = payload.database !== undefined - const params = { - orgId: tenant.orgId, - startTime: payload.startTime, - endTime: payload.endTime, - ...(payload.database !== undefined ? { database: payload.database } : {}), - } // Utilization gauges + the two-level connections rollup run // concurrently, then merge by database(+branch). Routed through the // org's configured warehouse like the metric explorer reads the same // scraped `planetscale_*` metrics. - const gaugesCompiled = byBranch - ? CH.compile(Integrations.planetscaleBranchGaugesSQL(), params, { - rowSchema: Integrations.planetscaleBranchStatsRowSchema, - }) - : CH.compile(Integrations.planetscaleGaugesSQL(), params, { - rowSchema: Integrations.planetscaleDatabaseStatsRowSchema, - }) - const connectionsCompiled = byBranch - ? CH.compile(Integrations.planetscaleBranchConnectionsSQL(), params, { - rowSchema: Integrations.planetscaleBranchConnectionsRowSchema, - }) - : CH.compile(Integrations.planetscaleConnectionsSQL(), params, { - rowSchema: Integrations.planetscaleConnectionsRowSchema, - }) - const storageCompiled = byBranch - ? CH.compile(Integrations.planetscaleBranchStorageSQL(), params, { - rowSchema: Integrations.planetscaleBranchStorageRowSchema, - }) - : CH.compile(Integrations.planetscaleStorageSQL(), params, { - rowSchema: Integrations.planetscaleStorageRowSchema, - }) const [gaugeRows, connectionRows, storageRows] = yield* Effect.all( [ - mapExecError( - warehouse.compiledQuery(tenant, gaugesCompiled, { - profile: "aggregation", - context: "planetscaleServiceGauges", - }), - "planetscaleServiceGauges query failed", - ), - mapExecError( - warehouse.compiledQuery(tenant, connectionsCompiled, { - profile: "aggregation", - context: "planetscaleServiceConnections", - }), - "planetscaleServiceConnections query failed", - ), - mapExecError( - warehouse.compiledQuery(tenant, storageCompiled, { - profile: "aggregation", - context: "planetscaleServiceStorage", - }), - "planetscaleServiceStorage query failed", - ), + runQuery(Queries.planetscaleServiceGauges, tenant, payload), + runQuery(Queries.planetscaleServiceConnections, tenant, payload), + runQuery(Queries.planetscaleServiceStorage, tenant, payload), ], { concurrency: 3 }, ) @@ -635,54 +567,21 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", { ...base, branch: payload.branch }, { rowSchema: Integrations.planetscaleInfraTimeseriesRowSchema }, ) - const rows = yield* mapExecError( - warehouse.compiledQuery(tenant, compiled, { - profile: "aggregation", - context: "planetscaleInfraTimeseries", - }), - "planetscaleInfraTimeseries query failed", - ) + const rows = yield* runQuery(Queries.planetscaleInfraTimeseries, tenant, payload) return new PlanetScaleInfraTimeseriesResponse({ data: rows.map((row) => ({ ...row })) }) }), ) .handle("cloudflareInfraZones", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const params = { - orgId: tenant.orgId, - startTime: payload.startTime, - endTime: payload.endTime, - } // Counters (metrics_sum) + percentiles (metrics_gauge) run // concurrently, then merge by ServiceName — same shape as // serviceCloudflareStats above. const filters = toCloudflareFilters(payload) - const countersCompiled = CH.compile( - Integrations.cloudflareZoneCountersSQL(filters), - params, - { - rowSchema: Integrations.cloudflareZoneCountersRowSchema, - }, - ) - const latencyCompiled = CH.compile(Integrations.cloudflareZoneLatencySQL(), params, { - rowSchema: Integrations.cloudflareZoneLatencyRowSchema, - }) const [counterRows, latencyRows] = yield* Effect.all( [ - mapExecError( - warehouse.compiledQuery(tenant, countersCompiled, { - profile: "aggregation", - context: "cloudflareInfraZoneCounters", - }), - "cloudflareInfraZoneCounters query failed", - ), - mapExecError( - warehouse.compiledQuery(tenant, latencyCompiled, { - profile: "aggregation", - context: "cloudflareInfraZoneLatency", - }), - "cloudflareInfraZoneLatency query failed", - ), + runQuery(Queries.cloudflareInfraZoneCounters, tenant, payload), + runQuery(Queries.cloudflareInfraZoneLatency, tenant, payload), ], { concurrency: 2 }, ) @@ -732,58 +631,12 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("cloudflareInfraZoneDetail", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const params = { - orgId: tenant.orgId, - serviceName: payload.serviceName, - startTime: payload.startTime, - endTime: payload.endTime, - bucketSeconds: payload.bucketSeconds, - } const filters = toCloudflareFilters(payload) - const statusCompiled = CH.compile( - Integrations.cloudflareZoneStatusTimeseriesSQL(filters), - params, - { - rowSchema: Integrations.cloudflareZoneStatusTimeseriesRowSchema, - }, - ) - const cacheCompiled = CH.compile( - Integrations.cloudflareZoneCacheTimeseriesSQL(filters), - params, - { - rowSchema: Integrations.cloudflareZoneCacheTimeseriesRowSchema, - }, - ) - const latencyCompiled = CH.compile( - Integrations.cloudflareZoneLatencyTimeseriesSQL(), - params, - { - rowSchema: Integrations.cloudflareZoneLatencyTimeseriesRowSchema, - }, - ) const [statusRows, cacheRows, latencyRows] = yield* Effect.all( [ - mapExecError( - warehouse.compiledQuery(tenant, statusCompiled, { - profile: "aggregation", - context: "cloudflareInfraZoneDetailStatus", - }), - "cloudflareInfraZoneDetailStatus query failed", - ), - mapExecError( - warehouse.compiledQuery(tenant, cacheCompiled, { - profile: "aggregation", - context: "cloudflareInfraZoneDetailCache", - }), - "cloudflareInfraZoneDetailCache query failed", - ), - mapExecError( - warehouse.compiledQuery(tenant, latencyCompiled, { - profile: "aggregation", - context: "cloudflareInfraZoneDetailLatency", - }), - "cloudflareInfraZoneDetailLatency query failed", - ), + runQuery(Queries.cloudflareInfraZoneDetailStatus, tenant, payload), + runQuery(Queries.cloudflareInfraZoneDetailCache, tenant, payload), + runQuery(Queries.cloudflareInfraZoneDetailLatency, tenant, payload), ], { concurrency: 3 }, ) @@ -804,41 +657,11 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("cloudflareInfraZoneHosts", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const params = { - orgId: tenant.orgId, - serviceName: payload.serviceName, - startTime: payload.startTime, - endTime: payload.endTime, - } const filters = toCloudflareFilters(payload) - const totalsCompiled = CH.compile( - Integrations.cloudflareZoneHostBreakdownSQL(filters), - params, - { - rowSchema: Integrations.cloudflareZoneHostBreakdownRowSchema, - }, - ) - const bucketsCompiled = CH.compile( - Integrations.cloudflareZoneHostTimeseriesSQL(filters), - { ...params, bucketSeconds: payload.bucketSeconds }, - { rowSchema: Integrations.cloudflareZoneHostTimeseriesRowSchema }, - ) const [totalRows, bucketRows] = yield* Effect.all( [ - mapExecError( - warehouse.compiledQuery(tenant, totalsCompiled, { - profile: "aggregation", - context: "cloudflareInfraZoneHostTotals", - }), - "cloudflareInfraZoneHostTotals query failed", - ), - mapExecError( - warehouse.compiledQuery(tenant, bucketsCompiled, { - profile: "aggregation", - context: "cloudflareInfraZoneHostTimeseries", - }), - "cloudflareInfraZoneHostTimeseries query failed", - ), + runQuery(Queries.cloudflareInfraZoneHostTotals, tenant, payload), + runQuery(Queries.cloudflareInfraZoneHostTimeseries, tenant, payload), ], { concurrency: 2 }, ) @@ -854,41 +677,11 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("cloudflareInfraZoneSecurity", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const params = { - orgId: tenant.orgId, - serviceName: payload.serviceName, - startTime: payload.startTime, - endTime: payload.endTime, - } const filters = toCloudflareFilters(payload) - const bucketsCompiled = CH.compile( - Integrations.cloudflareZoneFirewallTimeseriesSQL(filters), - { ...params, bucketSeconds: payload.bucketSeconds }, - { rowSchema: Integrations.cloudflareZoneFirewallTimeseriesRowSchema }, - ) - const topCompiled = CH.compile( - Integrations.cloudflareZoneFirewallTopSQL(filters), - params, - { - rowSchema: Integrations.cloudflareZoneFirewallTopRowSchema, - }, - ) const [bucketRows, topRows] = yield* Effect.all( [ - mapExecError( - warehouse.compiledQuery(tenant, bucketsCompiled, { - profile: "aggregation", - context: "cloudflareInfraZoneFirewallTimeseries", - }), - "cloudflareInfraZoneFirewallTimeseries query failed", - ), - mapExecError( - warehouse.compiledQuery(tenant, topCompiled, { - profile: "aggregation", - context: "cloudflareInfraZoneFirewallTop", - }), - "cloudflareInfraZoneFirewallTop query failed", - ), + runQuery(Queries.cloudflareInfraZoneFirewallTimeseries, tenant, payload), + runQuery(Queries.cloudflareInfraZoneFirewallTop, tenant, payload), ], { concurrency: 2 }, ) @@ -904,41 +697,11 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("cloudflareInfraZoneDns", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const params = { - orgId: tenant.orgId, - serviceName: payload.serviceName, - startTime: payload.startTime, - endTime: payload.endTime, - } const filters = toCloudflareFilters(payload) - const bucketsCompiled = CH.compile( - Integrations.cloudflareZoneDnsTimeseriesSQL(filters), - { ...params, bucketSeconds: payload.bucketSeconds }, - { rowSchema: Integrations.cloudflareZoneDnsTimeseriesRowSchema }, - ) - const namesCompiled = CH.compile( - Integrations.cloudflareZoneDnsBreakdownSQL(filters), - params, - { - rowSchema: Integrations.cloudflareZoneDnsBreakdownRowSchema, - }, - ) const [bucketRows, nameRows] = yield* Effect.all( [ - mapExecError( - warehouse.compiledQuery(tenant, bucketsCompiled, { - profile: "aggregation", - context: "cloudflareInfraZoneDnsTimeseries", - }), - "cloudflareInfraZoneDnsTimeseries query failed", - ), - mapExecError( - warehouse.compiledQuery(tenant, namesCompiled, { - profile: "aggregation", - context: "cloudflareInfraZoneDnsBreakdown", - }), - "cloudflareInfraZoneDnsBreakdown query failed", - ), + runQuery(Queries.cloudflareInfraZoneDnsTimeseries, tenant, payload), + runQuery(Queries.cloudflareInfraZoneDnsBreakdown, tenant, payload), ], { concurrency: 2 }, ) @@ -1141,33 +904,10 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("cloudflareInfraPlatformResources", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const params = { - orgId: tenant.orgId, - startTime: payload.startTime, - endTime: payload.endTime, - } - const queuesCompiled = CH.compile(Integrations.cloudflareQueueGaugesSQL(), params, { - rowSchema: Integrations.cloudflareQueueGaugesRowSchema, - }) - const doCompiled = CH.compile(Integrations.cloudflareDurableObjectCountersSQL(), params, { - rowSchema: Integrations.cloudflareDurableObjectCountersRowSchema, - }) const [queueRows, doRows] = yield* Effect.all( [ - mapExecError( - warehouse.compiledQuery(tenant, queuesCompiled, { - profile: "aggregation", - context: "cloudflareInfraQueueGauges", - }), - "cloudflareInfraQueueGauges query failed", - ), - mapExecError( - warehouse.compiledQuery(tenant, doCompiled, { - profile: "aggregation", - context: "cloudflareInfraDurableObjects", - }), - "cloudflareInfraDurableObjects query failed", - ), + runQuery(Queries.cloudflareInfraQueueGauges, tenant, payload), + runQuery(Queries.cloudflareInfraDurableObjects, tenant, payload), ], { concurrency: 2 }, ) @@ -1180,33 +920,10 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("cloudflareInfraWorkers", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const params = { - orgId: tenant.orgId, - startTime: payload.startTime, - endTime: payload.endTime, - } - const countersCompiled = CH.compile(Integrations.cloudflareWorkerCountersSQL(), params, { - rowSchema: Integrations.cloudflareWorkerCountersRowSchema, - }) - const latencyCompiled = CH.compile(Integrations.cloudflareWorkerLatencySQL(), params, { - rowSchema: Integrations.cloudflareWorkerLatencyRowSchema, - }) const [counterRows, latencyRows] = yield* Effect.all( [ - mapExecError( - warehouse.compiledQuery(tenant, countersCompiled, { - profile: "aggregation", - context: "cloudflareInfraWorkerCounters", - }), - "cloudflareInfraWorkerCounters query failed", - ), - mapExecError( - warehouse.compiledQuery(tenant, latencyCompiled, { - profile: "aggregation", - context: "cloudflareInfraWorkerLatency", - }), - "cloudflareInfraWorkerLatency query failed", - ), + runQuery(Queries.cloudflareInfraWorkerCounters, tenant, payload), + runQuery(Queries.cloudflareInfraWorkerLatency, tenant, payload), ], { concurrency: 2 }, ) @@ -1295,46 +1012,12 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("serviceDbQuerySummary", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const params = { - orgId: tenant.orgId, - dbSystem: payload.dbSystem, - dbNamespace: payload.dbNamespace, - startTime: payload.startTime, - endTime: payload.endTime, - sourceService: payload.sourceService, - deploymentEnv: payload.deploymentEnv, - bucketSeconds: payload.bucketSeconds, - topN: payload.topN, - } - const summaryCompiled = CH.serviceDbQuerySummarySQL(params) - const timeseriesCompiled = CH.serviceDbQueryTimeseriesSQL(params) - const topQueriesCompiled = CH.serviceDbTopQueriesSQL(params) const [summary, timeseriesRows, topQueryRows] = yield* Effect.all( [ - mapExecError( - warehouse - .compiledQueryFirst(tenant, summaryCompiled, { - profile: "aggregation", - context: "serviceDbQuerySummary", - }) - .pipe(Effect.map(Option.getOrNull)), - "serviceDbQuerySummary query failed", - ), - mapExecError( - warehouse.compiledQuery(tenant, timeseriesCompiled, { - profile: "aggregation", - context: "serviceDbQueryTimeseries", - }), - "serviceDbQueryTimeseries query failed", - ), - mapExecError( - warehouse.compiledQuery(tenant, topQueriesCompiled, { - profile: "aggregation", - context: "serviceDbTopQueries", - }), - "serviceDbTopQueries query failed", - ), + runQueryFirst(Queries.serviceDbQuerySummary, tenant, payload), + runQuery(Queries.serviceDbQueryTimeseries, tenant, payload), + runQuery(Queries.serviceDbTopQueries, tenant, payload), ], { concurrency: 3 }, ) @@ -1393,17 +1076,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("servicePlatforms", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const compiled = CH.servicePlatformsSQL( - { deploymentEnv: payload.deploymentEnv }, - { orgId: tenant.orgId, startTime: payload.startTime, endTime: payload.endTime }, - ) - const rows = yield* mapExecError( - warehouse.compiledQuery(tenant, compiled, { - profile: "aggregation", - context: "servicePlatforms", - }), - "servicePlatforms query failed", - ) + const rows = yield* runQuery(Queries.servicePlatforms, tenant, payload) return new ServicePlatformsResponse({ data: rows.map((row) => { const k8sCluster = String(row.k8sCluster ?? "") @@ -2264,34 +1937,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("podFacets", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const compiled = CH.compileUnion( - CH.podFacetsQuery({ - search: payload.search, - podNames: payload.podNames, - namespaces: payload.namespaces, - nodeNames: payload.nodeNames, - clusters: payload.clusters, - deployments: payload.deployments, - statefulsets: payload.statefulsets, - daemonsets: payload.daemonsets, - jobs: payload.jobs, - environments: payload.environments, - computeTypes: payload.computeTypes, - }), - { orgId: tenant.orgId, startTime: payload.startTime, endTime: payload.endTime }, - ) - const rows = yield* mapExecError( - warehouse.compiledQuery(tenant, compiled, { - profile: "discovery", - // 10 UNION branches each re-read the wide ResourceAttributes Map column; - // cap read-thread concurrency so the per-thread decompression buffers stay - // inside the discovery memory budget (bound is ~independent of time range). - settings: { maxThreads: 4 }, - context: "podFacets", - }), - "podFacets query failed", - ) - const typedRows = rows + const rows = yield* runQuery(Queries.podFacets, tenant, payload) const buckets = { pods: [] as Array<{ name: string; count: number }>, namespaces: [] as Array<{ name: string; count: number }>, @@ -2304,7 +1950,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", environments: [] as Array<{ name: string; count: number }>, computeTypes: [] as Array<{ name: string; count: number }>, } - for (const row of typedRows) { + for (const row of rows) { const entry = { name: String(row.name), count: Number(row.count) || 0 } switch (row.facetType) { case "pod": @@ -2345,32 +1991,13 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("nodeFacets", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const compiled = CH.compileUnion( - CH.nodeFacetsQuery({ - search: payload.search, - nodeNames: payload.nodeNames, - clusters: payload.clusters, - environments: payload.environments, - }), - { orgId: tenant.orgId, startTime: payload.startTime, endTime: payload.endTime }, - ) - const rows = yield* mapExecError( - warehouse.compiledQuery(tenant, compiled, { - profile: "discovery", - // See podFacets: cap read-thread concurrency to bound Map-column - // decompression memory across the fan-out of UNION branches. - settings: { maxThreads: 4 }, - context: "nodeFacets", - }), - "nodeFacets query failed", - ) - const typedRows = rows + const rows = yield* runQuery(Queries.nodeFacets, tenant, payload) const buckets = { nodes: [] as Array<{ name: string; count: number }>, clusters: [] as Array<{ name: string; count: number }>, environments: [] as Array<{ name: string; count: number }>, } - for (const row of typedRows) { + for (const row of rows) { const entry = { name: String(row.name), count: Number(row.count) || 0 } switch (row.facetType) { case "node": @@ -2390,29 +2017,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("workloadFacets", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const compiled = CH.compileUnion( - CH.workloadFacetsQuery({ - kind: payload.kind, - search: payload.search, - workloadNames: payload.workloadNames, - namespaces: payload.namespaces, - clusters: payload.clusters, - environments: payload.environments, - computeTypes: payload.computeTypes, - }), - { orgId: tenant.orgId, startTime: payload.startTime, endTime: payload.endTime }, - ) - const rows = yield* mapExecError( - warehouse.compiledQuery(tenant, compiled, { - profile: "discovery", - // See podFacets: cap read-thread concurrency to bound Map-column - // decompression memory across the fan-out of UNION branches. - settings: { maxThreads: 4 }, - context: "workloadFacets", - }), - "workloadFacets query failed", - ) - const typedRows = rows + const rows = yield* runQuery(Queries.workloadFacets, tenant, payload) const buckets = { workloads: [] as Array<{ name: string; count: number }>, namespaces: [] as Array<{ name: string; count: number }>, @@ -2420,7 +2025,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", environments: [] as Array<{ name: string; count: number }>, computeTypes: [] as Array<{ name: string; count: number }>, } - for (const row of typedRows) { + for (const row of rows) { const entry = { name: String(row.name), count: Number(row.count) || 0 } switch (row.facetType) { case "workload": diff --git a/packages/query-engine/src/registry/queries.ts b/packages/query-engine/src/registry/queries.ts index ae46161c3..c1add541a 100644 --- a/packages/query-engine/src/registry/queries.ts +++ b/packages/query-engine/src/registry/queries.ts @@ -1,4 +1,9 @@ import type { + NodeFacetsRequest, + PodFacetsRequest, + WorkloadFacetsRequest, + ServiceDbQuerySummaryRequest, + ServicePlatformsRequest, ServiceDbEdgesRequest, ServiceDependenciesRequest, ServiceWorkloadsRequest, @@ -570,3 +575,124 @@ export const serviceWorkloads = defineQuery({ { orgId, startTime: payload.startTime, endTime: payload.endTime }, ), }) + +export const servicePlatforms = defineQuery({ + id: "servicePlatforms", + profile: "aggregation", + cache: undefined, + compile: (payload: ServicePlatformsRequest, orgId: string) => + CH.servicePlatformsSQL( + { deploymentEnv: payload.deploymentEnv }, + { orgId, startTime: payload.startTime, endTime: payload.endTime }, + ), +}) + +// --- serviceDbQuerySummary's three sub-queries ---------------------------- +// All three take the identical params object, so it is built once per def from +// the payload rather than threaded in from the handler. + +const dbQueryParams = (payload: ServiceDbQuerySummaryRequest, orgId: string) => ({ + orgId, + dbSystem: payload.dbSystem, + dbNamespace: payload.dbNamespace, + startTime: payload.startTime, + endTime: payload.endTime, + sourceService: payload.sourceService, + deploymentEnv: payload.deploymentEnv, + bucketSeconds: payload.bucketSeconds, + topN: payload.topN, +}) + +/** Single-row: read through `runQueryFirst`. */ +export const serviceDbQuerySummary = defineQuery({ + id: "serviceDbQuerySummary", + profile: "aggregation", + cache: undefined, + compile: (payload: ServiceDbQuerySummaryRequest, orgId: string) => + CH.serviceDbQuerySummarySQL(dbQueryParams(payload, orgId)), +}) + +export const serviceDbQueryTimeseries = defineQuery({ + id: "serviceDbQueryTimeseries", + profile: "aggregation", + cache: undefined, + compile: (payload: ServiceDbQuerySummaryRequest, orgId: string) => + CH.serviceDbQueryTimeseriesSQL(dbQueryParams(payload, orgId)), +}) + +export const serviceDbTopQueries = defineQuery({ + id: "serviceDbTopQueries", + profile: "aggregation", + cache: undefined, + compile: (payload: ServiceDbQuerySummaryRequest, orgId: string) => + CH.serviceDbTopQueriesSQL(dbQueryParams(payload, orgId)), +}) + +// --- Facet queries (UNION of per-dimension branches) ---------------------- + +export const podFacets = defineQuery({ + id: "podFacets", + profile: "discovery", + // Cap read-thread concurrency to bound Map-column decompression memory + // across the fan-out of UNION branches. + settings: { maxThreads: 4 }, + cache: undefined, + compile: (payload: PodFacetsRequest, orgId: string) => + CH.compileUnion( + CH.podFacetsQuery({ + search: payload.search, + podNames: payload.podNames, + namespaces: payload.namespaces, + nodeNames: payload.nodeNames, + clusters: payload.clusters, + deployments: payload.deployments, + statefulsets: payload.statefulsets, + daemonsets: payload.daemonsets, + jobs: payload.jobs, + environments: payload.environments, + computeTypes: payload.computeTypes, + }), + { orgId: orgId, startTime: payload.startTime, endTime: payload.endTime }, + ), +}) + +export const nodeFacets = defineQuery({ + id: "nodeFacets", + profile: "discovery", + // Cap read-thread concurrency to bound Map-column decompression memory + // across the fan-out of UNION branches. + settings: { maxThreads: 4 }, + cache: undefined, + compile: (payload: NodeFacetsRequest, orgId: string) => + CH.compileUnion( + CH.nodeFacetsQuery({ + search: payload.search, + nodeNames: payload.nodeNames, + clusters: payload.clusters, + environments: payload.environments, + }), + { orgId: orgId, startTime: payload.startTime, endTime: payload.endTime }, + ), +}) + +export const workloadFacets = defineQuery({ + id: "workloadFacets", + profile: "discovery", + // Cap read-thread concurrency to bound Map-column decompression memory + // across the fan-out of UNION branches. + settings: { maxThreads: 4 }, + cache: undefined, + compile: (payload: WorkloadFacetsRequest, orgId: string) => + CH.compileUnion( + CH.workloadFacetsQuery({ + kind: payload.kind, + search: payload.search, + workloadNames: payload.workloadNames, + namespaces: payload.namespaces, + clusters: payload.clusters, + environments: payload.environments, + computeTypes: payload.computeTypes, + }), + { orgId: orgId, startTime: payload.startTime, endTime: payload.endTime }, + ), +})