diff --git a/apps/api/src/routes/query-runner.ts b/apps/api/src/routes/query-runner.ts index a9c8170ae..73acce31e 100644 --- a/apps/api/src/routes/query-runner.ts +++ b/apps/api/src/routes/query-runner.ts @@ -1,3 +1,4 @@ +import { baselineWarehouseCapabilities } from "@maple/query-engine" import type { QueryDef } from "@maple/query-engine/registry" import type { QueryEngineDirectError } from "@maple/query-engine/runtime" import { Clock, Effect, Option } from "effect" @@ -88,17 +89,32 @@ export const makeQueryRunners = ({ warehouse, queryEngine }: QueryRunnerDeps) => def: QueryDef, tenant: TenantContext, payload: Payload, - ) => - withPolicy( + ) => { + const options = { + profile: def.profile, + ...resolveSettings(def, payload), + context: def.id, + } + return withPolicy( def, tenant, payload, - warehouse.compiledQuery(tenant, def.compile(payload, tenant.orgId), { - profile: def.profile, - ...resolveSettings(def, payload), - context: def.id, - }), + // Capability-aware defs get the backend's real index support, which + // costs a `system.*` probe on BYO ClickHouse; everything else compiles + // against the baseline and so emits exactly the SQL it did before. + def.capabilityAware + ? warehouse.compiledQueryWithCapabilities( + tenant, + (capabilities) => def.compile(payload, tenant.orgId, capabilities), + options, + ) + : warehouse.compiledQuery( + tenant, + def.compile(payload, tenant.orgId, baselineWarehouseCapabilities()), + options, + ), ) + } /** * Run a `QueryDef` that returns at most one row, as `Row | null`. @@ -116,7 +132,7 @@ export const makeQueryRunners = ({ warehouse, queryEngine }: QueryRunnerDeps) => tenant, payload, warehouse - .compiledQueryFirst(tenant, def.compile(payload, tenant.orgId), { + .compiledQueryFirst(tenant, def.compile(payload, tenant.orgId, baselineWarehouseCapabilities()), { profile: def.profile, ...resolveSettings(def, payload), context: def.id, diff --git a/apps/api/src/routes/v1/query-engine.http.ts b/apps/api/src/routes/v1/query-engine.http.ts index edc7d60b5..c53b17278 100644 --- a/apps/api/src/routes/v1/query-engine.http.ts +++ b/apps/api/src/routes/v1/query-engine.http.ts @@ -399,6 +399,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", // concurrently, then merge by ServiceName. Routed through the org's // configured warehouse exactly like the metric explorer reads these // same `cloudflare.*` metrics — no special ingest pin needed. + yield* warehouse.warmRoute(tenant) const [counterRows, latencyRows] = yield* Effect.all( [ runQuery(Queries.cloudflareServiceCounters, tenant, payload), @@ -428,6 +429,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", // concurrently, then merge by database(+branch). Routed through the // org's configured warehouse like the metric explorer reads the same // scraped `planetscale_*` metrics. + yield* warehouse.warmRoute(tenant) const [gaugeRows, connectionRows, storageRows] = yield* Effect.all( [ runQuery(Queries.planetscaleServiceGauges, tenant, payload), @@ -527,6 +529,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", // concurrently, then merge by ServiceName — same shape as // serviceCloudflareStats above. const filters = toCloudflareFilters(payload) + yield* warehouse.warmRoute(tenant) const [counterRows, latencyRows] = yield* Effect.all( [ runQuery(Queries.cloudflareInfraZoneCounters, tenant, payload), @@ -581,6 +584,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const filters = toCloudflareFilters(payload) + yield* warehouse.warmRoute(tenant) const [statusRows, cacheRows, latencyRows] = yield* Effect.all( [ runQuery(Queries.cloudflareInfraZoneDetailStatus, tenant, payload), @@ -607,6 +611,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const filters = toCloudflareFilters(payload) + yield* warehouse.warmRoute(tenant) const [totalRows, bucketRows] = yield* Effect.all( [ runQuery(Queries.cloudflareInfraZoneHostTotals, tenant, payload), @@ -627,6 +632,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const filters = toCloudflareFilters(payload) + yield* warehouse.warmRoute(tenant) const [bucketRows, topRows] = yield* Effect.all( [ runQuery(Queries.cloudflareInfraZoneFirewallTimeseries, tenant, payload), @@ -647,6 +653,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const filters = toCloudflareFilters(payload) + yield* warehouse.warmRoute(tenant) const [bucketRows, nameRows] = yield* Effect.all( [ runQuery(Queries.cloudflareInfraZoneDnsTimeseries, tenant, payload), @@ -667,6 +674,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const filters = toCloudflareFilters(payload) + yield* warehouse.warmRoute(tenant) const [totalRows, coverageRows, zoneRows] = yield* Effect.all( [ runQuery(Queries.cloudflareInfraZoneBreakdownTotals, tenant, payload), @@ -760,6 +768,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("cloudflareInfraPlatformResources", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context + yield* warehouse.warmRoute(tenant) const [queueRows, doRows] = yield* Effect.all( [ runQuery(Queries.cloudflareInfraQueueGauges, tenant, payload), @@ -776,6 +785,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("cloudflareInfraWorkers", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context + yield* warehouse.warmRoute(tenant) const [counterRows, latencyRows] = yield* Effect.all( [ runQuery(Queries.cloudflareInfraWorkerCounters, tenant, payload), @@ -818,6 +828,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", // browser->Worker round-trips. The primary chart keeps its own // execute-path cache; releases is uncached (mirrors the standalone // handler); environments is edge-cached on a service-scoped key. + yield* warehouse.warmRoute(tenant) const [timeseries, releaseRows, environmentRows] = yield* Effect.all( [ queryEngine.execute(tenant, payload.timeseries), @@ -850,6 +861,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", // Dependencies tab in one Worker invocation: the three service-map // edge queries run concurrently and share a single config // resolution, replacing three independent round-trips. + yield* warehouse.warmRoute(tenant) const [dependencyRows, dbEdgeRows, externalEdgeRows] = yield* Effect.all( [ runQuery(Queries.serviceDependenciesForService, tenant, payload), @@ -869,6 +881,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", Effect.gen(function* () { const tenant = yield* CurrentTenant.Context + yield* warehouse.warmRoute(tenant) const [summary, timeseriesRows, topQueryRows] = yield* Effect.all( [ runQueryFirst(Queries.serviceDbQuerySummary, tenant, payload), @@ -1386,6 +1399,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .handle("listPods", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context + yield* warehouse.warmRoute(tenant) const [rows, countRow] = yield* Effect.all( [ runQuery(Queries.listPods, tenant, payload), diff --git a/packages/query-engine/src/registry/queries.ts b/packages/query-engine/src/registry/queries.ts index ce44de898..184d1c682 100644 --- a/packages/query-engine/src/registry/queries.ts +++ b/packages/query-engine/src/registry/queries.ts @@ -34,6 +34,7 @@ import type { WorkloadDetailSummaryRequest, } from "@maple/domain/http" import { Match } from "effect" +import { attributeIndexMode, logBodySearchMode } from "../capabilities" import * as CH from "../ch" import { LOGS_BODY_SEARCH_SETTINGS } from "../profiles" import { makeDirectRouteCachePolicy } from "../runtime/query-engine" @@ -258,14 +259,28 @@ export const serviceDbEdgesForService = defineQuery({ ), }) +/** + * Log search. + * + * `capabilityAware` closes a real gap rather than adding a nicety. The pipe path + * (`list_logs`, which the `maple` CLI uses) has always passed these two modes, + * so CLI log search gets bloom/tokenbf index acceleration while the dashboard's + * HTTP path — compiling without capabilities — silently did full scans of the + * same data. Same builder, same table, different plans. + */ export const listLogs = defineQuery({ id: "listLogs", profile: "list", - settings: (payload) => (payload.search ? LOGS_BODY_SEARCH_SETTINGS : undefined), + // Annotated rather than inferred: with a three-parameter `compile`, TS + // resolves this callback before it can pin `Payload` from `compile`. + settings: (payload: ListLogsRequest) => (payload.search ? LOGS_BODY_SEARCH_SETTINGS : undefined), cache: 15, - compile: (payload: ListLogsRequest, orgId: string) => + capabilityAware: true, + compile: (payload: ListLogsRequest, orgId: string, capabilities) => CH.compile( CH.logsListQuery({ + attributeIndexMode: attributeIndexMode(capabilities, "logs"), + bodySearchMode: logBodySearchMode(capabilities), serviceName: payload.service, severity: payload.severity, minSeverity: payload.minSeverity, diff --git a/packages/query-engine/src/registry/query-def.ts b/packages/query-engine/src/registry/query-def.ts index c6a5ddedf..1e10810d9 100644 --- a/packages/query-engine/src/registry/query-def.ts +++ b/packages/query-engine/src/registry/query-def.ts @@ -1,4 +1,5 @@ import type { CompiledQuery } from "@maple-dev/clickhouse-builder" +import type { WarehouseCapabilities } from "../capabilities" import type { QueryProfileName, WarehouseQuerySettings } from "../profiles/query-profile" import type { DirectRouteCachePolicyInput } from "../runtime/query-engine" @@ -78,6 +79,22 @@ export interface QueryDef { | undefined | ((payload: Payload, nowMs: number) => DirectRouteCachePolicyInput | undefined) + /** + * Resolve the backend's skip-index capabilities and hand them to `compile`. + * + * Off by default because it is not free: on BYO ClickHouse it costs a live + * `system.*` probe (measured p50 262ms), and only queries that can actually + * exploit bloom/tokenbf indices get anything back for it. Managed backends + * answer from a static snapshot, so for them it is free. + * + * Set it on the queries the executor already treats as capability-aware — + * log and trace list/search shapes, whose `attributeIndexMode` and + * `bodySearchMode` decide whether the emitted SQL can use an index at all. + * Declaring it changes the SQL those queries emit, so the SQL baseline moves + * with the change. + */ + readonly capabilityAware?: boolean + /** * Build the compiled query from the request payload. * @@ -85,8 +102,16 @@ export interface QueryDef { * comes from the authenticated tenant, never from user input — every query * must filter `OrgId`, and that guarantee should not depend on a client * sending the right value. + * + * `capabilities` is the backend's index support. It is the BASELINE (all + * indices assumed absent) unless the def sets `capabilityAware`, so a query + * that ignores the argument keeps emitting exactly the SQL it did before. */ - readonly compile: (payload: Payload, orgId: string) => CompiledQuery + readonly compile: ( + payload: Payload, + orgId: string, + capabilities: WarehouseCapabilities, + ) => CompiledQuery } /**