From 4e74953cf4d92c3795c6f4e87697923e1b78f7ce Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 23 Oct 2026 01:13:31 +0200 Subject: [PATCH] feat(ci): materialize CI runs and jobs off github-actions traces ci_runs and ci_jobs keep one row per workflow run and per job with the cicd.* attributes as columns, for a year. The CI analytics read raw traces today and scan every step span's Map columns: a page load on a busy org read 1.3-9.8 GB. Migration 0047 creates both, their views, and backfills the retained traces without doubling rows the views write meanwhile. --- .github/workflows/ci.yml | 1 + apps/cli/src/server/local-schema-history.ts | 13 + apps/cli/src/server/local-schema-version.ts | 2 +- .../server/local-store-migrations/steps.ts | 36 + apps/cli/src/server/schema-identity.ts | 3 + apps/cli/src/server/schema/local-inserts.json | 2 +- .../src/server/schema/local-schema-v38.sql | 2346 +++++++++++++++++ apps/cli/src/server/schema/local-schema.sql | 106 +- apps/cli/src/server/scoped-delete.ts | 2 + apps/cli/test/local-store-migrations.test.ts | 29 +- apps/cli/test/native-local-store-migration.sh | 2 +- apps/ingest/src/clickhouse_insert_mappings.rs | 2 +- docs/self-hosted-clickhouse.md | 12 +- ...uns-materialization.clickhouse.e2e.test.ts | 259 ++ .../services/warehouse/warehouse-catalog.ts | 10 + .../migrations/0047_ci_runs_jobs.ts | 100 + .../src/clickhouse/migrations/index.test.ts | 9 +- .../domain/src/clickhouse/migrations/index.ts | 2 + .../domain/src/generated/clickhouse-schema.ts | 6 +- .../generated/tinybird-project-manifest.ts | 22 +- packages/domain/src/tinybird/datasources.ts | 79 + .../domain/src/tinybird/materializations.ts | 80 + .../materialized-projection-order.test.ts | 2 + .../src/tinybird/retention-matrix.test.ts | 3 + packages/query-engine/src/ch/tables.ts | 6 + scripts/purge-org-traces.ts | 2 + 26 files changed, 3111 insertions(+), 25 deletions(-) create mode 100644 apps/cli/src/server/schema/local-schema-v38.sql create mode 100644 packages/backend/src/services/warehouse/ci-runs-materialization.clickhouse.e2e.test.ts create mode 100644 packages/domain/src/clickhouse/migrations/0047_ci_runs_jobs.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 259e6f9427..15bc4097c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -395,6 +395,7 @@ jobs: src/services/warehouse/alert-window-alignment.clickhouse.e2e.test.ts src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts src/services/warehouse/trace-facets-hourly-materialization.clickhouse.e2e.test.ts + src/services/warehouse/ci-runs-materialization.clickhouse.e2e.test.ts # Every SQL shape the product emits, against the real analyzer: unit tests # only compare SQL text, which is how a NO_COMMON_TYPE reached production. diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index c119ab8655..1ba310aa25 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -472,6 +472,19 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "9f47ab6ca94633c667375133a2a2bc8fab1065a0494fb89973ba2230d8f9d516", projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", }), + Object.freeze({ + // v38 adds ci_runs, ci_jobs and their views; nothing is rewritten, and nothing is + // backfilled because CI traces never reach a local store. + // + // projectRevision is carried forward deliberately: it is a hardcoded + // constant that no longer tracks the generator's header, and the identity + // this gate compares is the fingerprint/digest pair. + version: 38, + fingerprint: "b5b7d31165206b3b", + digest: "b5b7d31165206b3bb5b9d7067625afb518b78767d066db56903641628900a5a8", + manifestDigest: "ff3610deae33950cec09239468ee3752c2869a8232f3d1e1c252a61ca29992db", + projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", + }), ] as const) /** Immutable SQLite control DDL identities, checked by clickhouse:schema:check. */ diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index a228824fd8..c0ad7f9e8d 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,7 +1,7 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 37 as const +export const LOCAL_SCHEMA_VERSION = 38 as const /** SQLite eventing state has its own independent version sequence. */ export const LOCAL_CONTROL_SCHEMA_VERSION = 1 as const diff --git a/apps/cli/src/server/local-store-migrations/steps.ts b/apps/cli/src/server/local-store-migrations/steps.ts index 68dd130adf..647cbb6bc5 100644 --- a/apps/cli/src/server/local-store-migrations/steps.ts +++ b/apps/cli/src/server/local-store-migrations/steps.ts @@ -1452,5 +1452,41 @@ export const LOCAL_STORE_STEPS: ReadonlyArray = [ }, ], }, + { + // Purely additive and filled forward only: CI traces come from the GitHub App's + // webhooks, which never reach a local store, so there is nothing to backfill. + id: "local-0037-to-0038-ci-runs-jobs", + from: 37, + to: 38, + description: "ci_runs + ci_jobs and their views: CI runs and jobs off github-actions traces", + clonedBefore: "additive DDL runs", + plan: [ + [ + "install-ci-runs-jobs", + "Install the ci_runs and ci_jobs projections and their materialized views via the v38 bootstrap", + ], + ], + verifies: "Verify the v38 physical schema and the retained raw telemetry counts", + dispositions: [ + { + name: "ci_runs", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: "Filled forward from github-actions root spans written after the migration.", + preservationInterval: "from the migration forward", + sourceRetentionDays: 30, + targetRetentionDays: 365, + }, + { + name: "ci_jobs", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: "Filled forward from github-actions job spans written after the migration.", + preservationInterval: "from the migration forward", + sourceRetentionDays: 30, + targetRetentionDays: 365, + }, + ], + }, // local-schema:bump appends the next step above this line. ] diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index 7e5ecd49d3..6eab4576dd 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -36,6 +36,7 @@ import schemaV34Sql from "./schema/local-schema-v34.sql" with { type: "text" } import schemaV35Sql from "./schema/local-schema-v35.sql" with { type: "text" } import schemaV36Sql from "./schema/local-schema-v36.sql" with { type: "text" } import schemaV37Sql from "./schema/local-schema-v37.sql" with { type: "text" } +import schemaV38Sql from "./schema/local-schema-v38.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -117,6 +118,7 @@ const SNAPSHOT_SQL: ReadonlyArray = [ schemaV35Sql, schemaV36Sql, schemaV37Sql, + schemaV38Sql, ] export interface LocalSchemaSnapshot { @@ -210,6 +212,7 @@ export const LOCAL_SCHEMA_V33 = localSchemaIdentity(33) export const LOCAL_SCHEMA_V34 = localSchemaIdentity(34) export const LOCAL_SCHEMA_V36 = localSchemaIdentity(36) export const LOCAL_SCHEMA_V37 = localSchemaIdentity(37) +export const LOCAL_SCHEMA_V38 = localSchemaIdentity(38) export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 9be0afd7f5..530cfbb922 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "7fed5d73a33c0c907e0634ed216631b41ecd24f213e58dcc5b9df4b9c1f314c5", + "projectRevision": "254be0396778febcafa4a5b5f6dfff07c0af73b2078c528fcf2bfa98cfa7a884", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v38.sql b/apps/cli/src/server/schema/local-schema-v38.sql new file mode 100644 index 0000000000..0ebd450177 --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v38.sql @@ -0,0 +1,2346 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 254be0396778febcafa4a5b5f6dfff07c0af73b2078c528fcf2bfa98cfa7a884 +-- localSchemaVersion: 37 + +CREATE TABLE IF NOT EXISTS ai_crawler_requests ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + ServiceName LowCardinality(String), + Crawler LowCardinality(String), + Host LowCardinality(String), + Path String, + HttpStatus UInt16 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS ai_trace_index ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SessionId String, + VendorId LowCardinality(String), + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + Model LowCardinality(String), + AgentName LowCardinality(String), + ToolName LowCardinality(String), + SpanId String, + ParentSpanId String, + Duration UInt64, + IsError UInt8, + IsLlmCall UInt8, + IsToolCall UInt8, + Tokens Float64, + Cost Float64, + ResponseId String, + VendorVersion LowCardinality(String), + InputTokens Float64, + CacheReadTokens Float64, + CacheWriteTokens Float64, + OutputTokens Float64, + ReasoningTokens Float64, + ErrorType LowCardinality(String), + StatusMessage String, + ToolDescription String, + FailedToolCallResult String, + ErrorFingerprint UInt64 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String), + SkipReason LowCardinality(String) DEFAULT '', + Level LowCardinality(String) DEFAULT '' +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS audit_log ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + RecordedAt DateTime64(3), + ActorType LowCardinality(String), + UserId String, + ApiKeyId String, + ActorId String, + ActorLabel String, + AffectedUserId String, + Source LowCardinality(String), + Action LowCardinality(String), + Outcome LowCardinality(String), + DenialReason String, + ResourceType LowCardinality(String), + ResourceId String, + ChangedFields Array(String), + Changes String, + Metadata String, + RequestId String, + OriginIp String, + OriginCountry LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 2190 DAY; + +CREATE TABLE IF NOT EXISTS ci_jobs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + Owner LowCardinality(String), + Repository LowCardinality(String), + Pipeline LowCardinality(String), + RunId String, + Attempt UInt16, + Job LowCardinality(String), + Result LowCardinality(String), + StatusCode LowCardinality(String), + Duration UInt64, + QueueMs UInt32, + Url String, + FailureReason String, + RunnerName LowCardinality(String), + RunnerLabels LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS ci_runs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + Owner LowCardinality(String), + Repository LowCardinality(String), + Pipeline LowCardinality(String), + RunId String, + RunNumber UInt64, + Attempt UInt16, + Result LowCardinality(String), + Duration UInt64, + Url String, + Branch String, + Revision String, + TriggerEvent LowCardinality(String), + Actor LowCardinality(String), + PullRequest UInt32, + WorkflowPath LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL toDate(Timestamp) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL toDate(Timestamp) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS identity_links ( + OrgId LowCardinality(String), + VisitorId String, + UserId String, + FirstSeen SimpleAggregateFunction(min, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY tuple() +ORDER BY (OrgId, VisitorId, UserId) +TTL toDate(FirstSeen) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS product_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + Source LowCardinality(String) DEFAULT 'browser', + SessionId String DEFAULT '', + Seq UInt32 DEFAULT 0, + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String) DEFAULT '', + PagePath String DEFAULT '', + Url String DEFAULT '', + ServiceName LowCardinality(String) DEFAULT '', + Attributes Map(String, String) DEFAULT map(), + TraceId String DEFAULT '', + SpanId String DEFAULT '', + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4, + INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + SpanKind LowCardinality(String) DEFAULT '', + IsRoot UInt8 DEFAULT 0, + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS source_usage_hourly ( + OrgId LowCardinality(String), + SourceId String, + Hour DateTime, + SpanCount UInt64, + SpanSizeBytes UInt64, + LogCount UInt64, + LogSizeBytes UInt64, + MetricCount UInt64, + MetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, SourceId, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_facets_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName String, + HttpMethod LowCardinality(String), + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + HasError UInt8, + TraceCount SimpleAggregateFunction(sum, UInt64), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, HttpMethod, HttpStatusCode, DeploymentEnv, ServiceNamespace, HasError) +TTL toDate(Hour) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_index ( + OrgId LowCardinality(String), + TraceId String, + ServiceName LowCardinality(String), + Bucket DateTime, + MinTimestamp SimpleAggregateFunction(min, DateTime64(9)), + MaxTimestamp SimpleAggregateFunction(max, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Bucket) +ORDER BY (OrgId, TraceId, ServiceName, Bucket) +TTL toDate(Bucket) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_name SpanName TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, toStartOfFiveMinutes(Timestamp), TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ai_crawler_requests_mv TO ai_crawler_requests AS +SELECT + OrgId, + Timestamp, + TraceId, + ServiceName, + arrayElement(['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'Meta-ExternalAgent', 'Meta-WebIndexer', 'Meta-ExternalFetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai'], multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'meta-externalagent', 'meta-webindexer', 'meta-externalfetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai'])) AS Crawler, + lower(replaceRegexpOne(coalesce(nullIf(SpanAttributes['server.address'], ''), nullIf(SpanAttributes['http.host'], ''), nullIf(SpanAttributes['net.host.name'], ''), nullIf(domain(SpanAttributes['url.full']), ''), nullIf(domain(SpanAttributes['http.url']), ''), ''), ':[0-9]+$', '')) AS Host, + leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) AS Path, + toUInt16OrZero(coalesce(nullIf(SpanAttributes['http.response.status_code'], ''), nullIf(SpanAttributes['http.status_code'], ''), '')) AS HttpStatus + FROM traces + WHERE SpanKind = 'Server' + AND multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'meta-externalagent', 'meta-webindexer', 'meta-externalfetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai']) > 0 + AND leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ai_trace_index_mv TO ai_trace_index AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanAttributes['maple_ai.session.id'] AS SessionId, + SpanAttributes['maple_ai.vendor.id'] AS VendorId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + SpanAttributes['maple_ai.model'] AS Model, + SpanAttributes['maple_ai.agent.name'] AS AgentName, + SpanAttributes['maple_ai.tool.name'] AS ToolName, + SpanId, + ParentSpanId, + Duration, + toUInt8(SpanAttributes['maple_ai.error'] = '1') AS IsError, + toUInt8(SpanAttributes['maple_ai.llm_call'] = '1') AS IsLlmCall, + toUInt8(SpanAttributes['maple_ai.tool_call'] = '1') AS IsToolCall, + toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS Tokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.cost']) AS Cost, + SpanAttributes['maple_ai.response.id'] AS ResponseId, + SpanAttributes['maple_ai.vendor.version'] AS VendorVersion, + toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) AS InputTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) AS CacheReadTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) AS CacheWriteTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) AS OutputTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS ReasoningTokens, + SpanAttributes['error.type'] AS ErrorType, + leftUTF8(StatusMessage, 400) AS StatusMessage, + SpanAttributes['maple_ai.tool.description'] AS ToolDescription, + SpanAttributes['maple_ai.tool.error_result'] AS FailedToolCallResult, + if(SpanAttributes['maple_ai.error'] = '1', cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(leftUTF8(coalesce(nullIf(SpanAttributes['maple_ai.tool.error_result'], ''), leftUTF8(StatusMessage, 400)), 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#')), 0) AS ErrorFingerprint + FROM traces + WHERE SpanAttributes['maple_ai.vendor.id'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ci_jobs_mv TO ci_jobs AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ResourceAttributes['vcs.owner.name'] AS Owner, + ResourceAttributes['vcs.repository.name'] AS Repository, + SpanAttributes['cicd.pipeline.name'] AS Pipeline, + SpanAttributes['cicd.pipeline.run.id'] AS RunId, + toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt, + SpanAttributes['cicd.pipeline.task.name'] AS Job, + SpanAttributes['cicd.pipeline.task.run.result'] AS Result, + StatusCode, + Duration, + toUInt32OrZero(SpanAttributes['maple.cicd.queue_duration_ms']) AS QueueMs, + SpanAttributes['cicd.pipeline.task.run.url.full'] AS Url, + SpanAttributes['maple.cicd.failure.reason'] AS FailureReason, + SpanAttributes['cicd.worker.name'] AS RunnerName, + SpanAttributes['maple.cicd.runner_labels'] AS RunnerLabels + FROM traces + WHERE ServiceName = 'github-actions' + AND ParentSpanId != '' + AND SpanAttributes['cicd.pipeline.task.name'] != '' + AND SpanAttributes['maple.cicd.step.number'] = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ci_runs_mv TO ci_runs AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ResourceAttributes['vcs.owner.name'] AS Owner, + ResourceAttributes['vcs.repository.name'] AS Repository, + SpanAttributes['cicd.pipeline.name'] AS Pipeline, + SpanAttributes['cicd.pipeline.run.id'] AS RunId, + toUInt64OrZero(SpanAttributes['maple.cicd.run_number']) AS RunNumber, + toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt, + SpanAttributes['cicd.pipeline.result'] AS Result, + Duration, + SpanAttributes['cicd.pipeline.run.url.full'] AS Url, + SpanAttributes['vcs.ref.head.name'] AS Branch, + SpanAttributes['vcs.ref.head.revision'] AS Revision, + SpanAttributes['maple.cicd.trigger_event'] AS TriggerEvent, + SpanAttributes['maple.cicd.actor'] AS Actor, + toUInt32OrZero(SpanAttributes['vcs.change.id']) AS PullRequest, + SpanAttributes['maple.cicd.workflow_path'] AS WorkflowPath + FROM traces + WHERE ServiceName = 'github-actions' + AND ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, + -- JSON detection for the message signature below. + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(_msgText) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\x1b\\[[0-9;]*m', ''), '^((\\[Error\\]|[\\s\\S]*?DB::Exception:)[\\s\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\s\\S]*', '\\1'), '(^|[ \\t\\r\\n\'"(`=])~/[^ \'"`:,)]*', '\\1PATH'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\b', 'ID'), '\\b[0-9A-HJKMNP-TV-Z]{26}\\b', 'ID'), '\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\b', '\\1\\2§\\3'), '\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\b', '\\1\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\1'), + 1, 120 + ) + ) AS _msgSig, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+') + AND position(line, '(definition)') = 0 + AND NOT match(line, '^[ \\t]*at \\[') + AND NOT match(line, '^[ \\t]*at ([^(:]|:[^0-9])*:?$'), + splitByChar('\n', _exStack) + ), + -- A namespaced type plus its message names one failure: keep only the + -- throw site. With no message the frames are all that is left; keep 3. + 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3) + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*', '@#'), '\\(index [0-9]+\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\.css', '.css'), '(/_astro/[^/ ).]+)\\.[A-Za-z0-9_-]{8}\\.js', '\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, + if( + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + left(_msgText, multiIf( + position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1, + position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1, + position(_msgText, '\n') > 3, toInt64(position(_msgText, '\n')) - 1, + least(toInt64(length(_msgText)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, + -- JSON detection for the message signature below. + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(_msgText) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\x1b\\[[0-9;]*m', ''), '^((\\[Error\\]|[\\s\\S]*?DB::Exception:)[\\s\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\s\\S]*', '\\1'), '(^|[ \\t\\r\\n\'"(`=])~/[^ \'"`:,)]*', '\\1PATH'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\b', 'ID'), '\\b[0-9A-HJKMNP-TV-Z]{26}\\b', 'ID'), '\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\b', '\\1\\2§\\3'), '\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\b', '\\1\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\1'), + 1, 120 + ) + ) AS _msgSig, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+') + AND position(line, '(definition)') = 0 + AND NOT match(line, '^[ \\t]*at \\[') + AND NOT match(line, '^[ \\t]*at ([^(:]|:[^0-9])*:?$'), + splitByChar('\n', _exStack) + ), + -- A namespaced type plus its message names one failure: keep only the + -- throw site. With no message the frames are all that is left; keep 3. + 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3) + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*', '@#'), '\\(index [0-9]+\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\.css', '.css'), '(/_astro/[^/ ).]+)\\.[A-Za-z0-9_-]{8}\\.js', '\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, + if( + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + left(_msgText, multiIf( + position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1, + position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1, + position(_msgText, '\n') > 3, toInt64(position(_msgText, '\n')) - 1, + least(toInt64(length(_msgText)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS +SELECT + OrgId, + VisitorId, + UserId, + StartTime AS FirstSeen + FROM session_replays + WHERE VisitorId != '' AND UserId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes, + '' AS TraceId, + '' AS SpanId + FROM session_events + WHERE Type IN ('navigation', 'custom'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId + FROM traces + WHERE SpanAttributes['maple.product_event.name'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', + if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles, + sum(ClassifiedSpanCount) AS ClassifiedSpanCount, + sum(ServerSpanCount) AS ServerSpanCount, + sum(RoutedSpanCount) AS RoutedSpanCount + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles, + count() AS ClassifiedSpanCount, + countIf(SpanKind IN ('Server', 'Consumer')) AS ServerSpanCount, + countIf(SpanAttributes['http.route'] != '') AS RoutedSpanCount + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + SpanKind, + toUInt8(ParentSpanId = '') AS IsRoot + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_logs_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(TimestampTime) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS MetricCount, + 0 AS MetricSizeBytes + FROM logs + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_exp_histogram_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS MetricCount, + count() * 300 AS MetricSizeBytes + FROM metrics_exponential_histogram + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_gauge_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS MetricCount, + count() * 150 AS MetricSizeBytes + FROM metrics_gauge + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_histogram_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS MetricCount, + count() * 250 AS MetricSizeBytes + FROM metrics_histogram + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_sum_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS MetricCount, + count() * 150 AS MetricSizeBytes + FROM metrics_sum + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_traces_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + count() AS SpanCount, + sum(length(SpanName) + 300) AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS MetricCount, + 0 AS MetricSizeBytes + FROM traces + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_facets_hourly_mv TO trace_facets_hourly AS +SELECT + OrgId, + toStartOfHour(Timestamp) AS Hour, + ServiceName, + SpanName, + HttpMethod, + HttpStatusCode, + DeploymentEnv, + ServiceNamespace, + HasError, + count() AS TraceCount, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM trace_list_mv + GROUP BY OrgId, Hour, ServiceName, SpanName, HttpMethod, HttpStatusCode, DeploymentEnv, ServiceNamespace, HasError; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_index_mv TO trace_index AS +SELECT + OrgId, + TraceId, + ServiceName, + toStartOfFiveMinutes(Timestamp) AS Bucket, + min(Timestamp) AS MinTimestamp, + max(Timestamp) AS MaxTimestamp + FROM traces + GROUP BY OrgId, TraceId, ServiceName, Bucket; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index e047e6e81a..e4afe2fe48 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 7fed5d73a33c0c907e0634ed216631b41ecd24f213e58dcc5b9df4b9c1f314c5 --- localSchemaVersion: 37 +-- projectRevision: 254be0396778febcafa4a5b5f6dfff07c0af73b2078c528fcf2bfa98cfa7a884 +-- localSchemaVersion: 38 CREATE TABLE IF NOT EXISTS ai_crawler_requests ( OrgId LowCardinality(String), @@ -138,6 +138,57 @@ PARTITION BY toYYYYMM(OccurredAt) ORDER BY (OrgId, OccurredAt, Id) TTL toDate(OccurredAt) + INTERVAL 2190 DAY; +CREATE TABLE IF NOT EXISTS ci_jobs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + Owner LowCardinality(String), + Repository LowCardinality(String), + Pipeline LowCardinality(String), + RunId String, + Attempt UInt16, + Job LowCardinality(String), + Result LowCardinality(String), + StatusCode LowCardinality(String), + Duration UInt64, + QueueMs UInt32, + Url String, + FailureReason String, + RunnerName LowCardinality(String), + RunnerLabels LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS ci_runs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + Owner LowCardinality(String), + Repository LowCardinality(String), + Pipeline LowCardinality(String), + RunId String, + RunNumber UInt64, + Attempt UInt16, + Result LowCardinality(String), + Duration UInt64, + Url String, + Branch String, + Revision String, + TriggerEvent LowCardinality(String), + Actor LowCardinality(String), + PullRequest UInt32, + WorkflowPath LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + CREATE TABLE IF NOT EXISTS error_events ( OrgId LowCardinality(String), Timestamp DateTime, @@ -1030,6 +1081,57 @@ SELECT FROM traces WHERE SpanAttributes['maple_ai.vendor.id'] != ''; +CREATE MATERIALIZED VIEW IF NOT EXISTS ci_jobs_mv TO ci_jobs AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ResourceAttributes['vcs.owner.name'] AS Owner, + ResourceAttributes['vcs.repository.name'] AS Repository, + SpanAttributes['cicd.pipeline.name'] AS Pipeline, + SpanAttributes['cicd.pipeline.run.id'] AS RunId, + toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt, + SpanAttributes['cicd.pipeline.task.name'] AS Job, + SpanAttributes['cicd.pipeline.task.run.result'] AS Result, + StatusCode, + Duration, + toUInt32OrZero(SpanAttributes['maple.cicd.queue_duration_ms']) AS QueueMs, + SpanAttributes['cicd.pipeline.task.run.url.full'] AS Url, + SpanAttributes['maple.cicd.failure.reason'] AS FailureReason, + SpanAttributes['cicd.worker.name'] AS RunnerName, + SpanAttributes['maple.cicd.runner_labels'] AS RunnerLabels + FROM traces + WHERE ServiceName = 'github-actions' + AND ParentSpanId != '' + AND SpanAttributes['cicd.pipeline.task.name'] != '' + AND SpanAttributes['maple.cicd.step.number'] = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ci_runs_mv TO ci_runs AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ResourceAttributes['vcs.owner.name'] AS Owner, + ResourceAttributes['vcs.repository.name'] AS Repository, + SpanAttributes['cicd.pipeline.name'] AS Pipeline, + SpanAttributes['cicd.pipeline.run.id'] AS RunId, + toUInt64OrZero(SpanAttributes['maple.cicd.run_number']) AS RunNumber, + toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt, + SpanAttributes['cicd.pipeline.result'] AS Result, + Duration, + SpanAttributes['cicd.pipeline.run.url.full'] AS Url, + SpanAttributes['vcs.ref.head.name'] AS Branch, + SpanAttributes['vcs.ref.head.revision'] AS Revision, + SpanAttributes['maple.cicd.trigger_event'] AS TriggerEvent, + SpanAttributes['maple.cicd.actor'] AS Actor, + toUInt32OrZero(SpanAttributes['vcs.change.id']) AS PullRequest, + SpanAttributes['maple.cicd.workflow_path'] AS WorkflowPath + FROM traces + WHERE ServiceName = 'github-actions' + AND ParentSpanId = ''; + CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS WITH arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, diff --git a/apps/cli/src/server/scoped-delete.ts b/apps/cli/src/server/scoped-delete.ts index 6861f9a27f..aff723a70d 100644 --- a/apps/cli/src/server/scoped-delete.ts +++ b/apps/cli/src/server/scoped-delete.ts @@ -103,6 +103,8 @@ export const TABLE_DELETE_PLAN = { attribute_keys_hourly: unkeyed("Hour"), attribute_values_hourly: unkeyed("Hour"), audit_log: { strategy: "excluded", reason: NOT_TELEMETRY }, + ci_jobs: unkeyed("Timestamp"), + ci_runs: unkeyed("Timestamp"), error_events: derived("Timestamp"), error_events_by_time: derived("Timestamp"), error_fingerprints_minutely: unkeyed("Minute"), diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index d60013c0cd..2340519689 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -39,7 +39,7 @@ import { LOCAL_SCHEMA_V33, LOCAL_SCHEMA_V34, LOCAL_SCHEMA_V36, - LOCAL_SCHEMA_V37, + LOCAL_SCHEMA_V38, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -105,11 +105,11 @@ const [v10ToV11ProductEventsModule] = localStoreMigrations.filter( ) describe("current local schema identity", () => { - it("matches the generated v37 revision and keeps the issue-297 identity frozen", () => { + it("matches the generated v38 revision and keeps the issue-297 identity frozen", () => { // oxlint-disable-next-line test-slop/no-constant-pin -- rewritten by local-schema:bump; catches an unbumped schema edit - expect(SCHEMA_FINGERPRINT).toBe("1772296e571539fd") + expect(SCHEMA_FINGERPRINT).toBe("b5b7d31165206b3b") // oxlint-disable-next-line test-slop/no-constant-pin -- rewritten by local-schema:bump; catches an unbumped schema edit - expect(SCHEMA_DIGEST).toBe("1772296e571539fdff7d2289c58910c1a8a496967f23613d5b2e81935541a786") + expect(SCHEMA_DIGEST).toBe("b5b7d31165206b3bb5b9d7067625afb518b78767d066db56903641628900a5a8") // oxlint-disable-next-line test-slop/no-constant-pin -- frozen identity that stores already on disk carry expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", @@ -117,8 +117,8 @@ describe("current local schema identity", () => { expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) // oxlint-disable-next-line test-slop/no-constant-pin -- anchor that local-schema:bump rewrites on every bump - expect(CURRENT_LOCAL_SCHEMA.version).toBe(37) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V37) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(38) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V38) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -188,7 +188,7 @@ describe("current local schema identity", () => { // its view; v11 replaces `web_events` with `product_events` and adds // `identity_links`; v17 adds `audit_log`, which local mode creates but // never writes; v23 adds `ai_crawler_requests` and its view; v24 adds - // `trace_facets_hourly` and its view; v33 adds `trace_index` and its view; v35 removes `trace_detail_spans` and its view; v37 adds `source_usage_hourly` and its six views. Asserted as an exact set difference rather than a + // `trace_facets_hourly` and its view; v33 adds `trace_index` and its view; v35 removes `trace_detail_spans` and its view; v37 adds `source_usage_hourly` and its six views; v38 adds `ci_runs`, `ci_jobs` and their views. Asserted as an exact set difference rather than a // relaxed check, so a future edge still cannot add or drop an object // unnoticed. expect([...v5Names].filter((name) => !currentNames.has(name))).toEqual([ @@ -205,6 +205,10 @@ describe("current local schema identity", () => { "ai_trace_index", "ai_trace_index_mv", "audit_log", + "ci_jobs", + "ci_jobs_mv", + "ci_runs", + "ci_runs_mv", "identity_links", "identity_links_mv", "product_events", @@ -310,7 +314,8 @@ describe("current local schema identity", () => { // v12 replaces two view bodies and v13 adds columns to two rollups; neither // adds an object. v14 is exactly the GenAI span index and its view, created // empty and filled forward; v17 adds `audit_log`, created empty and never - // written in local mode; v35 drops `trace_detail_spans` and its view. + // written in local mode; v35 drops `trace_detail_spans` and its view; v38 adds + // `ci_runs`, `ci_jobs` and their views, filled forward. const v12Names = new Set(LOCAL_SCHEMA_V12_MANIFEST.objects.map((object) => object.name)) const v13Names = new Set(LOCAL_SCHEMA_V13_MANIFEST.objects.map((object) => object.name)) const currentSchemaNames = new Set(LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name)) @@ -323,6 +328,10 @@ describe("current local schema identity", () => { "ai_trace_index", "ai_trace_index_mv", "audit_log", + "ci_jobs", + "ci_jobs_mv", + "ci_runs", + "ci_runs_mv", "product_events_traces_mv", "source_usage_hourly", "source_usage_logs_mv", @@ -413,7 +422,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 38, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 39, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) @@ -570,6 +579,7 @@ describe("local migration registry", () => { "local-0034-to-0035-drop-trace-detail-spans", "local-0035-to-0036-aligned-ttls", "local-0036-to-0037-source-usage-hourly", + "local-0037-to-0038-ci-runs-jobs", ] const versions = (id: string) => { const match = /^local-(\d{4})-to-(\d{4})-[a-z0-9]+(-[a-z0-9]+)*$/.exec(id) @@ -1559,6 +1569,7 @@ describe("v10 -> v11 product events module", () => { "local-0034-to-0035-drop-trace-detail-spans", "local-0035-to-0036-aligned-ttls", "local-0036-to-0037-source-usage-hourly", + "local-0037-to-0038-ci-runs-jobs", ]) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V11) // The dropped table is declared, and the backfilled ones say what they diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index b4cc5cc76e..a6e710ff57 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -142,7 +142,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching # LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. -jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 37 and .schema == "1772296e571539fd"' \ +jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 38 and .schema == "b5b7d31165206b3b"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" step "reopening promoted store in a fresh server" diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 29383c92cb..3e3c2d6aa7 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "7fed5d73a33c0c907e0634ed216631b41ecd24f213e58dcc5b9df4b9c1f314c5"; +pub const PROJECT_REVISION: &str = "254be0396778febcafa4a5b5f6dfff07c0af73b2078c528fcf2bfa98cfa7a884"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/docs/self-hosted-clickhouse.md b/docs/self-hosted-clickhouse.md index 45e5dda2bd..1f69520d4c 100644 --- a/docs/self-hosted-clickhouse.md +++ b/docs/self-hosted-clickhouse.md @@ -98,8 +98,8 @@ and `bun run tinybird:manifest` after editing either, or CI's drift gate fails. `metrics_exponential_histogram`, `metrics_gauge`, `metrics_histogram`, `metrics_sum`, `service_address_resolutions_hourly`, `service_map_edges_hourly_ingest` (a `Null`-engine bridge), `session_events`, `session_replay_events`, `session_replays`, `traces` -- **MV-populated tables** (30): `ai_crawler_requests`, `ai_trace_index`, `attribute_keys_hourly`, - `attribute_values_hourly`, `error_events`, `error_events_by_time`, +- **MV-populated tables** (32): `ai_crawler_requests`, `ai_trace_index`, `attribute_keys_hourly`, + `attribute_values_hourly`, `ci_jobs`, `ci_runs`, `error_events`, `error_events_by_time`, `error_fingerprints_minutely`, `identity_links`, `logs_aggregates_hourly`, `metric_catalog`, `product_events` (also written directly by `POST /v1/events`), `service_external_edges_hourly`, `service_map_children`, `service_map_db_edges_hourly`, @@ -109,7 +109,7 @@ and `bun run tinybird:manifest` after editing either, or CI's drift gate fails. `service_usage`, `source_usage_hourly`, `span_metrics_calls_hourly`, `trace_facets_hourly`, `trace_index`, `trace_list_mv`, `traces_aggregates_hourly` -- **Materialized views** (50): fan out from the source tables to populate the +- **Materialized views** (52): fan out from the source tables to populate the MV-populated tables. Several targets are fed by more than one MV: `service_usage` and `source_usage_hourly` by six each, `attribute_values_hourly` / `attribute_keys_hourly` / `metric_catalog` by four each, and `product_events` by two. @@ -117,13 +117,13 @@ and `bun run tinybird:manifest` after editing either, or CI's drift gate fails. See [`warehouse-rollups.md`](warehouse-rollups.md) for when a materialized view is the right answer and which tier a query should read. -Every stored table carries a TTL, tiered by how raw the data is. All but `identity_links`, `service_usage` and `source_usage_hourly` are partitioned by date (most by day, `service_overview_hourly`, `service_operations_hourly`, `error_fingerprints_minutely` and `audit_log` by month), and migration 0045 sets `ttl_only_drop_parts = 1` on those, so eviction drops whole parts instead of rewriting them: +Every stored table carries a TTL, tiered by how raw the data is. All but `identity_links`, `service_usage` and `source_usage_hourly` are partitioned by date (most by day, `service_overview_hourly`, `service_operations_hourly`, `error_fingerprints_minutely`, `ci_runs`, `ci_jobs` and `audit_log` by month), and migrations 0045 and 0047 (`ci_*`) set `ttl_only_drop_parts = 1` on those, so eviction drops whole parts instead of rewriting them: | Retention | Tables | | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | **30 days** | `traces`, `trace_index`, `logs`, `service_map_spans`, `service_map_children`, `service_overview_spans`, `trace_list_mv`, `ai_trace_index`, `session_events`, `session_replay_events`, `session_replays` | -| **90 days** | `error_events`, `error_events_by_time`, `error_fingerprints_minutely`, `metrics_*`, `attribute_*_hourly`, `metric_catalog`, `*_minutely` rollups, `span_metrics_calls_hourly`, `source_usage_hourly` | -| **365 days** | the other hourly rollups (`*_hourly`), `service_usage`, `alert_checks`, `product_events`, `identity_links` | +| **90 days** | `error_events`, `error_events_by_time`, `error_fingerprints_minutely`, `metrics_*`, `attribute_*_hourly`, `metric_catalog`, `*_minutely` rollups, `span_metrics_calls_hourly`, `source_usage_hourly` | +| **365 days** | the other hourly rollups (`*_hourly`), `service_usage`, `alert_checks`, `product_events`, `identity_links`, `ci_*` | | **2190 days** | `audit_log` | Adjust by writing a follow-up migration if your retention requirements differ. diff --git a/packages/backend/src/services/warehouse/ci-runs-materialization.clickhouse.e2e.test.ts b/packages/backend/src/services/warehouse/ci-runs-materialization.clickhouse.e2e.test.ts new file mode 100644 index 0000000000..ff18c49fc7 --- /dev/null +++ b/packages/backend/src/services/warehouse/ci-runs-materialization.clickhouse.e2e.test.ts @@ -0,0 +1,259 @@ +// SAFETY-FILE: JSON in this test is emitted by ClickHouse before its fields are asserted. +// `ci_runs` and `ci_jobs` are filtered projections of `github-actions` traces, and a projection +// that silently drops or doubles rows reads as plausible CI numbers. So this proves rows: spans +// shaped like CiTraceService's, inserted into `traces` on a database built from the real +// migration chain, must land in the projections exactly, and migration 0047's backfill must +// rebuild them without doubling the rows the views already wrote. + +import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" +import { migrations, renderStatementFull } from "@maple/domain/clickhouse" +import { + applyRealMigrations, + clickhouseE2eEnabled, + clickhouseExec, + uniqueDatabase, +} from "./clickhouse-e2e-support" + +const database = uniqueDatabase("maple_ci_runs_e2e") +const ORG_ID = "org_ci_runs_e2e" + +// Anchored to now: `traces` carries a 30-day TTL enforced at insert. +const BASE_MS = Date.now() - 2 * 3_600_000 +const chDateTime = (epochMs: number): string => new Date(epochMs).toISOString().replace("T", " ").slice(0, 19) + +const REPO = "map('service.name', 'github-actions', 'vcs.owner.name', 'acme', 'vcs.repository.name', 'api')" + +interface SeedSpan { + readonly traceId: string + readonly spanId: string + readonly parentSpanId: string + readonly name: string + readonly service?: string + readonly durationNs: number + readonly status?: string + readonly attributes: Record +} + +const run = (traceId: string, result: string, extra: Record = {}): SeedSpan => ({ + traceId, + spanId: `run-${traceId}`, + parentSpanId: "", + name: "RUN CI", + durationNs: 120_000_000_000, + attributes: { + "cicd.pipeline.name": "CI", + "cicd.pipeline.run.id": `9${traceId.length}`, + "cicd.pipeline.result": result, + "cicd.pipeline.run.url.full": `https://github.com/acme/api/actions/runs/${traceId}`, + "maple.cicd.run_attempt": "1", + "maple.cicd.run_number": "42", + "maple.cicd.trigger_event": "pull_request", + "maple.cicd.actor": "octocat", + "maple.cicd.workflow_path": ".github/workflows/ci.yml", + "vcs.change.id": "17", + "vcs.ref.head.name": "feat/x", + "vcs.ref.head.revision": "abc123", + ...extra, + }, +}) + +const job = ( + traceId: string, + name: string, + result: string, + extra: Record = {}, +): SeedSpan => ({ + traceId, + spanId: `job-${traceId}-${name}`, + parentSpanId: `run-${traceId}`, + name, + durationNs: 60_000_000_000, + status: result === "failure" ? "Error" : "Unset", + attributes: { + "cicd.pipeline.name": "CI", + "cicd.pipeline.task.name": name, + "cicd.pipeline.task.run.result": result, + "cicd.pipeline.task.run.url.full": `https://github.com/acme/api/actions/runs/${traceId}/job/1`, + "cicd.worker.name": "GitHub Actions 7", + "maple.cicd.queue_duration_ms": "3000", + "maple.cicd.run_attempt": "1", + "maple.cicd.runner_labels": "ubuntu-latest", + ...extra, + }, +}) + +const SPANS: ReadonlyArray = [ + run("t1", "failure"), + job("t1", "lint", "success"), + job("t1", "test", "failure", { "maple.cicd.failure.reason": "Process completed with exit code 1." }), + // A step and a runner wait sit under a job: neither is a job. + { + traceId: "t1", + spanId: "step-t1-1", + parentSpanId: "job-t1-test", + name: "Run tests", + durationNs: 1_000_000_000, + attributes: { "cicd.pipeline.task.name": "Run tests", "maple.cicd.step.number": "3" }, + }, + { + traceId: "t1", + spanId: "wait-t1", + parentSpanId: "job-t1-test", + name: "Waiting for a runner", + durationNs: 3_000_000_000, + attributes: { "maple.cicd.job.name": "test", "maple.cicd.queue_duration_ms": "3000" }, + }, + // A skipped run is kept (the runs list shows it); one on a push has no pull request. + run("t2", "skip", { + "vcs.change.id": "", + "maple.cicd.trigger_event": "push", + "maple.cicd.run_attempt": "2", + }), + // Another service's root span is not a CI run. + { + traceId: "t3", + spanId: "api-root", + parentSpanId: "", + name: "GET /", + service: "api", + durationNs: 5, + attributes: {}, + }, +] + +const insert = (spans: ReadonlyArray) => + clickhouseExec( + `INSERT INTO traces + (OrgId, Timestamp, TraceId, SpanId, ParentSpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode, StatusMessage, SampleRate, SpanAttributes, ResourceAttributes) + VALUES\n${spans + .map((span, index) => { + const attributes = Object.entries(span.attributes) + .map(([key, value]) => `'${key}', '${value}'`) + .join(", ") + return `('${ORG_ID}', '${chDateTime(BASE_MS + index * 1000)}', '${span.traceId}', '${span.spanId}', '${span.parentSpanId}', '${span.name}', 'Internal', '${span.service ?? "github-actions"}', ${span.durationNs}, '${span.status ?? "Unset"}', '', 1, map(${attributes}), ${REPO})` + }) + .join("\n,")}`, + database, + ) + +const runJson = async (sql: string): Promise>> => { + const body = await clickhouseExec(`${sql} FORMAT JSON`, database, { + output_format_json_quote_64bit_integers: "0", + }) + return (JSON.parse(body) as { readonly data?: ReadonlyArray> }).data ?? [] +} + +const runs = () => + runJson( + `SELECT TraceId, Owner, Repository, Pipeline, RunId, RunNumber, Attempt, Result, Duration, Url, Branch, + Revision, TriggerEvent, Actor, PullRequest, WorkflowPath + FROM ci_runs WHERE OrgId = '${ORG_ID}' ORDER BY TraceId`, + ) +const jobs = () => + runJson( + `SELECT TraceId, Pipeline, Job, Result, StatusCode, QueueMs, FailureReason, RunnerName, RunnerLabels + FROM ci_jobs WHERE OrgId = '${ORG_ID}' ORDER BY TraceId, Job`, + ) + +const backfills = migrations + .find((migration) => migration.version === 47)! + .statements.filter((statement) => typeof statement !== "string") + +describe.skipIf(!clickhouseE2eEnabled)("ci_runs / ci_jobs materialization", () => { + beforeAll(async () => { + await clickhouseExec(`CREATE DATABASE ${database}`) + await applyRealMigrations(database) + await insert(SPANS) + }, 180_000) + + afterAll(async () => { + await clickhouseExec(`DROP DATABASE IF EXISTS ${database}`) + }, 30_000) + + it("keeps one row per run root span with its attributes as columns", async () => { + assert.deepStrictEqual(await runs(), [ + { + TraceId: "t1", + Owner: "acme", + Repository: "api", + Pipeline: "CI", + RunId: "92", + RunNumber: 42, + Attempt: 1, + Result: "failure", + Duration: 120_000_000_000, + Url: "https://github.com/acme/api/actions/runs/t1", + Branch: "feat/x", + Revision: "abc123", + TriggerEvent: "pull_request", + Actor: "octocat", + PullRequest: 17, + WorkflowPath: ".github/workflows/ci.yml", + }, + { + TraceId: "t2", + Owner: "acme", + Repository: "api", + Pipeline: "CI", + RunId: "92", + RunNumber: 42, + Attempt: 2, + Result: "skip", + Duration: 120_000_000_000, + Url: "https://github.com/acme/api/actions/runs/t2", + Branch: "feat/x", + Revision: "abc123", + TriggerEvent: "push", + Actor: "octocat", + PullRequest: 0, + WorkflowPath: ".github/workflows/ci.yml", + }, + ]) + }) + + it("keeps jobs and leaves out their steps and runner waits", async () => { + assert.deepStrictEqual(await jobs(), [ + { + TraceId: "t1", + Pipeline: "CI", + Job: "lint", + Result: "success", + StatusCode: "Unset", + QueueMs: 3000, + FailureReason: "", + RunnerName: "GitHub Actions 7", + RunnerLabels: "ubuntu-latest", + }, + { + TraceId: "t1", + Pipeline: "CI", + Job: "test", + Result: "failure", + StatusCode: "Error", + QueueMs: 3000, + FailureReason: "Process completed with exit code 1.", + RunnerName: "GitHub Actions 7", + RunnerLabels: "ubuntu-latest", + }, + ]) + }) + + it("backfills the same rows, and never doubles a row the view already wrote", async () => { + const [liveRuns, liveJobs] = [await runs(), await jobs()] + assert.strictEqual(backfills.length, 2) + + // Over tables the views already filled: every span is skipped. + for (const backfill of backfills) + await clickhouseExec(renderStatementFull(backfill, database), database) + assert.deepStrictEqual(await runs(), liveRuns) + assert.deepStrictEqual(await jobs(), liveJobs) + + // Over empty tables: rebuilt from the raw spans alone. + await clickhouseExec("TRUNCATE TABLE ci_runs", database) + await clickhouseExec("TRUNCATE TABLE ci_jobs", database) + for (const backfill of backfills) + await clickhouseExec(renderStatementFull(backfill, database), database) + assert.deepStrictEqual(await runs(), liveRuns) + assert.deepStrictEqual(await jobs(), liveJobs) + }) +}) diff --git a/packages/backend/src/services/warehouse/warehouse-catalog.ts b/packages/backend/src/services/warehouse/warehouse-catalog.ts index a2c157ac3a..a2766286d9 100644 --- a/packages/backend/src/services/warehouse/warehouse-catalog.ts +++ b/packages/backend/src/services/warehouse/warehouse-catalog.ts @@ -56,6 +56,16 @@ const TABLE_NOTES: Record> = { "Scanners borrow crawler user agents and mostly get 404s: count pages actually read with `HttpStatus < 400` (0 means the span carried no status).", "Sorting key: `(OrgId, Timestamp, TraceId)`; filled forward by its MV from migration 0033, never backfilled.", ], + ci_runs: [ + "One row per CI workflow run attempt: the root span of a `github-actions` trace, with `Owner`, `Repository`, `Pipeline` (workflow name), `Result` (success, failure, timeout, error, cancellation, skip), `Branch`, `Revision`, `PullRequest` (0 = none) and `Attempt` pre-extracted. ALWAYS prefer this over `traces` + `SpanAttributes['cicd.pipeline.*']`.", + "`Duration` is NANOSECONDS. Leave out `Result = 'skip'` for durations and success rates: a skipped run did nothing.", + "Keeps 365 days, past the 30-day raw trace its `TraceId` links to. Sorting key: `(OrgId, Timestamp, TraceId)`.", + ], + ci_jobs: [ + "One row per CI job (a child of a run span; steps and runner waits are not included), with `Job`, `Result`, `QueueMs` (wait for a runner), `RunnerName`, `RunnerLabels` and `FailureReason` pre-extracted. Join to `ci_runs` on `TraceId`.", + "`Duration` is NANOSECONDS and includes the queue wait. A skipped job, or one cancelled before a runner took it (`Duration = 0`), never ran.", + "Keeps 365 days. Sorting key: `(OrgId, Timestamp, TraceId)`.", + ], service_overview_spans: [ "Pre-materialized projection of entry-point spans only (Server/Consumer kinds + root spans). Use for per-service request count, error rate, p50/p95/p99 latency.", "`Duration` is NANOSECONDS — divide by 1e6 for ms.", diff --git a/packages/domain/src/clickhouse/migrations/0047_ci_runs_jobs.ts b/packages/domain/src/clickhouse/migrations/0047_ci_runs_jobs.ts new file mode 100644 index 0000000000..3b23b0fd82 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0047_ci_runs_jobs.ts @@ -0,0 +1,100 @@ +import type { BackfillSpec } from "../backfill" + +/** + * Migration 0047: `ci_runs` and `ci_jobs`, the CI analytics' filtered projections of + * `github-actions` traces. Reading the run and job attributes off raw `traces` scanned every + * step span's Map columns; these keep one row per run and per job, for a year. + * + * The views go first, then the retained raw history is backfilled. Nothing is emptied first: a + * replay must keep the rows older than raw retention, so the backfill skips every span its target + * already holds, which also covers spans the views write meanwhile. `requiredForIngest: false`. + * + * The CREATE statements are the verbatim DDL the schema emitter produced at v47. + * Frozen history: never re-derive it from a later snapshot. + */ +const CI_RUNS_PROJECTION_SQL = + " OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ResourceAttributes['vcs.owner.name'] AS Owner,\n ResourceAttributes['vcs.repository.name'] AS Repository,\n SpanAttributes['cicd.pipeline.name'] AS Pipeline,\n SpanAttributes['cicd.pipeline.run.id'] AS RunId,\n toUInt64OrZero(SpanAttributes['maple.cicd.run_number']) AS RunNumber,\n toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt,\n SpanAttributes['cicd.pipeline.result'] AS Result,\n Duration,\n SpanAttributes['cicd.pipeline.run.url.full'] AS Url,\n SpanAttributes['vcs.ref.head.name'] AS Branch,\n SpanAttributes['vcs.ref.head.revision'] AS Revision,\n SpanAttributes['maple.cicd.trigger_event'] AS TriggerEvent,\n SpanAttributes['maple.cicd.actor'] AS Actor,\n toUInt32OrZero(SpanAttributes['vcs.change.id']) AS PullRequest,\n SpanAttributes['maple.cicd.workflow_path'] AS WorkflowPath" +const CI_RUNS_FILTER = "ServiceName = 'github-actions'\n AND ParentSpanId = ''" +const CI_JOBS_PROJECTION_SQL = + " OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ResourceAttributes['vcs.owner.name'] AS Owner,\n ResourceAttributes['vcs.repository.name'] AS Repository,\n SpanAttributes['cicd.pipeline.name'] AS Pipeline,\n SpanAttributes['cicd.pipeline.run.id'] AS RunId,\n toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt,\n SpanAttributes['cicd.pipeline.task.name'] AS Job,\n SpanAttributes['cicd.pipeline.task.run.result'] AS Result,\n StatusCode,\n Duration,\n toUInt32OrZero(SpanAttributes['maple.cicd.queue_duration_ms']) AS QueueMs,\n SpanAttributes['cicd.pipeline.task.run.url.full'] AS Url,\n SpanAttributes['maple.cicd.failure.reason'] AS FailureReason,\n SpanAttributes['cicd.worker.name'] AS RunnerName,\n SpanAttributes['maple.cicd.runner_labels'] AS RunnerLabels" +const CI_JOBS_FILTER = + "ServiceName = 'github-actions'\n AND ParentSpanId != ''\n AND SpanAttributes['cicd.pipeline.task.name'] != ''\n AND SpanAttributes['maple.cicd.step.number'] = ''" + +/** Unqualified like the ALTERs: both apply paths set the session database. */ +const notYetIn = (target: string) => `(TraceId, SpanId) NOT IN (SELECT TraceId, SpanId FROM ${target})` + +const ciRunsBackfill: BackfillSpec = { + kind: "backfill", + target: "ci_runs", + columns: [ + "OrgId", + "Timestamp", + "TraceId", + "SpanId", + "Owner", + "Repository", + "Pipeline", + "RunId", + "RunNumber", + "Attempt", + "Result", + "Duration", + "Url", + "Branch", + "Revision", + "TriggerEvent", + "Actor", + "PullRequest", + "WorkflowPath", + ], + from: "traces", + tsColumn: "Timestamp", + select: CI_RUNS_PROJECTION_SQL, + where: `${CI_RUNS_FILTER} AND ${notYetIn("ci_runs")}`, +} + +const ciJobsBackfill: BackfillSpec = { + kind: "backfill", + target: "ci_jobs", + columns: [ + "OrgId", + "Timestamp", + "TraceId", + "SpanId", + "Owner", + "Repository", + "Pipeline", + "RunId", + "Attempt", + "Job", + "Result", + "StatusCode", + "Duration", + "QueueMs", + "Url", + "FailureReason", + "RunnerName", + "RunnerLabels", + ], + from: "traces", + tsColumn: "Timestamp", + select: CI_JOBS_PROJECTION_SQL, + where: `${CI_JOBS_FILTER} AND ${notYetIn("ci_jobs")}`, +} + +export const migration_0047_ci_runs_jobs = { + version: 47, + description: + "Create ci_runs + ci_jobs and their views: CI runs and jobs off github-actions traces, backfilled", + requiredForIngest: false, + statements: [ + "CREATE TABLE IF NOT EXISTS ci_runs (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n Owner LowCardinality(String),\n Repository LowCardinality(String),\n Pipeline LowCardinality(String),\n RunId String,\n RunNumber UInt64,\n Attempt UInt16,\n Result LowCardinality(String),\n Duration UInt64,\n Url String,\n Branch String,\n Revision String,\n TriggerEvent LowCardinality(String),\n Actor LowCardinality(String),\n PullRequest UInt32,\n WorkflowPath LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toYYYYMM(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", + "CREATE TABLE IF NOT EXISTS ci_jobs (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n Owner LowCardinality(String),\n Repository LowCardinality(String),\n Pipeline LowCardinality(String),\n RunId String,\n Attempt UInt16,\n Job LowCardinality(String),\n Result LowCardinality(String),\n StatusCode LowCardinality(String),\n Duration UInt64,\n QueueMs UInt32,\n Url String,\n FailureReason String,\n RunnerName LowCardinality(String),\n RunnerLabels LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toYYYYMM(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", + "ALTER TABLE ci_runs MODIFY SETTING ttl_only_drop_parts = 1", + "ALTER TABLE ci_jobs MODIFY SETTING ttl_only_drop_parts = 1", + `CREATE MATERIALIZED VIEW IF NOT EXISTS ci_runs_mv TO ci_runs AS\nSELECT\n${CI_RUNS_PROJECTION_SQL}\n FROM traces\n WHERE ${CI_RUNS_FILTER}`, + `CREATE MATERIALIZED VIEW IF NOT EXISTS ci_jobs_mv TO ci_jobs AS\nSELECT\n${CI_JOBS_PROJECTION_SQL}\n FROM traces\n WHERE ${CI_JOBS_FILTER}`, + ciRunsBackfill, + ciJobsBackfill, + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index e715d00cae..a93c26bb6e 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -1,6 +1,6 @@ import { String as Str } from "effect" import { describe, expect, it } from "vitest" -import { isBackfill, renderStatementFull } from "../backfill" +import { type BackfillSpec, isBackfill, renderStatementFull } from "../backfill" import { migration_0004_service_namespace_projections } from "./0004_service_namespace_projections" import { migration_0008_service_operations_minutely, @@ -239,10 +239,15 @@ describe("ClickHouse migrations", () => { // identity_links and trace_index are exempt because a replay is a merge // no-op: their only aggregates are SimpleAggregateFunction(min/max) keyed by // the full sorting key, so re-inserted rows collapse to the values stored. + // A backfill that skips every span its target already holds (0047) converges + // without emptying a target that outlives its source. const mergeIdempotentTargets = new Set(["identity_links", "trace_index"]) + const skipsStored = (stmt: BackfillSpec) => + stmt.where?.includes(`(TraceId, SpanId) NOT IN (SELECT TraceId, SpanId FROM ${stmt.target})`) === + true for (const migration of migrations) { migration.statements.forEach((stmt, index) => { - if (!isBackfill(stmt) || mergeIdempotentTargets.has(stmt.target)) return + if (!isBackfill(stmt) || mergeIdempotentTargets.has(stmt.target) || skipsStored(stmt)) return const before = migration.statements .slice(0, index) .filter((s): s is string => typeof s === "string") diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index c09b42612a..68e79dec05 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -45,6 +45,7 @@ import { migration_0043_traces_span_name_index } from "./0043_traces_span_name_i import { migration_0044_drop_trace_detail_spans } from "./0044_drop_trace_detail_spans" import { migration_0045_ttl_only_drop_parts } from "./0045_ttl_only_drop_parts" import { migration_0046_source_usage_hourly } from "./0046_source_usage_hourly" +import { migration_0047_ci_runs_jobs } from "./0047_ci_runs_jobs" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -122,6 +123,7 @@ export const migrations: ReadonlyArray = [ migration_0044_drop_trace_detail_spans, migration_0045_ttl_only_drop_parts, migration_0046_source_usage_hourly, + migration_0047_ci_runs_jobs, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index e3395cdcec..932f3079c7 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "7fed5d73a33c0c907e0634ed216631b41ecd24f213e58dcc5b9df4b9c1f314c5" as const +export const projectRevision = "254be0396778febcafa4a5b5f6dfff07c0af73b2078c528fcf2bfa98cfa7a884" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS ai_crawler_requests (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n ServiceName LowCardinality(String),\n Crawler LowCardinality(String),\n Host LowCardinality(String),\n Path String,\n HttpStatus UInt16\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", @@ -10,6 +10,8 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS attribute_keys_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, AttributeScope, Hour, AttributeKey)\nTTL toDate(Hour) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS attribute_values_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeValue String,\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue)\nTTL toDate(Hour) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS audit_log (\n OrgId LowCardinality(String),\n Id String,\n OccurredAt DateTime64(3),\n RecordedAt DateTime64(3),\n ActorType LowCardinality(String),\n UserId String,\n ApiKeyId String,\n ActorId String,\n ActorLabel String,\n AffectedUserId String,\n Source LowCardinality(String),\n Action LowCardinality(String),\n Outcome LowCardinality(String),\n DenialReason String,\n ResourceType LowCardinality(String),\n ResourceId String,\n ChangedFields Array(String),\n Changes String,\n Metadata String,\n RequestId String,\n OriginIp String,\n OriginCountry LowCardinality(String)\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toYYYYMM(OccurredAt)\nORDER BY (OrgId, OccurredAt, Id)\nTTL toDate(OccurredAt) + INTERVAL 2190 DAY", + "CREATE TABLE IF NOT EXISTS ci_jobs (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n Owner LowCardinality(String),\n Repository LowCardinality(String),\n Pipeline LowCardinality(String),\n RunId String,\n Attempt UInt16,\n Job LowCardinality(String),\n Result LowCardinality(String),\n StatusCode LowCardinality(String),\n Duration UInt64,\n QueueMs UInt32,\n Url String,\n FailureReason String,\n RunnerName LowCardinality(String),\n RunnerLabels LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toYYYYMM(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", + "CREATE TABLE IF NOT EXISTS ci_runs (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n Owner LowCardinality(String),\n Repository LowCardinality(String),\n Pipeline LowCardinality(String),\n RunId String,\n RunNumber UInt64,\n Attempt UInt16,\n Result LowCardinality(String),\n Duration UInt64,\n Url String,\n Branch String,\n Revision String,\n TriggerEvent LowCardinality(String),\n Actor LowCardinality(String),\n PullRequest UInt32,\n WorkflowPath LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toYYYYMM(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS error_events (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String,\n ServiceVersion LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, FingerprintHash, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_events_by_time (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String,\n ServiceVersion LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, FingerprintHash)\nTTL toDate(Timestamp) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_fingerprints_minutely (\n OrgId LowCardinality(String),\n Minute DateTime,\n FingerprintHash UInt64,\n ServiceName SimpleAggregateFunction(anyLast, String),\n ExceptionType SimpleAggregateFunction(anyLast, String),\n ExceptionMessage SimpleAggregateFunction(anyLast, String),\n ErrorLabel SimpleAggregateFunction(anyLast, String),\n TopFrame SimpleAggregateFunction(anyLast, String),\n OccurrenceCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime),\n ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String))\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toYYYYMM(Minute)\nORDER BY (OrgId, Minute, FingerprintHash)\nTTL toDate(Minute) + INTERVAL 90 DAY", @@ -49,6 +51,8 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS traces_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE MATERIALIZED VIEW IF NOT EXISTS ai_crawler_requests_mv TO ai_crawler_requests AS\nSELECT\n OrgId,\n Timestamp,\n TraceId,\n ServiceName,\n arrayElement(['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'Meta-ExternalAgent', 'Meta-WebIndexer', 'Meta-ExternalFetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai'], multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'meta-externalagent', 'meta-webindexer', 'meta-externalfetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai'])) AS Crawler,\n lower(replaceRegexpOne(coalesce(nullIf(SpanAttributes['server.address'], ''), nullIf(SpanAttributes['http.host'], ''), nullIf(SpanAttributes['net.host.name'], ''), nullIf(domain(SpanAttributes['url.full']), ''), nullIf(domain(SpanAttributes['http.url']), ''), ''), ':[0-9]+$', '')) AS Host,\n leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) AS Path,\n toUInt16OrZero(coalesce(nullIf(SpanAttributes['http.response.status_code'], ''), nullIf(SpanAttributes['http.status_code'], ''), '')) AS HttpStatus\n FROM traces\n WHERE SpanKind = 'Server'\n AND multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'meta-externalagent', 'meta-webindexer', 'meta-externalfetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai']) > 0\n AND leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS ai_trace_index_mv TO ai_trace_index AS\nSELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanAttributes['maple_ai.session.id'] AS SessionId,\n SpanAttributes['maple_ai.vendor.id'] AS VendorId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n SpanAttributes['maple_ai.model'] AS Model,\n SpanAttributes['maple_ai.agent.name'] AS AgentName,\n SpanAttributes['maple_ai.tool.name'] AS ToolName,\n SpanId,\n ParentSpanId,\n Duration,\n toUInt8(SpanAttributes['maple_ai.error'] = '1') AS IsError,\n toUInt8(SpanAttributes['maple_ai.llm_call'] = '1') AS IsLlmCall,\n toUInt8(SpanAttributes['maple_ai.tool_call'] = '1') AS IsToolCall,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS Tokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.cost']) AS Cost,\n SpanAttributes['maple_ai.response.id'] AS ResponseId,\n SpanAttributes['maple_ai.vendor.version'] AS VendorVersion,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) AS InputTokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) AS CacheReadTokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) AS CacheWriteTokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) AS OutputTokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS ReasoningTokens,\n SpanAttributes['error.type'] AS ErrorType,\n leftUTF8(StatusMessage, 400) AS StatusMessage,\n SpanAttributes['maple_ai.tool.description'] AS ToolDescription,\n SpanAttributes['maple_ai.tool.error_result'] AS FailedToolCallResult,\n if(SpanAttributes['maple_ai.error'] = '1', cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(leftUTF8(coalesce(nullIf(SpanAttributes['maple_ai.tool.error_result'], ''), leftUTF8(StatusMessage, 400)), 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#')), 0) AS ErrorFingerprint\n FROM traces\n WHERE SpanAttributes['maple_ai.vendor.id'] != ''", + "CREATE MATERIALIZED VIEW IF NOT EXISTS ci_jobs_mv TO ci_jobs AS\nSELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ResourceAttributes['vcs.owner.name'] AS Owner,\n ResourceAttributes['vcs.repository.name'] AS Repository,\n SpanAttributes['cicd.pipeline.name'] AS Pipeline,\n SpanAttributes['cicd.pipeline.run.id'] AS RunId,\n toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt,\n SpanAttributes['cicd.pipeline.task.name'] AS Job,\n SpanAttributes['cicd.pipeline.task.run.result'] AS Result,\n StatusCode,\n Duration,\n toUInt32OrZero(SpanAttributes['maple.cicd.queue_duration_ms']) AS QueueMs,\n SpanAttributes['cicd.pipeline.task.run.url.full'] AS Url,\n SpanAttributes['maple.cicd.failure.reason'] AS FailureReason,\n SpanAttributes['cicd.worker.name'] AS RunnerName,\n SpanAttributes['maple.cicd.runner_labels'] AS RunnerLabels\n FROM traces\n WHERE ServiceName = 'github-actions'\n AND ParentSpanId != ''\n AND SpanAttributes['cicd.pipeline.task.name'] != ''\n AND SpanAttributes['maple.cicd.step.number'] = ''", + "CREATE MATERIALIZED VIEW IF NOT EXISTS ci_runs_mv TO ci_runs AS\nSELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ResourceAttributes['vcs.owner.name'] AS Owner,\n ResourceAttributes['vcs.repository.name'] AS Repository,\n SpanAttributes['cicd.pipeline.name'] AS Pipeline,\n SpanAttributes['cicd.pipeline.run.id'] AS RunId,\n toUInt64OrZero(SpanAttributes['maple.cicd.run_number']) AS RunNumber,\n toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt,\n SpanAttributes['cicd.pipeline.result'] AS Result,\n Duration,\n SpanAttributes['cicd.pipeline.run.url.full'] AS Url,\n SpanAttributes['vcs.ref.head.name'] AS Branch,\n SpanAttributes['vcs.ref.head.revision'] AS Revision,\n SpanAttributes['maple.cicd.trigger_event'] AS TriggerEvent,\n SpanAttributes['maple.cicd.actor'] AS Actor,\n toUInt32OrZero(SpanAttributes['vcs.change.id']) AS PullRequest,\n SpanAttributes['maple.cicd.workflow_path'] AS WorkflowPath\n FROM traces\n WHERE ServiceName = 'github-actions'\n AND ParentSpanId = ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\\\x1b\\\\[[0-9;]*m', ''), '^((\\\\[Error\\\\]|[\\\\s\\\\S]*?DB::Exception:)[\\\\s\\\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\\\s\\\\S]*', '\\\\1'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])~/[^ \\'\"`:,)]*', '\\\\1PATH'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '\\\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\\\b', 'ID'), '\\\\b[0-9A-HJKMNP-TV-Z]{26}\\\\b', 'ID'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\\\b', '\\\\1\\\\2§\\\\3'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\\\b', '\\\\1\\\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\\\1'),\n 1, 120\n )\n ) AS _msgSig,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+')\n AND position(line, '(definition)') = 0\n AND NOT match(line, '^[ \\\\t]*at \\\\[')\n AND NOT match(line, '^[ \\\\t]*at ([^(:]|:[^0-9])*:?$'),\n splitByChar('\\n', _exStack)\n ),\n -- A namespaced type plus its message names one failure: keep only the\n -- throw site. With no message the frames are all that is left; keep 3.\n 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3)\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\\\.[0-9]+\\\\.[0-9]+[^/ )]*', '@#'), '\\\\(index [0-9]+\\\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\\\.css', '.css'), '(/_astro/[^/ ).]+)\\\\.[A-Za-z0-9_-]{8}\\\\.js', '\\\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\\\x1b\\\\[[0-9;]*m', ''), '^((\\\\[Error\\\\]|[\\\\s\\\\S]*?DB::Exception:)[\\\\s\\\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\\\s\\\\S]*', '\\\\1'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])~/[^ \\'\"`:,)]*', '\\\\1PATH'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '\\\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\\\b', 'ID'), '\\\\b[0-9A-HJKMNP-TV-Z]{26}\\\\b', 'ID'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\\\b', '\\\\1\\\\2§\\\\3'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\\\b', '\\\\1\\\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\\\1'),\n 1, 120\n )\n ) AS _msgSig,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+')\n AND position(line, '(definition)') = 0\n AND NOT match(line, '^[ \\\\t]*at \\\\[')\n AND NOT match(line, '^[ \\\\t]*at ([^(:]|:[^0-9])*:?$'),\n splitByChar('\\n', _exStack)\n ),\n -- A namespaced type plus its message names one failure: keep only the\n -- throw site. With no message the frames are all that is left; keep 3.\n 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3)\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\\\.[0-9]+\\\\.[0-9]+[^/ )]*', '@#'), '\\\\(index [0-9]+\\\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\\\.css', '.css'), '(/_astro/[^/ ).]+)\\\\.[A-Za-z0-9_-]{8}\\\\.js', '\\\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(Timestamp) AS Minute,\n FingerprintHash,\n anyLast(ServiceName) AS ServiceName,\n anyLast(ExceptionType) AS ExceptionType,\n anyLast(ExceptionMessage) AS ExceptionMessage,\n anyLast(ErrorLabel) AS ErrorLabel,\n anyLast(TopFrame) AS TopFrame,\n count() AS OccurrenceCount,\n min(Timestamp) AS FirstSeen,\n max(Timestamp) AS LastSeen,\n -- Distinct builds, not a sample: see ServiceVersions on the datasource.\n groupUniqArray(ServiceVersion) AS ServiceVersions\n FROM error_events\n GROUP BY OrgId, Minute, FingerprintHash", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 235129e8bc..9b85964955 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "7fed5d73a33c0c907e0634ed216631b41ecd24f213e58dcc5b9df4b9c1f314c5" as const +export const projectRevision = "254be0396778febcafa4a5b5f6dfff07c0af73b2078c528fcf2bfa98cfa7a884" as const export const datasources = [ { @@ -34,6 +34,16 @@ export const datasources = [ content: 'DESCRIPTION >\n Org-wide audit trail: allowed and denied actions plus telemetry/session-replay reads, attributed to the user, API key, or agent that performed them. Admin-only; read through GET /v2/audit_log.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.OrgId`,\n Id String `json:$.Id`,\n OccurredAt DateTime64(3) `json:$.OccurredAt`,\n RecordedAt DateTime64(3) `json:$.RecordedAt`,\n ActorType LowCardinality(String) `json:$.ActorType`,\n UserId String `json:$.UserId`,\n ApiKeyId String `json:$.ApiKeyId`,\n ActorId String `json:$.ActorId`,\n ActorLabel String `json:$.ActorLabel`,\n AffectedUserId String `json:$.AffectedUserId`,\n Source LowCardinality(String) `json:$.Source`,\n Action LowCardinality(String) `json:$.Action`,\n Outcome LowCardinality(String) `json:$.Outcome`,\n DenialReason String `json:$.DenialReason`,\n ResourceType LowCardinality(String) `json:$.ResourceType`,\n ResourceId String `json:$.ResourceId`,\n ChangedFields Array(String) `json:$.ChangedFields[:]`,\n Changes String `json:$.Changes`,\n Metadata String `json:$.Metadata`,\n RequestId String `json:$.RequestId`,\n OriginIp String `json:$.OriginIp`,\n OriginCountry LowCardinality(String) `json:$.OriginCountry`\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "toYYYYMM(OccurredAt)"\nENGINE_SORTING_KEY "OrgId, OccurredAt, Id"\nENGINE_TTL "toDate(OccurredAt) + INTERVAL 2190 DAY"', }, + { + name: "ci_jobs", + content: + 'DESCRIPTION >\n One row per CI job (a child of a github-actions run span) with workflow, job name, result, queue time, runner and failure reason pre-extracted. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n Owner LowCardinality(String),\n Repository LowCardinality(String),\n Pipeline LowCardinality(String),\n RunId String,\n Attempt UInt16,\n Job LowCardinality(String),\n Result LowCardinality(String),\n StatusCode LowCardinality(String),\n Duration UInt64,\n QueueMs UInt32,\n Url String,\n FailureReason String,\n RunnerName LowCardinality(String),\n RunnerLabels LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toYYYYMM(Timestamp)"\nENGINE_SORTING_KEY "OrgId, Timestamp, TraceId"\nENGINE_TTL "toDate(Timestamp) + INTERVAL 365 DAY"', + }, + { + name: "ci_runs", + content: + 'DESCRIPTION >\n One row per CI workflow run attempt (the root span of a github-actions trace) with repository, workflow, result, branch, revision and pull request pre-extracted. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n Owner LowCardinality(String),\n Repository LowCardinality(String),\n Pipeline LowCardinality(String),\n RunId String,\n RunNumber UInt64,\n Attempt UInt16,\n Result LowCardinality(String),\n Duration UInt64,\n Url String,\n Branch String,\n Revision String,\n TriggerEvent LowCardinality(String),\n Actor LowCardinality(String),\n PullRequest UInt32,\n WorkflowPath LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toYYYYMM(Timestamp)"\nENGINE_SORTING_KEY "OrgId, Timestamp, TraceId"\nENGINE_TTL "toDate(Timestamp) + INTERVAL 365 DAY"', + }, { name: "error_events", content: @@ -232,6 +242,16 @@ export const pipes = [ content: "DESCRIPTION >\n Populates ai_trace_index with GenAI agent spans (maple_ai.vendor.id stamped), projecting the facts the ingest gateway stamped on each (maple_ai.*: identity, model/agent/tool, kind, usage, failure, a failed tool call's result, the tool's description) plus the environment, the error.type, the status message and the failure's fingerprint, to plain columns.\n\nNODE ai_trace_index_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanAttributes['maple_ai.session.id'] AS SessionId,\n SpanAttributes['maple_ai.vendor.id'] AS VendorId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n SpanAttributes['maple_ai.model'] AS Model,\n SpanAttributes['maple_ai.agent.name'] AS AgentName,\n SpanAttributes['maple_ai.tool.name'] AS ToolName,\n SpanId,\n ParentSpanId,\n Duration,\n toUInt8(SpanAttributes['maple_ai.error'] = '1') AS IsError,\n toUInt8(SpanAttributes['maple_ai.llm_call'] = '1') AS IsLlmCall,\n toUInt8(SpanAttributes['maple_ai.tool_call'] = '1') AS IsToolCall,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS Tokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.cost']) AS Cost,\n SpanAttributes['maple_ai.response.id'] AS ResponseId,\n SpanAttributes['maple_ai.vendor.version'] AS VendorVersion,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) AS InputTokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) AS CacheReadTokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) AS CacheWriteTokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) AS OutputTokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS ReasoningTokens,\n SpanAttributes['error.type'] AS ErrorType,\n leftUTF8(StatusMessage, 400) AS StatusMessage,\n SpanAttributes['maple_ai.tool.description'] AS ToolDescription,\n SpanAttributes['maple_ai.tool.error_result'] AS FailedToolCallResult,\n if(SpanAttributes['maple_ai.error'] = '1', cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(leftUTF8(coalesce(nullIf(SpanAttributes['maple_ai.tool.error_result'], ''), leftUTF8(StatusMessage, 400)), 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#')), 0) AS ErrorFingerprint\n FROM traces\n WHERE SpanAttributes['maple_ai.vendor.id'] != ''\n\nTYPE MATERIALIZED\nDATASOURCE ai_trace_index\nDEPLOYMENT_METHOD alter", }, + { + name: "ci_jobs_mv", + content: + "DESCRIPTION >\n Populates ci_jobs with the job spans of every github-actions trace, pre-extracting workflow, job, result, queue time, runner and failure reason.\n\nNODE ci_jobs_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ResourceAttributes['vcs.owner.name'] AS Owner,\n ResourceAttributes['vcs.repository.name'] AS Repository,\n SpanAttributes['cicd.pipeline.name'] AS Pipeline,\n SpanAttributes['cicd.pipeline.run.id'] AS RunId,\n toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt,\n SpanAttributes['cicd.pipeline.task.name'] AS Job,\n SpanAttributes['cicd.pipeline.task.run.result'] AS Result,\n StatusCode,\n Duration,\n toUInt32OrZero(SpanAttributes['maple.cicd.queue_duration_ms']) AS QueueMs,\n SpanAttributes['cicd.pipeline.task.run.url.full'] AS Url,\n SpanAttributes['maple.cicd.failure.reason'] AS FailureReason,\n SpanAttributes['cicd.worker.name'] AS RunnerName,\n SpanAttributes['maple.cicd.runner_labels'] AS RunnerLabels\n FROM traces\n WHERE ServiceName = 'github-actions'\n AND ParentSpanId != ''\n AND SpanAttributes['cicd.pipeline.task.name'] != ''\n AND SpanAttributes['maple.cicd.step.number'] = ''\n\nTYPE MATERIALIZED\nDATASOURCE ci_jobs", + }, + { + name: "ci_runs_mv", + content: + "DESCRIPTION >\n Populates ci_runs with the root span of every github-actions trace, pre-extracting repository, workflow, result, branch, revision and pull request.\n\nNODE ci_runs_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ResourceAttributes['vcs.owner.name'] AS Owner,\n ResourceAttributes['vcs.repository.name'] AS Repository,\n SpanAttributes['cicd.pipeline.name'] AS Pipeline,\n SpanAttributes['cicd.pipeline.run.id'] AS RunId,\n toUInt64OrZero(SpanAttributes['maple.cicd.run_number']) AS RunNumber,\n toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt,\n SpanAttributes['cicd.pipeline.result'] AS Result,\n Duration,\n SpanAttributes['cicd.pipeline.run.url.full'] AS Url,\n SpanAttributes['vcs.ref.head.name'] AS Branch,\n SpanAttributes['vcs.ref.head.revision'] AS Revision,\n SpanAttributes['maple.cicd.trigger_event'] AS TriggerEvent,\n SpanAttributes['maple.cicd.actor'] AS Actor,\n toUInt32OrZero(SpanAttributes['vcs.change.id']) AS PullRequest,\n SpanAttributes['maple.cicd.workflow_path'] AS WorkflowPath\n FROM traces\n WHERE ServiceName = 'github-actions'\n AND ParentSpanId = ''\n\nTYPE MATERIALIZED\nDATASOURCE ci_runs", + }, { name: "error_events_by_time_mv", content: diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index 4be0ad6f51..1306d67d28 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -1314,6 +1314,85 @@ export const aiCrawlerRequests = defineDatasource("ai_crawler_requests", { export type AiCrawlerRequestsRow = InferRow +/** + * CI workflow runs: the root span of each `github-actions` trace (CiTraceService), with its + * `cicd.*` attributes as columns. Skipped runs are kept; the reads leave them out. + * Tiny (one row per run) so it keeps a year, past the raw trace a run links to. + */ +export const ciRuns = defineDatasource("ci_runs", { + tenantColumn: "OrgId", + description: + "One row per CI workflow run attempt (the root span of a github-actions trace) with repository, workflow, result, branch, revision and pull request pre-extracted. Populated by materialized view.", + jsonPaths: false, + schema: { + OrgId: t.string().lowCardinality().brand(OrgId), + Timestamp: utcDateTime64(9), + TraceId: t.string(), + SpanId: t.string(), + Owner: t.string().lowCardinality(), + Repository: t.string().lowCardinality(), + Pipeline: t.string().lowCardinality(), + RunId: t.string(), + RunNumber: t.uint64(), + Attempt: t.uint16(), + Result: t.string().lowCardinality(), + Duration: t.uint64(), + Url: t.string(), + Branch: t.string(), + Revision: t.string(), + TriggerEvent: t.string().lowCardinality(), + Actor: t.string().lowCardinality(), + /** 0 when the run has no pull request. */ + PullRequest: t.uint32(), + WorkflowPath: t.string().lowCardinality(), + }, + engine: engine.mergeTree({ + partitionKey: "toYYYYMM(Timestamp)", + sortingKey: ["OrgId", "Timestamp", "TraceId"], + ttl: "toDate(Timestamp) + INTERVAL 365 DAY", + }), +}) + +export type CiRunsRow = InferRow + +/** + * CI jobs: the run span's children that name a task but not a step. Steps and runner waits are + * left out; a job carries its own queue time. Skipped jobs are kept; the reads leave them out. + */ +export const ciJobs = defineDatasource("ci_jobs", { + tenantColumn: "OrgId", + description: + "One row per CI job (a child of a github-actions run span) with workflow, job name, result, queue time, runner and failure reason pre-extracted. Populated by materialized view.", + jsonPaths: false, + schema: { + OrgId: t.string().lowCardinality().brand(OrgId), + Timestamp: utcDateTime64(9), + TraceId: t.string(), + SpanId: t.string(), + Owner: t.string().lowCardinality(), + Repository: t.string().lowCardinality(), + Pipeline: t.string().lowCardinality(), + RunId: t.string(), + Attempt: t.uint16(), + Job: t.string().lowCardinality(), + Result: t.string().lowCardinality(), + StatusCode: t.string().lowCardinality(), + Duration: t.uint64(), + QueueMs: t.uint32(), + Url: t.string(), + FailureReason: t.string(), + RunnerName: t.string().lowCardinality(), + RunnerLabels: t.string().lowCardinality(), + }, + engine: engine.mergeTree({ + partitionKey: "toYYYYMM(Timestamp)", + sortingKey: ["OrgId", "Timestamp", "TraceId"], + ttl: "toDate(Timestamp) + INTERVAL 365 DAY", + }), +}) + +export type CiJobsRow = InferRow + /** * OpenTelemetry sum/counter metrics datasource */ diff --git a/packages/domain/src/tinybird/materializations.ts b/packages/domain/src/tinybird/materializations.ts index 29576e2199..6be192a4d5 100644 --- a/packages/domain/src/tinybird/materializations.ts +++ b/packages/domain/src/tinybird/materializations.ts @@ -33,6 +33,8 @@ import { errorFingerprintsMinutely, aiTraceIndex, aiCrawlerRequests, + ciJobs, + ciRuns, TRACE_BUCKET_EXPR, traceIndex, traceListMv, @@ -1236,6 +1238,84 @@ export const aiCrawlerRequestsMv = defineMaterializedView("ai_crawler_requests_m ], }) +/** `service.name` of the CI traces CiTraceService writes. */ +const CI_SERVICE_NAME = "github-actions" + +/** Populates `ci_runs` with the root span of each CI trace: one row per workflow run attempt. */ +export const ciRunsMv = defineMaterializedView("ci_runs_mv", { + description: + "Populates ci_runs with the root span of every github-actions trace, pre-extracting repository, workflow, result, branch, revision and pull request.", + datasource: ciRuns, + nodes: [ + node({ + name: "ci_runs_mv_node", + sql: ` + SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ResourceAttributes['vcs.owner.name'] AS Owner, + ResourceAttributes['vcs.repository.name'] AS Repository, + SpanAttributes['cicd.pipeline.name'] AS Pipeline, + SpanAttributes['cicd.pipeline.run.id'] AS RunId, + toUInt64OrZero(SpanAttributes['maple.cicd.run_number']) AS RunNumber, + toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt, + SpanAttributes['cicd.pipeline.result'] AS Result, + Duration, + SpanAttributes['cicd.pipeline.run.url.full'] AS Url, + SpanAttributes['vcs.ref.head.name'] AS Branch, + SpanAttributes['vcs.ref.head.revision'] AS Revision, + SpanAttributes['maple.cicd.trigger_event'] AS TriggerEvent, + SpanAttributes['maple.cicd.actor'] AS Actor, + toUInt32OrZero(SpanAttributes['vcs.change.id']) AS PullRequest, + SpanAttributes['maple.cicd.workflow_path'] AS WorkflowPath + FROM traces + WHERE ServiceName = '${CI_SERVICE_NAME}' + AND ParentSpanId = '' + `, + }), + ], +}) + +/** Populates `ci_jobs` with each run's jobs: children that name a task but not a step. */ +export const ciJobsMv = defineMaterializedView("ci_jobs_mv", { + description: + "Populates ci_jobs with the job spans of every github-actions trace, pre-extracting workflow, job, result, queue time, runner and failure reason.", + datasource: ciJobs, + nodes: [ + node({ + name: "ci_jobs_mv_node", + sql: ` + SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ResourceAttributes['vcs.owner.name'] AS Owner, + ResourceAttributes['vcs.repository.name'] AS Repository, + SpanAttributes['cicd.pipeline.name'] AS Pipeline, + SpanAttributes['cicd.pipeline.run.id'] AS RunId, + toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt, + SpanAttributes['cicd.pipeline.task.name'] AS Job, + SpanAttributes['cicd.pipeline.task.run.result'] AS Result, + StatusCode, + Duration, + toUInt32OrZero(SpanAttributes['maple.cicd.queue_duration_ms']) AS QueueMs, + SpanAttributes['cicd.pipeline.task.run.url.full'] AS Url, + SpanAttributes['maple.cicd.failure.reason'] AS FailureReason, + SpanAttributes['cicd.worker.name'] AS RunnerName, + SpanAttributes['maple.cicd.runner_labels'] AS RunnerLabels + FROM traces + WHERE ServiceName = '${CI_SERVICE_NAME}' + AND ParentSpanId != '' + AND SpanAttributes['cicd.pipeline.task.name'] != '' + AND SpanAttributes['maple.cicd.step.number'] = '' + `, + }), + ], +}) + export const traceListMvMv = defineMaterializedView("trace_list_mv_mv", { description: "Populates trace_list_mv from root spans with pre-extracted HTTP attributes and normalized span names.", diff --git a/packages/domain/src/tinybird/materialized-projection-order.test.ts b/packages/domain/src/tinybird/materialized-projection-order.test.ts index 2609561c97..5dae83dbf8 100644 --- a/packages/domain/src/tinybird/materialized-projection-order.test.ts +++ b/packages/domain/src/tinybird/materialized-projection-order.test.ts @@ -111,6 +111,8 @@ describe("materialized projection order", () => { const targets = [ ["ai_trace_index", "ai_trace_index_mv"], ["ai_crawler_requests", "ai_crawler_requests_mv"], + ["ci_runs", "ci_runs_mv"], + ["ci_jobs", "ci_jobs_mv"], ["service_map_edges_hourly", "service_map_edges_hourly_ingest_mv"], ["service_overview_spans", "service_overview_spans_mv"], ["service_overview_hourly", "service_overview_hourly_mv"], diff --git a/packages/domain/src/tinybird/retention-matrix.test.ts b/packages/domain/src/tinybird/retention-matrix.test.ts index c418cf0eba..590b781081 100644 --- a/packages/domain/src/tinybird/retention-matrix.test.ts +++ b/packages/domain/src/tinybird/retention-matrix.test.ts @@ -3,6 +3,9 @@ import { tinybirdProjectManifest } from "../generated/tinybird-project-manifest" const RETENTION_DAYS = { alert_checks: 365, + // CI projections outlive their raw traces on purpose: a year of run history. + ci_jobs: 365, + ci_runs: 365, // Raw-tier projection of `traces`: rebuilt from it, so holding // it past the source's own retention would store rows detection can no // longer cross-check against a raw trace. diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index 5d6fbf51a2..a5e8c370ee 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -152,6 +152,12 @@ export const AiTraceIndex = Datasources.aiTraceIndex */ export const AiCrawlerRequests = Datasources.aiCrawlerRequests +/** CI workflow runs (migration 0047): the root span of each `github-actions` trace. */ +export const CiRuns = Datasources.ciRuns + +/** CI jobs (migration 0047): a run's children that name a task but not a step. */ +export const CiJobs = Datasources.ciJobs + export const TraceListMv = Datasources.traceListMv export const TraceFacetsHourly = Datasources.traceFacetsHourly diff --git a/scripts/purge-org-traces.ts b/scripts/purge-org-traces.ts index 05f0409b55..443ba06b09 100644 --- a/scripts/purge-org-traces.ts +++ b/scripts/purge-org-traces.ts @@ -61,6 +61,8 @@ const TARGETS: ReadonlyArray = [ raw("error_events_by_time"), raw("ai_trace_index"), raw("ai_crawler_requests"), + raw("ci_runs"), + raw("ci_jobs"), raw("service_map_spans"), raw("service_map_children"), raw("service_overview_spans"),