From 8951fcd3b751cb4cac6436600497f9d211de652b Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:17:53 +0000 Subject: [PATCH] feat: add log-based CoderdPanic alert via the Loki ruler --- README.md | 2 +- coder-observability/runbooks/coderd.md | 18 ++++++++- .../templates/_loki-alerts.tpl | 38 +++++++++++++++++++ .../templates/configmap-loki-alerts.yaml | 15 ++++++++ coder-observability/values.yaml | 5 +++ compiled/resources.yaml | 34 +++++++++++++++-- 6 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 coder-observability/templates/_loki-alerts.tpl create mode 100644 coder-observability/templates/configmap-loki-alerts.yaml diff --git a/README.md b/README.md index ff5a20c..05c91f1 100644 --- a/README.md +++ b/README.md @@ -313,7 +313,7 @@ values which are defined [here](https://github.com/grafana/helm-charts/tree/main |-----|------|---------|-------------| | global.alerts.enabled | bool | `true` | enable or disable alerting | | global.alerts.kind | string | `"configmap"` | the container resource kind in which alerts should be created; valid values are "prometheusrule" or "configmap" | -| global.coder.alerts | object | `{"coderd":{"groups":{"CPU":{"delay":"10m","enabled":true,"period":"10m","thresholds":{"critical":0.9,"warning":0.8}},"IneligiblePrebuilds":{"delay":"10m","enabled":true,"thresholds":{"notify":1}},"Memory":{"delay":"10m","enabled":true,"thresholds":{"critical":0.9,"warning":0.8}},"Replicas":{"delay":"5m","enabled":true,"thresholds":{"critical":1,"notify":3,"warning":2}},"Restarts":{"delay":"1m","enabled":true,"period":"10m","thresholds":{"critical":3,"notify":1,"warning":2}},"UnprovisionedPrebuiltWorkspaces":{"delay":"10m","enabled":true,"thresholds":{"warn":1}},"WorkspaceBuildFailures":{"delay":"10m","enabled":true,"period":"10m","thresholds":{"critical":10,"notify":2,"warning":5}}}},"enterprise":{"groups":{"Licences":{"delay":"1m","enabled":true,"thresholds":{"critical":1,"warning":0.9}}}},"provisionerd":{"groups":{"Replicas":{"delay":"5m","enabled":true,"thresholds":{"critical":1,"notify":3,"warning":2}}}}}` | alerts for the various aspects of Coder | +| global.coder.alerts | object | `{"coderd":{"groups":{"CPU":{"delay":"10m","enabled":true,"period":"10m","thresholds":{"critical":0.9,"warning":0.8}},"IneligiblePrebuilds":{"delay":"10m","enabled":true,"thresholds":{"notify":1}},"Memory":{"delay":"10m","enabled":true,"thresholds":{"critical":0.9,"warning":0.8}},"Panics":{"enabled":true,"period":"5m","thresholds":{"critical":0}},"Replicas":{"delay":"5m","enabled":true,"thresholds":{"critical":1,"notify":3,"warning":2}},"Restarts":{"delay":"1m","enabled":true,"period":"10m","thresholds":{"critical":3,"notify":1,"warning":2}},"UnprovisionedPrebuiltWorkspaces":{"delay":"10m","enabled":true,"thresholds":{"warn":1}},"WorkspaceBuildFailures":{"delay":"10m","enabled":true,"period":"10m","thresholds":{"critical":10,"notify":2,"warning":5}}}},"enterprise":{"groups":{"Licences":{"delay":"1m","enabled":true,"thresholds":{"critical":1,"warning":0.9}}}},"provisionerd":{"groups":{"Replicas":{"delay":"5m","enabled":true,"thresholds":{"critical":1,"notify":3,"warning":2}}}}}` | alerts for the various aspects of Coder | | global.coder.coderdSelector | string | `"pod=~`coder.*`, pod!~`.*provisioner.*`"` | series selector for Prometheus/Loki to locate provisioner pods. ensure this uses backticks for quotes! | | global.coder.controlPlaneNamespace | string | `"coder"` | the namespace into which the control plane has been deployed. | | global.coder.externalProvisionersNamespace | string | `"coder"` | the namespace into which any external provisioners have been deployed. | diff --git a/coder-observability/runbooks/coderd.md b/coder-observability/runbooks/coderd.md index 4a42444..ee46570 100644 --- a/coder-observability/runbooks/coderd.md +++ b/coder-observability/runbooks/coderd.md @@ -132,4 +132,20 @@ The prebuilds subsystem runs a _reconciliation loop_ which monitors the state of number of instances are present at all times. Workspace Prebuilds is currently a BETA feature and so there could be a bug in this _reconciliation loop_, which should be reported to Coder. -Examine your coderd logs for any errors or warnings relating to prebuilds. \ No newline at end of file +Examine your coderd logs for any errors or warnings relating to prebuilds. +## CoderdPanic + +A Coder pod logged a panic or fatal runtime error, or recovered from a panic in +an HTTP handler. Panics indicate a bug in Coder. + +Find the traceback in the affected pod's logs: + +```console +kubectl -n coder logs | grep -B 1 -A 30 -E '^(panic|fatal error): |panic serving http request' +``` + +An unrecovered panic also restarts the pod, so check `--previous` logs if the +current logs contain no traceback. + +Report the traceback to [coder/coder](https://github.com/coder/coder/issues) if +an issue does not already exist for it. diff --git a/coder-observability/templates/_loki-alerts.tpl b/coder-observability/templates/_loki-alerts.tpl new file mode 100644 index 0000000..c01f2cb --- /dev/null +++ b/coder-observability/templates/_loki-alerts.tpl @@ -0,0 +1,38 @@ +{{/* +Log-based alerting rules, evaluated by the Loki ruler (LogQL, not PromQL). +See https://grafana.com/docs/loki/latest/alert/ +*/}} +{{- define "coderd-loki-alerts" -}} + {{- $service := dict "service" "coderd" -}} + {{- with .Values.global.coder.alerts.coderd }} + {{- with .groups.Panics }} + {{- $group := . }} + {{- if .enabled }} + - name: Panics + rules: + {{ $alert := "CoderdPanic" }} + {{- range $severity, $threshold := .thresholds }} + - alert: {{ $alert }} + # Panic and fatal-error output is untimestamped, so the collector's multiline + # stage appends it to the preceding timestamped log entry; (?m) lets ^ match + # the start of the embedded panic line. "panic serving http request" is logged + # by coderd when it recovers a panic in an HTTP handler. aibridge loggers echo + # LLM chat content, which can quote Go tracebacks pasted by users, and are + # excluded. + expr: | + sum by (namespace, pod) ( + count_over_time( + { {{- include "coderd-selector" $ -}}, logger!~`coderd\.aibridge.*` } + |~ `(?m)^(panic|fatal error): |panic serving http request` [{{ $group.period }}] + ) + ) > {{ $threshold }} + annotations: + summary: The Coder instance {{ `{{ $labels.pod }}` }} logged a panic in the last {{ $group.period }}. + labels: + severity: {{ $severity }} + runbook_url: {{ template "runbook-url" (deepCopy $ | merge (dict "alert" $alert) $service) }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/coder-observability/templates/configmap-loki-alerts.yaml b/coder-observability/templates/configmap-loki-alerts.yaml new file mode 100644 index 0000000..af36c1d --- /dev/null +++ b/coder-observability/templates/configmap-loki-alerts.yaml @@ -0,0 +1,15 @@ +{{- $coderdLogAlerts := include "coderd-loki-alerts" . -}} +{{- if and .Values.global.alerts.enabled .Values.loki.enabled (trim $coderdLogAlerts) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: coder-logs-alerts + namespace: {{ .Release.Namespace }} + labels: + # the Loki rules sidecar watches for this label and loads the rules into the ruler + loki_rule: "" +data: + coderd.yaml: |- + groups: + {{- $coderdLogAlerts }} +{{- end }} diff --git a/coder-observability/values.yaml b/coder-observability/values.yaml index 0765884..ba25d84 100644 --- a/coder-observability/values.yaml +++ b/coder-observability/values.yaml @@ -86,6 +86,11 @@ global: delay: 10m thresholds: warn: 1 + Panics: + enabled: true + period: 5m + thresholds: + critical: 0 provisionerd: groups: Replicas: diff --git a/compiled/resources.yaml b/compiled/resources.yaml index a687ae5..7a36d7a 100644 --- a/compiled/resources.yaml +++ b/compiled/resources.yaml @@ -787,6 +787,18 @@ metadata: data: config.river: "\n// Discover k8s nodes\ndiscovery.kubernetes \"nodes\" {\n role = \"node\"\n}\n\n// Discover k8s pods\ndiscovery.kubernetes \"pods\" {\n role = \"pod\"\n selectors {\n role = \"pod\"\n }\n}\n\ndiscovery.relabel \"pod_logs\" {\n targets = discovery.kubernetes.pods.targets\n \n rule {\n source_labels = [\"__meta_kubernetes_namespace\"]\n target_label = \"namespace\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_name\"]\n target_label = \"pod\"\n }\n // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n rule {\n source_labels = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n separator = \"/\"\n target_label = \"__meta_app\"\n action = \"replace\"\n regex = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n replacement = \"${1}\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n separator = \"/\"\n target_label = \"job\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_container_name\"]\n target_label = \"container\"\n }\n rule {\n regex = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n action = \"labeldrop\"\n }\n rule {\n regex = \"pod_template_generation\"\n action = \"labeldrop\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_phase\"]\n regex = \"Pending|Succeeded|Failed|Completed\"\n action = \"drop\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_node_name\"]\n action = \"replace\"\n target_label = \"node\"\n }\n rule {\n action = \"labelmap\"\n regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n replacement = \"__param_$1\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_uid\", \"__meta_kubernetes_pod_container_name\"]\n separator = \"/\"\n action = \"replace\"\n replacement = \"/var/log/pods/*$1/*.log\"\n target_label = \"__path__\"\n }\n rule {\n action = \"replace\"\n source_labels = [\"__meta_kubernetes_pod_container_id\"]\n regex = \"^(\\\\w+):\\\\/\\\\/.+$\"\n replacement = \"$1\"\n target_label = \"tmp_container_runtime\"\n }\n}\n\ndiscovery.relabel \"pod_metrics\" {\n targets = discovery.kubernetes.pods.targets\n \n rule {\n source_labels = [\"__meta_kubernetes_namespace\"]\n target_label = \"namespace\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_name\"]\n target_label = \"pod\"\n }\n // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n rule {\n source_labels = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n separator = \"/\"\n target_label = \"__meta_app\"\n action = \"replace\"\n regex = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n replacement = \"${1}\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n separator = \"/\"\n target_label = \"job\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_container_name\"]\n target_label = \"container\"\n }\n rule {\n regex = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n action = \"labeldrop\"\n }\n rule {\n regex = \"pod_template_generation\"\n action = \"labeldrop\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_phase\"]\n regex = \"Pending|Succeeded|Failed|Completed\"\n action = \"drop\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_node_name\"]\n action = \"replace\"\n target_label = \"node\"\n }\n rule {\n action = \"labelmap\"\n regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n replacement = \"__param_$1\"\n }\n // drop ports that do not expose Prometheus metrics, but might otherwise be exposed by a container which *also*\n // exposes an HTTP port which exposes metrics\n rule {\n source_labels = [\"__meta_kubernetes_pod_container_port_name\"]\n regex = \"grpc|http-(memberlist|console)\"\n action = \"drop\"\n }\n // adapted from the Prometheus helm chart\n // https://github.com/prometheus-community/helm-charts/blob/862870fc3c847e32479b509e511584d5283126a3/charts/prometheus/values.yaml#L1070\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scrape\"]\n action = \"keep\"\n regex = \"true\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scheme\"]\n action = \"replace\"\n regex = \"(https?)\"\n target_label = \"__scheme__\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_path\"]\n action = \"replace\"\n target_label = \"__metrics_path__\"\n regex = \"(.+)\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n action = \"replace\"\n regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n replacement = \"[$2]:$1\"\n target_label = \"__address__\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n action = \"replace\"\n regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\"\n replacement = \"$2:$1\"\n target_label = \"__address__\"\n }\n}\n\ndiscovery.relabel \"pod_pprof\" {\n targets = discovery.kubernetes.pods.targets\n \n rule {\n source_labels = [\"__meta_kubernetes_namespace\"]\n target_label = \"namespace\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_name\"]\n target_label = \"pod\"\n }\n // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n rule {\n source_labels = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n separator = \"/\"\n target_label = \"__meta_app\"\n action = \"replace\"\n regex = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n replacement = \"${1}\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n separator = \"/\"\n target_label = \"job\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_container_name\"]\n target_label = \"container\"\n }\n rule {\n regex = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n action = \"labeldrop\"\n }\n rule {\n regex = \"pod_template_generation\"\n action = \"labeldrop\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_phase\"]\n regex = \"Pending|Succeeded|Failed|Completed\"\n action = \"drop\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_node_name\"]\n action = \"replace\"\n target_label = \"node\"\n }\n rule {\n action = \"labelmap\"\n regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n replacement = \"__param_$1\"\n }\n // The relabeling allows the actual pod scrape endpoint to be configured via the\n // following annotations:\n //\n // * `pyroscope.io/scrape`: Only scrape pods that have a value of `true`.\n // * `pyroscope.io/application-name`: Name of the application being profiled.\n // * `pyroscope.io/scheme`: If the metrics endpoint is secured then you will need\n // to set this to `https` & most likely set the `tls_config` of the scrape config.\n // * `pyroscope.io/port`: Scrape the pod on the indicated port.\n //\n // Kubernetes labels will be added as Pyroscope labels on metrics via the\n // `labelmap` relabeling action.\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_scrape\"]\n action = \"keep\"\n regex = \"true\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_application_name\"]\n action = \"replace\"\n target_label = \"__name__\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_scheme\"]\n action = \"replace\"\n regex = \"(https?)\"\n target_label = \"__scheme__\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_port\", \"__meta_kubernetes_pod_ip\"]\n action = \"replace\"\n regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n replacement = \"[$2]:$1\"\n target_label = \"__address__\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_port\", \"__meta_kubernetes_pod_ip\"]\n action = \"replace\"\n regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\"\n replacement = \"$2:$1\"\n target_label = \"__address__\"\n }\n}\n\nlocal.file_match \"pod_logs\" {\n path_targets = discovery.relabel.pod_logs.output\n}\n\nloki.source.file \"pod_logs\" {\n targets = local.file_match.pod_logs.targets\n forward_to = [loki.process.pod_logs.receiver]\n}\n\nloki.process \"pod_logs\" {\n stage.match {\n selector = \"{tmp_container_runtime=\\\"containerd\\\"}\"\n // the cri processing stage extracts the following k/v pairs: log, stream, time, flags\n stage.cri {}\n // Set the extract flags and stream values as labels\n stage.labels {\n values = {\n flags = \"\",\n stream = \"\",\n }\n }\n }\n\n // if the label tmp_container_runtime from above is docker parse using docker\n stage.match {\n selector = \"{tmp_container_runtime=\\\"docker\\\"}\"\n // the docker processing stage extracts the following k/v pairs: log, stream, time\n stage.docker {}\n\n // Set the extract stream value as a label\n stage.labels {\n values = {\n stream = \"\",\n }\n }\n }\n\n // drop the temporary container runtime label as it is no longer needed\n stage.label_drop {\n values = [\"tmp_container_runtime\"]\n }\n\n // parse Coder logs and extract level & logger for efficient filtering\n stage.match {\n selector = \"{pod=~\\\"coder.*\\\"}\" // TODO: make configurable\n\n stage.multiline {\n firstline = \"^(?P\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\"\n max_wait_time = \"10s\"\n }\n\n stage.regex {\n expression = \"^(?P\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\\\\s\\\\[(?P\\\\w+)\\\\]\\\\s\\\\s(?P[^:]+):\\\\s(?P.+)\"\n }\n\n stage.timestamp {\n source = \"ts\"\n format = \"2006-01-02 15:04:05.000\"\n action_on_failure = \"fudge\" // rather have inaccurate time than drop the log line\n }\n\n stage.labels {\n values = {\n level = \"\",\n logger = \"\",\n }\n }\n }\n\n forward_to = [loki.write.loki.receiver]\n}\n\nloki.write \"loki\" {\n endpoint {\n url = \"http://loki-gateway.coder-observability.svc/loki/api/v1/push\"\n }\n}\n\n\n\nprometheus.scrape \"pods\" {\n targets = discovery.relabel.pod_metrics.output\n forward_to = [prometheus.relabel.pods.receiver]\n\n scrape_interval = \"15s\"\n scrape_timeout = \"12s\"\n enable_protobuf_negotiation = false\n}\n\n// These are metric_relabel_configs while discovery.relabel are relabel_configs.\n// See https://github.com/grafana/agent/blob/main/internal/converter/internal/prometheusconvert/prometheusconvert.go#L95-L106\nprometheus.relabel \"pods\" {\n forward_to = [prometheus.remote_write.default.receiver]\n\n // Drop kube-state-metrics' labels which clash with ours\n rule {\n source_labels = [\"__name__\", \"container\"]\n regex = \"kube_pod.+;(.+)\"\n target_label = \"container\"\n replacement = \"\"\n }\n rule {\n source_labels = [\"__name__\", \"pod\"]\n regex = \"kube_pod.+;(.+)\"\n target_label = \"pod\"\n replacement = \"\"\n }\n rule {\n source_labels = [\"__name__\", \"namespace\"]\n regex = \"kube_pod.+;(.+)\"\n target_label = \"namespace\"\n replacement = \"\"\n }\n rule {\n source_labels = [\"__name__\", \"exported_container\"]\n // don't replace an empty label\n regex = \"^kube_pod.+;(.+)$\"\n target_label = \"container\"\n replacement = \"$1\"\n }\n rule {\n source_labels = [\"__name__\", \"exported_pod\"]\n // don't replace an empty label\n regex = \"^kube_pod.+;(.+)$\"\n target_label = \"pod\"\n replacement = \"$1\"\n }\n rule {\n source_labels = [\"__name__\", \"exported_namespace\"]\n // don't replace an empty label\n regex = \"^kube_pod.+;(.+)$\"\n target_label = \"namespace\"\n replacement = \"$1\"\n }\n rule {\n regex = \"^(exported_.*|image_.*|container_id|id|uid)$\"\n action = \"labeldrop\"\n }\n}\n\ndiscovery.relabel \"cadvisor\" {\n targets = discovery.kubernetes.nodes.targets\n rule {\n replacement = \"/metrics/cadvisor\"\n target_label = \"__metrics_path__\"\n }\n}\n\nprometheus.scrape \"cadvisor\" {\n targets = discovery.relabel.cadvisor.output\n forward_to = [ prometheus.relabel.cadvisor.receiver ]\n scheme = \"https\"\n tls_config {\n insecure_skip_verify = true\n }\n bearer_token_file = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n scrape_interval = \"15s\"\n scrape_timeout = \"12s\"\n enable_protobuf_negotiation = false\n}\n\nprometheus.relabel \"cadvisor\" {\n forward_to = [ prometheus.remote_write.default.receiver ]\n\n // Drop empty container labels, addressing https://github.com/google/cadvisor/issues/2688\n rule {\n source_labels = [\"__name__\",\"container\"]\n separator = \"@\"\n regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*)@\"\n action = \"drop\"\n }\n // Drop empty image labels, addressing https://github.com/google/cadvisor/issues/2688\n rule {\n source_labels = [\"__name__\",\"image\"]\n separator = \"@\"\n regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*|container_network_.*)@\"\n action = \"drop\"\n }\n // Drop irrelevant series\n rule {\n source_labels = [\"container\"]\n regex = \"^POD$\"\n action = \"drop\"\n }\n // Drop unnecessary labels\n rule {\n source_labels = [\"id\"]\n target_label = \"id\"\n replacement = \"\"\n }\n rule {\n source_labels = [\"job\"]\n target_label = \"job\"\n replacement = \"\"\n }\n rule {\n source_labels = [\"name\"]\n target_label = \"name\"\n replacement = \"\"\n }\n}\n\nprometheus.remote_write \"default\" {\n endpoint {\n send_native_histograms = false\n url =\"http://prometheus.coder-observability.svc/api/v1/write\"\n\n // drop instance label which unnecessarily adds new series when pods are restarted, since pod IPs are dynamically assigned\n // NOTE: \"__address__\" is mapped to \"instance\", so will contain :\n write_relabel_config {\n regex = \"instance\"\n action = \"labeldrop\"\n }\n }\n}" --- +# Source: coder-observability/templates/configmap-loki-alerts.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: coder-logs-alerts + namespace: coder-observability + labels: + # the Loki rules sidecar watches for this label and loads the rules into the ruler + loki_rule: "" +data: + coderd.yaml: "groups:\n- name: Panics\n rules:\n \n - alert: CoderdPanic\n # Panic and fatal-error output is untimestamped, so the collector's multiline\n # stage appends it to the preceding timestamped log entry; (?m) lets ^ match\n # the start of the embedded panic line. \"panic serving http request\" is logged\n # by coderd when it recovers a panic in an HTTP handler. aibridge loggers echo\n # LLM chat content, which can quote Go tracebacks pasted by users, and are\n # excluded.\n expr: |\n sum by (namespace, pod) (\n count_over_time(\n {pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, logger!~`coderd\\.aibridge.*` }\n |~ `(?m)^(panic|fatal error): |panic serving http request` [5m]\n )\n ) > 0\n annotations:\n summary: The Coder instance {{ $labels.pod }} logged a panic in the last 5m.\n labels:\n severity: critical\n runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdpanic" +--- # Source: coder-observability/templates/configmap-prometheus-alerts.yaml apiVersion: v1 kind: ConfigMap @@ -806,9 +818,9 @@ metadata: name: runbooks namespace: coder-observability annotations: - checksum/config: b0c41033d0385ee3d46488f08e85bcef0d939614dcb99194e0c5913dbf0c2c33 + checksum/config: e10d5e56434433e2ec8a267861cd982124b01a2faabb9f0c0a1a6f64d7733645 data: - coderd.md: |- + coderd.md: | # Coderd Runbooks ## CoderdCPUUsage @@ -944,6 +956,22 @@ data: in this _reconciliation loop_, which should be reported to Coder. Examine your coderd logs for any errors or warnings relating to prebuilds. + ## CoderdPanic + + A Coder pod logged a panic or fatal runtime error, or recovered from a panic in + an HTTP handler. Panics indicate a bug in Coder. + + Find the traceback in the affected pod's logs: + + ```console + kubectl -n coder logs | grep -B 1 -A 30 -E '^(panic|fatal error): |panic serving http request' + ``` + + An unrecovered panic also restarts the pod, so check `--previous` logs if the + current logs contain no traceback. + + Report the traceback to [coder/coder](https://github.com/coder/coder/issues) if + an issue does not already exist for it. postgres.md: | # Postgres Runbooks @@ -13575,7 +13603,7 @@ spec: template: metadata: annotations: - checksum/config: b0c41033d0385ee3d46488f08e85bcef0d939614dcb99194e0c5913dbf0c2c33 + checksum/config: e10d5e56434433e2ec8a267861cd982124b01a2faabb9f0c0a1a6f64d7733645 labels: app: runbook-viewer spec: