feat: add log-based CoderdPanic alert via the Loki ruler

README.md

@@ -313,7 +313,7 @@ values which are defined [here](https://github.com/grafana/helm-charts/tree/main
 |-----|------|---------|-------------|
 | global.alerts.enabled | bool | `true` | enable or disable alerting |
 | global.alerts.kind | string | `"configmap"` | the container resource kind in which alerts should be created; valid values are "prometheusrule" or "configmap" |
-| global.coder.alerts | object | `{"coderd":{"groups":{"CPU":{"delay":"10m","enabled":true,"period":"10m","thresholds":{"critical":0.9,"warning":0.8}},"IneligiblePrebuilds":{"delay":"10m","enabled":true,"thresholds":{"notify":1}},"Memory":{"delay":"10m","enabled":true,"thresholds":{"critical":0.9,"warning":0.8}},"Replicas":{"delay":"5m","enabled":true,"thresholds":{"critical":1,"notify":3,"warning":2}},"Restarts":{"delay":"1m","enabled":true,"period":"10m","thresholds":{"critical":3,"notify":1,"warning":2}},"UnprovisionedPrebuiltWorkspaces":{"delay":"10m","enabled":true,"thresholds":{"warn":1}},"WorkspaceBuildFailures":{"delay":"10m","enabled":true,"period":"10m","thresholds":{"critical":10,"notify":2,"warning":5}}}},"enterprise":{"groups":{"Licences":{"delay":"1m","enabled":true,"thresholds":{"critical":1,"warning":0.9}}}},"provisionerd":{"groups":{"Replicas":{"delay":"5m","enabled":true,"thresholds":{"critical":1,"notify":3,"warning":2}}}}}` | alerts for the various aspects of Coder |
+| global.coder.alerts | object | `{"coderd":{"groups":{"CPU":{"delay":"10m","enabled":true,"period":"10m","thresholds":{"critical":0.9,"warning":0.8}},"IneligiblePrebuilds":{"delay":"10m","enabled":true,"thresholds":{"notify":1}},"Memory":{"delay":"10m","enabled":true,"thresholds":{"critical":0.9,"warning":0.8}},"Panics":{"enabled":true,"period":"5m","thresholds":{"critical":0}},"Replicas":{"delay":"5m","enabled":true,"thresholds":{"critical":1,"notify":3,"warning":2}},"Restarts":{"delay":"1m","enabled":true,"period":"10m","thresholds":{"critical":3,"notify":1,"warning":2}},"UnprovisionedPrebuiltWorkspaces":{"delay":"10m","enabled":true,"thresholds":{"warn":1}},"WorkspaceBuildFailures":{"delay":"10m","enabled":true,"period":"10m","thresholds":{"critical":10,"notify":2,"warning":5}}}},"enterprise":{"groups":{"Licences":{"delay":"1m","enabled":true,"thresholds":{"critical":1,"warning":0.9}}}},"provisionerd":{"groups":{"Replicas":{"delay":"5m","enabled":true,"thresholds":{"critical":1,"notify":3,"warning":2}}}}}` | alerts for the various aspects of Coder |
 | global.coder.coderdSelector | string | `"pod=~`coder.*`, pod!~`.*provisioner.*`"` | series selector for Prometheus/Loki to locate provisioner pods. ensure this uses backticks for quotes! |
 | global.coder.controlPlaneNamespace | string | `"coder"` | the namespace into which the control plane has been deployed. |
 | global.coder.externalProvisionersNamespace | string | `"coder"` | the namespace into which any external provisioners have been deployed. |

coder-observability/runbooks/coderd.md

@@ -132,4 +132,20 @@ The prebuilds subsystem runs a _reconciliation loop_ which monitors the state of
 number of instances are present at all times. Workspace Prebuilds is currently a BETA feature and so there could be a bug
 in this _reconciliation loop_, which should be reported to Coder.
 
-Examine your coderd logs for any errors or warnings relating to prebuilds.
+Examine your coderd logs for any errors or warnings relating to prebuilds.
+## CoderdPanic
+
+A Coder pod logged a panic or fatal runtime error, or recovered from a panic in
+an HTTP handler. Panics indicate a bug in Coder.
+
+Find the traceback in the affected pod's logs:
+
+```console
+kubectl -n coder logs <pod> | grep -B 1 -A 30 -E '^(panic|fatal error): |panic serving http request'
+```
+
+An unrecovered panic also restarts the pod, so check `--previous` logs if the
+current logs contain no traceback.
+
+Report the traceback to [coder/coder](https://github.com/coder/coder/issues) if
+an issue does not already exist for it.

coder-observability/templates/_loki-alerts.tpl

@@ -0,0 +1,38 @@
+{{/*
+Log-based alerting rules, evaluated by the Loki ruler (LogQL, not PromQL).
+See https://grafana.com/docs/loki/latest/alert/
+*/}}
+{{- define "coderd-loki-alerts" -}}
+  {{- $service := dict "service" "coderd" -}}
+  {{- with .Values.global.coder.alerts.coderd }}
+  {{- with .groups.Panics }}
+  {{- $group := . }}
+  {{- if .enabled }}
+    - name: Panics
+      rules:
+      {{ $alert := "CoderdPanic" }}
+      {{- range $severity, $threshold := .thresholds }}
+      - alert: {{ $alert }}
+        # Panic and fatal-error output is untimestamped, so the collector's multiline
+        # stage appends it to the preceding timestamped log entry; (?m) lets ^ match
+        # the start of the embedded panic line. "panic serving http request" is logged
+        # by coderd when it recovers a panic in an HTTP handler. aibridge loggers echo
+        # LLM chat content, which can quote Go tracebacks pasted by users, and are
+        # excluded.
+        expr: |
+          sum by (namespace, pod) (
+            count_over_time(
+              { {{- include "coderd-selector" $ -}}, logger!~`coderd\.aibridge.*` }
+                |~ `(?m)^(panic|fatal error): |panic serving http request` [{{ $group.period }}]
+            )
+          ) > {{ $threshold }}
+        annotations:
+          summary: The Coder instance {{ `{{ $labels.pod }}` }} logged a panic in the last {{ $group.period }}.
+        labels:
+          severity: {{ $severity }}
+          runbook_url: {{ template "runbook-url" (deepCopy $ | merge (dict "alert" $alert) $service) }}
+      {{- end }}
+  {{- end }}
+  {{- end }}
+  {{- end }}
+{{- end }}

coder-observability/templates/configmap-loki-alerts.yaml

@@ -0,0 +1,15 @@
+{{- $coderdLogAlerts := include "coderd-loki-alerts" . -}}
+{{- if and .Values.global.alerts.enabled .Values.loki.enabled (trim $coderdLogAlerts) -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coder-logs-alerts
+  namespace: {{ .Release.Namespace }}
+  labels:
+    # the Loki rules sidecar watches for this label and loads the rules into the ruler
+    loki_rule: ""
+data:
+  coderd.yaml: |-
+    groups:
+    {{- $coderdLogAlerts }}
+{{- end }}

coder-observability/values.yaml

@@ -86,6 +86,11 @@ global:
             delay: 10m
             thresholds:
               warn: 1
+          Panics:
+            enabled: true
+            period: 5m
+            thresholds:
+              critical: 0
       provisionerd:
         groups:
           Replicas: