From def297686f8625cc2652967a0f1d959b9765aacd Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Tue, 1 Sep 2026 23:40:48 +0000 Subject: [PATCH 1/9] Basic setup --- .../gke-workers/base/core/workers.yaml | 3 + .../gke-workers/base/extra/workers-cves.yaml | 3 + .../base/extra/workers-reimport.yaml | 3 + .../base/feeds/data-quality-dashboard.yaml | 44 + .../oss-vdb-test/kustomization.yaml | 1 + .../dashboards/executive-overview.json | 794 +++++++++++++++++ .../monitoring/grafana_dashboards.yaml | 819 ++++++++++++++++++ .../monitoring/grafana_dashboards_prov.yaml | 19 + .../monitoring/grafana_datasources.yaml | 27 + .../monitoring/grafana_deployment.yaml | 94 ++ .../monitoring/grafana_ingress.yaml | 46 + .../monitoring/grafana_service.yaml | 19 + .../monitoring/grafana_service_account.yaml | 9 + .../monitoring/kustomization.yaml | 9 + .../monitoring/worker_pod_monitoring.yaml | 19 + go/cmd/worker/main.go | 5 + go/go.mod | 8 +- go/go.sum | 14 + go/internal/metrics/metrics.go | 97 +++ go/internal/metrics/metrics_test.go | 76 ++ go/internal/metrics/server.go | 76 ++ go/internal/worker/subscriber.go | 13 + 22 files changed, 2197 insertions(+), 1 deletion(-) create mode 100644 deployment/clouddeploy/gke-workers/base/feeds/data-quality-dashboard.yaml create mode 100644 deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json create mode 100644 deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards.yaml create mode 100644 deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards_prov.yaml create mode 100644 deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_datasources.yaml create mode 100644 deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_deployment.yaml create mode 100644 deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml create mode 100644 deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service.yaml create mode 100644 deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service_account.yaml create mode 100644 deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml create mode 100644 deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/worker_pod_monitoring.yaml create mode 100644 go/internal/metrics/metrics.go create mode 100644 go/internal/metrics/metrics_test.go create mode 100644 go/internal/metrics/server.go diff --git a/deployment/clouddeploy/gke-workers/base/core/workers.yaml b/deployment/clouddeploy/gke-workers/base/core/workers.yaml index c38119dce61..4f90ab89d6d 100644 --- a/deployment/clouddeploy/gke-workers/base/core/workers.yaml +++ b/deployment/clouddeploy/gke-workers/base/core/workers.yaml @@ -41,6 +41,9 @@ spec: value: failed-tasks - name: NOTIFY_PYPI value: "false" + ports: + - name: metrics + containerPort: 9090 resources: requests: cpu: 1 diff --git a/deployment/clouddeploy/gke-workers/base/extra/workers-cves.yaml b/deployment/clouddeploy/gke-workers/base/extra/workers-cves.yaml index 9bba470bd7a..44699b0612d 100644 --- a/deployment/clouddeploy/gke-workers/base/extra/workers-cves.yaml +++ b/deployment/clouddeploy/gke-workers/base/extra/workers-cves.yaml @@ -75,6 +75,9 @@ spec: value: failed-tasks - name: NOTIFY_PYPI value: "false" + ports: + - name: metrics + containerPort: 9090 resources: requests: cpu: "0.9" diff --git a/deployment/clouddeploy/gke-workers/base/extra/workers-reimport.yaml b/deployment/clouddeploy/gke-workers/base/extra/workers-reimport.yaml index 765566c9bde..181e40b106a 100644 --- a/deployment/clouddeploy/gke-workers/base/extra/workers-reimport.yaml +++ b/deployment/clouddeploy/gke-workers/base/extra/workers-reimport.yaml @@ -75,6 +75,9 @@ spec: value: failed-tasks - name: NOTIFY_PYPI value: "false" + ports: + - name: metrics + containerPort: 9090 resources: requests: cpu: "0.9" diff --git a/deployment/clouddeploy/gke-workers/base/feeds/data-quality-dashboard.yaml b/deployment/clouddeploy/gke-workers/base/feeds/data-quality-dashboard.yaml new file mode 100644 index 00000000000..9f16d77cb36 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/base/feeds/data-quality-dashboard.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: data-quality-dashboard +spec: + replicas: 1 + selector: + matchLabels: + app: data-quality-dashboard + template: + metadata: + labels: + app: data-quality-dashboard + spec: + containers: + - name: dashboard + image: data-quality-dashboard + imagePullPolicy: Always + ports: + - containerPort: 8080 + env: + - name: GCS_BUCKET + value: osv-test-cve-osv-conversion + - name: PORT + value: "8080" + resources: + requests: + cpu: "250m" + memory: "512Mi" + limits: + cpu: "1" + memory: "2Gi" +--- +apiVersion: v1 +kind: Service +metadata: + name: data-quality-dashboard +spec: + selector: + app: data-quality-dashboard + ports: + - protocol: TCP + port: 80 + targetPort: 8080 diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml index 7ec61ca37bb..50f02560ae8 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml @@ -2,6 +2,7 @@ resources: - ../../base - staging-api-test.yaml - osv-linter.yaml +- monitoring patches: - path: workers.yaml - path: scaler.yaml diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json new file mode 100644 index 00000000000..80419fbdab3 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json @@ -0,0 +1,794 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "doc", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "On-Call Triage Runbook", + "tooltip": "Open OSV On-Call Runbook", + "type": "link", + "url": "https://github.com/google/osv.dev/blob/master/docs/observability.md" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "🚀 Executive Service Level Objectives (SLOs) & Overall Health", + "type": "row" + }, + { + "datasource": { + "type": "googlecloud-monitoring-datasource", + "uid": "cloudmonitoring" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 95, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 99.5 + }, + { + "color": "green", + "value": 99.95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "11.5.0", + "title": "API Availability (SLO > 99.95%)", + "type": "gauge" + }, + { + "datasource": { + "type": "googlecloud-monitoring-datasource", + "uid": "cloudmonitoring" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 150 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.5.0", + "title": "API p99 Latency (SLO < 150ms)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3600 + }, + { + "color": "red", + "value": 7200 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(osv_pipeline_published_to_available_seconds_bucket[1h])) by (le))", + "legendFormat": "p95 Lag", + "range": true, + "refId": "A" + } + ], + "title": "E2E Pipeline Lag (p95, SLO < 2h)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "sum(osv_dataset_total_vulnerabilities)", + "legendFormat": "Total Vulns", + "range": true, + "refId": "A" + } + ], + "title": "Total Dataset Size", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 101, + "title": "🌐 Cloud Run Public API Serving Performance", + "type": "row" + }, + { + "datasource": { + "type": "googlecloud-monitoring-datasource", + "uid": "cloudmonitoring" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["mean", "max", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "title": "API Request Rate by HTTP Response Status Code", + "type": "timeseries" + }, + { + "datasource": { + "type": "googlecloud-monitoring-datasource", + "uid": "cloudmonitoring" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["mean", "max", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "title": "API Latency Percentiles (p50 / p95 / p99)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 102, + "title": "⚡ Ingestion Pipeline Throughput & Latency Lag", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 7, + "options": { + "legend": { + "calcs": ["mean", "max", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "sum(rate(osv_worker_tasks_processed_total[5m])) by (status, source)", + "legendFormat": "{{source}} ({{status}})", + "range": true, + "refId": "A" + } + ], + "title": "Worker Task Processing Rate (Velocity / Drain)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 7200 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 8, + "options": { + "legend": { + "calcs": ["mean", "max", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(osv_pipeline_published_to_available_seconds_bucket[15m])) by (le, source))", + "legendFormat": "p95 {{source}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(osv_pipeline_published_to_available_seconds_bucket[15m])) by (le, source))", + "legendFormat": "p50 {{source}}", + "range": true, + "refId": "B" + } + ], + "title": "E2E Published-to-Available Lag by Source", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 103, + "title": "🛡️ Subsystem Health & Multi-Window Error Budget Burn Rates", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "sum(rate(osv_worker_tasks_processed_total{status=\"error\"}[1h])) / (sum(rate(osv_worker_tasks_processed_total[1h])) > 0 or 1)", + "legendFormat": "1h Burn Rate", + "range": true, + "refId": "A" + } + ], + "title": "Worker Ingestion 1h Error Burn Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 24 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "sum(rate(osv_worker_tasks_processed_total{status=\"error\"}[6h])) / (sum(rate(osv_worker_tasks_processed_total[6h])) > 0 or 1)", + "legendFormat": "6h Burn Rate", + "range": true, + "refId": "A" + } + ], + "title": "Worker Ingestion 6h Error Burn Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 24 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "sum(rate(osv_worker_tasks_processed_total{status=\"error\"}[24h])) / (sum(rate(osv_worker_tasks_processed_total[24h])) > 0 or 1)", + "legendFormat": "24h Burn Rate", + "range": true, + "refId": "A" + } + ], + "title": "Worker Ingestion 24h Error Burn Rate", + "type": "stat" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "osv", + "executive", + "sla", + "overview", + "gmp" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "utc", + "title": "Executive Overview & SLAs", + "uid": "osv-executive-overview", + "version": 1 +} diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards.yaml new file mode 100644 index 00000000000..4494dd64f35 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards.yaml @@ -0,0 +1,819 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring +data: + executive-overview.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "doc", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "On-Call Triage Runbook", + "tooltip": "Open OSV On-Call Runbook", + "type": "link", + "url": "https://github.com/google/osv.dev/blob/master/docs/observability.md" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "🚀 Executive Service Level Objectives (SLOs) & Overall Health", + "type": "row" + }, + { + "datasource": { + "type": "googlecloud-monitoring-datasource", + "uid": "cloudmonitoring" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 95, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 99.5 + }, + { + "color": "green", + "value": 99.95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "11.5.0", + "title": "API Availability (SLO > 99.95%)", + "type": "gauge" + }, + { + "datasource": { + "type": "googlecloud-monitoring-datasource", + "uid": "cloudmonitoring" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 150 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.5.0", + "title": "API p99 Latency (SLO < 150ms)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3600 + }, + { + "color": "red", + "value": 7200 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(osv_pipeline_published_to_available_seconds_bucket[1h])) by (le))", + "legendFormat": "p95 Lag", + "range": true, + "refId": "A" + } + ], + "title": "E2E Pipeline Lag (p95, SLO < 2h)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "sum(osv_dataset_total_vulnerabilities)", + "legendFormat": "Total Vulns", + "range": true, + "refId": "A" + } + ], + "title": "Total Dataset Size", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 101, + "title": "🌐 Cloud Run Public API Serving Performance", + "type": "row" + }, + { + "datasource": { + "type": "googlecloud-monitoring-datasource", + "uid": "cloudmonitoring" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "title": "API Request Rate by HTTP Response Status Code", + "type": "timeseries" + }, + { + "datasource": { + "type": "googlecloud-monitoring-datasource", + "uid": "cloudmonitoring" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "title": "API Latency Percentiles (p50 / p95 / p99)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 102, + "title": "⚡ Ingestion Pipeline Throughput & Latency Lag", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "sum(rate(osv_worker_tasks_processed_total[5m])) by (status, source)", + "legendFormat": "{{source}} ({{status}})", + "range": true, + "refId": "A" + } + ], + "title": "Worker Task Processing Rate (Velocity / Drain)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 7200 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(osv_pipeline_published_to_available_seconds_bucket[15m])) by (le, source))", + "legendFormat": "p95 {{source}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(osv_pipeline_published_to_available_seconds_bucket[15m])) by (le, source))", + "legendFormat": "p50 {{source}}", + "range": true, + "refId": "B" + } + ], + "title": "E2E Published-to-Available Lag by Source", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 103, + "title": "🛡️ Subsystem Health & Multi-Window Error Budget Burn Rates", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "sum(rate(osv_worker_tasks_processed_total{status=\"error\"}[1h])) / (sum(rate(osv_worker_tasks_processed_total[1h])) > 0 or 1)", + "legendFormat": "1h Burn Rate", + "range": true, + "refId": "A" + } + ], + "title": "Worker Ingestion 1h Error Burn Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 24 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "sum(rate(osv_worker_tasks_processed_total{status=\"error\"}[6h])) / (sum(rate(osv_worker_tasks_processed_total[6h])) > 0 or 1)", + "legendFormat": "6h Burn Rate", + "range": true, + "refId": "A" + } + ], + "title": "Worker Ingestion 6h Error Burn Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 24 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "sum(rate(osv_worker_tasks_processed_total{status=\"error\"}[24h])) / (sum(rate(osv_worker_tasks_processed_total[24h])) > 0 or 1)", + "legendFormat": "24h Burn Rate", + "range": true, + "refId": "A" + } + ], + "title": "Worker Ingestion 24h Error Burn Rate", + "type": "stat" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "osv", + "executive", + "sla", + "overview", + "gmp" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "utc", + "title": "Executive Overview & SLAs", + "uid": "osv-executive-overview", + "version": 1 + } diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards_prov.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards_prov.yaml new file mode 100644 index 00000000000..d62e3307d67 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards_prov.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-prov + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring +data: + dashboards.yaml: | + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: false + options: + path: /etc/grafana/dashboards diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_datasources.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_datasources.yaml new file mode 100644 index 00000000000..031a613eabf --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_datasources.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring +data: + datasources.yaml: | + apiVersion: 1 + datasources: + - name: GMP (Prometheus) + type: prometheus + access: proxy + url: http://127.0.0.1:9090 + isDefault: true + editable: false + jsonData: + httpMethod: POST + timeInterval: 15s + - name: Cloud Monitoring + type: googlecloud-monitoring-datasource + access: proxy + editable: false + jsonData: + authenticationType: gce + defaultProject: oss-vdb-test diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_deployment.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_deployment.yaml new file mode 100644 index 00000000000..6464a5cfccc --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_deployment.yaml @@ -0,0 +1,94 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: grafana + template: + metadata: + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring + spec: + serviceAccountName: grafana-sa + containers: + - name: grafana + image: grafana/grafana:11.5.0@sha256:4a0808cf7ca4b2569e5d4cb05ef2222a27ffb2a0c6499318b76c8cb81a6c4b28 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 3000 + env: + - name: GF_SECURITY_ALLOW_EMBEDDING + value: "true" + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "false" + - name: GF_AUTH_BASIC_ENABLED + value: "false" + - name: GF_AUTH_PROXY_ENABLED + value: "true" + - name: GF_AUTH_PROXY_HEADER_NAME + value: "X-Goog-Authenticated-User-Email" + - name: GF_AUTH_PROXY_HEADER_PROPERTY + value: "email" + - name: GF_AUTH_PROXY_AUTO_SIGN_UP + value: "true" + - name: GF_SERVER_ROOT_URL + value: "https://monitoring.osv.dev" + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + volumeMounts: + - name: datasources + mountPath: /etc/grafana/provisioning/datasources + readOnly: true + - name: dashboards-prov + mountPath: /etc/grafana/provisioning/dashboards + readOnly: true + - name: dashboards + mountPath: /etc/grafana/dashboards + readOnly: true + - name: storage + mountPath: /var/lib/grafana + - name: gmp-frontend + image: gke.gcr.io/prometheus-engine/frontend:v0.18.2-gke.1@sha256:35ab356d969255dd7694ae371bf0991b97907451b4c7e1574b58a096d3be5977 + args: + - "--web.listen-address=127.0.0.1:9090" + - "--project-id=$(GOOGLE_CLOUD_PROJECT)" + env: + - name: GOOGLE_CLOUD_PROJECT + value: oss-vdb-test + ports: + - name: gmp-proxy + containerPort: 9090 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + volumes: + - name: datasources + configMap: + name: grafana-datasources + - name: dashboards-prov + configMap: + name: grafana-dashboards-prov + - name: dashboards + configMap: + name: grafana-dashboards + - name: storage + emptyDir: {} diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml new file mode 100644 index 00000000000..179306a8a49 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml @@ -0,0 +1,46 @@ +apiVersion: cloud.google.com/v1 +kind: BackendConfig +metadata: + name: grafana-backend-config + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring +spec: + iap: + enabled: true + oauthclientCredentials: + secretName: grafana-iap-oauth +--- +apiVersion: networking.gke.io/v1 +kind: ManagedCertificate +metadata: + name: grafana-managed-cert + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring +spec: + domains: + - monitoring.osv.dev +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana-ingress + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring + annotations: + kubernetes.io/ingress.class: "gce" + networking.gke.io/managed-certificates: grafana-managed-cert +spec: + rules: + - host: monitoring.osv.dev + http: + paths: + - path: /* + pathType: ImplementationSpecific + backend: + service: + name: grafana + port: + number: 3000 diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service.yaml new file mode 100644 index 00000000000..dd917314dfc --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: grafana + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring + annotations: + cloud.google.com/backend-config: '{"default": "grafana-backend-config"}' + cloud.google.com/neg: '{"ingress": true}' +spec: + type: ClusterIP + ports: + - name: http + port: 3000 + targetPort: 3000 + protocol: TCP + selector: + app.kubernetes.io/name: grafana diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service_account.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service_account.yaml new file mode 100644 index 00000000000..c36395c1421 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service_account.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: grafana-sa + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring + annotations: + iam.gke.io/gcp-service-account: osv-monitoring-sa@oss-vdb-test.iam.gserviceaccount.com diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml new file mode 100644 index 00000000000..8a6cb3a8dd7 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml @@ -0,0 +1,9 @@ +resources: +- worker_pod_monitoring.yaml +- grafana_service_account.yaml +- grafana_datasources.yaml +- grafana_dashboards_prov.yaml +- grafana_dashboards.yaml +- grafana_deployment.yaml +- grafana_service.yaml +- grafana_ingress.yaml diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/worker_pod_monitoring.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/worker_pod_monitoring.yaml new file mode 100644 index 00000000000..54947f9e31d --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/worker_pod_monitoring.yaml @@ -0,0 +1,19 @@ +apiVersion: monitoring.googleapis.com/v1 +kind: PodMonitoring +metadata: + name: worker-pod-monitoring + labels: + app.kubernetes.io/name: osv-worker + app.kubernetes.io/part-of: google-cloud-managed-prometheus +spec: + selector: + matchExpressions: + - key: name + operator: In + values: + - worker-private + - workers-reimport + - workers-cves + endpoints: + - port: metrics + interval: 15s diff --git a/go/cmd/worker/main.go b/go/cmd/worker/main.go index 2b9bbaf7ebb..4bf8f35ca04 100644 --- a/go/cmd/worker/main.go +++ b/go/cmd/worker/main.go @@ -17,6 +17,7 @@ import ( "cloud.google.com/go/pubsub/v2" "cloud.google.com/go/storage" db "github.com/google/osv.dev/go/internal/database/datastore" + "github.com/google/osv.dev/go/internal/metrics" "github.com/google/osv.dev/go/internal/worker" "github.com/google/osv.dev/go/internal/worker/pipeline/registry" "github.com/google/osv.dev/go/logger" @@ -59,6 +60,10 @@ func run() error { vulnBucket := envOrDefault("OSV_VULNERABILITIES_BUCKET", "osv-test-vulnerabilities") failTasksTopic := envOrDefault("FAILED_TASKS_TOPIC", "failed-tasks") notifyPyPI, _ := strconv.ParseBool(envOrDefault("NOTIFY_PYPI", "false")) // returns false on error + metricsPort := envOrDefault("METRICS_PORT", "9090") + + // Start Prometheus metrics server for GMP scraping + metrics.StartMetricsServer(ctx, metricsPort) // Plug in all the connections to the engine dsClient, err := datastore.NewClientWithDatabase(ctx, project, datastoreID) diff --git a/go/go.mod b/go/go.mod index fcd7d234234..ef294733d11 100644 --- a/go/go.mod +++ b/go/go.mod @@ -25,6 +25,7 @@ require ( github.com/ossf/osv-schema/bindings/go v0.0.0-20260806060209-f3f826310aec github.com/package-url/packageurl-go v0.1.6 github.com/pandatix/go-cvss v0.6.2 + github.com/prometheus/client_golang v1.23.2 github.com/redis/go-redis/v9 v9.22.0 github.com/tidwall/gjson v1.19.0 github.com/xeipuuv/gojsonschema v1.2.0 @@ -46,6 +47,7 @@ require ( k8s.io/apimachinery v0.36.3 mvdan.cc/sh/v3 v3.13.1 osv.dev/bindings/go v0.0.0-20260527044221-c1c72c1b0158 + sigs.k8s.io/yaml v1.6.0 ) require ( @@ -61,6 +63,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.4.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect github.com/charmbracelet/ultraviolet v0.0.0-20251205161215-1948445e3318 // indirect @@ -95,9 +98,13 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/muesli/cancelreader v0.2.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sergi/go-diff v1.4.0 // indirect github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect @@ -123,5 +130,4 @@ require ( google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go/go.sum b/go/go.sum index 4a67017042e..3ff99efaae3 100644 --- a/go/go.sum +++ b/go/go.sum @@ -52,6 +52,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -193,6 +195,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -211,6 +215,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nikolalohinski/gonja/v2 v2.9.0 h1:QICtNWj0siM3PF5xUjVod/l+C9XnGfI+wrfSIBGKQ6o= github.com/nikolalohinski/gonja/v2 v2.9.0/go.mod h1:UIzXPVuOsr5h7dZ5DUbqk3/Z7oFA/NLGQGMjqT4L2aU= github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= @@ -232,7 +238,15 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= diff --git a/go/internal/metrics/metrics.go b/go/internal/metrics/metrics.go new file mode 100644 index 00000000000..5956dc93e72 --- /dev/null +++ b/go/internal/metrics/metrics.go @@ -0,0 +1,97 @@ +// Package metrics provides Prometheus metric definitions and exposition for OSV services. +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + // PipelinePublishedToAvailableSeconds tracks the end-to-end latency between + // the upstream publish/source timestamp and when the vulnerability is saved in the OSV database. + PipelinePublishedToAvailableSeconds = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "osv_pipeline_published_to_available_seconds", + Help: "Seconds between upstream vulnerability publish/source timestamp and OSV database write.", + Buckets: []float64{ + 60, // 1 min + 300, // 5 min + 600, // 10 min + 1800, // 30 min + 3600, // 1 hour + 7200, // 2 hours (SLO threshold) + 14400, // 4 hours + 28800, // 8 hours + 86400, // 24 hours + 172800, // 48 hours + 604800, // 7 days + }, + }, + []string{"source"}, + ) + + // WorkerTasksProcessedTotal tracks the total number of ingestion/enrichment tasks processed by the worker. + WorkerTasksProcessedTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "osv_worker_tasks_processed_total", + Help: "Total number of tasks processed by the worker.", + }, + []string{"status", "source"}, + ) + + // WorkerTaskDurationSeconds tracks the duration taken by workers to process individual tasks. + WorkerTaskDurationSeconds = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "osv_worker_task_duration_seconds", + Help: "Duration in seconds to process a worker task.", + Buckets: []float64{ + 0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, + }, + }, + []string{"status", "source"}, + ) + + // DatasetTotalVulnerabilities tracks the total number of vulnerabilities in the OSV dataset. + DatasetTotalVulnerabilities = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "osv_dataset_total_vulnerabilities", + Help: "Total number of vulnerabilities in the OSV dataset.", + }, + []string{"ecosystem", "source"}, + ) +) + +// RecordTaskProcessed increments the task processed counter for a given status and source. +func RecordTaskProcessed(status, source string) { + if source == "" { + source = "unknown" + } + WorkerTasksProcessedTotal.WithLabelValues(status, source).Inc() +} + +// RecordTaskDuration records the processing duration of a task in seconds. +func RecordTaskDuration(status, source string, durationSeconds float64) { + if source == "" { + source = "unknown" + } + WorkerTaskDurationSeconds.WithLabelValues(status, source).Observe(durationSeconds) +} + +// RecordPipelinePublishedToAvailable records the end-to-end lag from publication to database write. +func RecordPipelinePublishedToAvailable(source string, durationSeconds float64) { + if source == "" { + source = "unknown" + } + PipelinePublishedToAvailableSeconds.WithLabelValues(source).Observe(durationSeconds) +} + +// SetDatasetTotalVulnerabilities sets the vulnerability count for an ecosystem and source. +func SetDatasetTotalVulnerabilities(ecosystem, source string, count float64) { + if source == "" { + source = "unknown" + } + if ecosystem == "" { + ecosystem = "unknown" + } + DatasetTotalVulnerabilities.WithLabelValues(ecosystem, source).Set(count) +} diff --git a/go/internal/metrics/metrics_test.go b/go/internal/metrics/metrics_test.go new file mode 100644 index 00000000000..bcdc70ad2d7 --- /dev/null +++ b/go/internal/metrics/metrics_test.go @@ -0,0 +1,76 @@ +package metrics_test + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/google/osv.dev/go/internal/metrics" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func TestMetricsRecording(t *testing.T) { + metrics.RecordTaskProcessed("success", "github-advisories") + metrics.RecordTaskProcessed("error", "nvd") + metrics.RecordTaskDuration("success", "github-advisories", 0.42) + metrics.RecordPipelinePublishedToAvailable("github-advisories", 120.0) + metrics.SetDatasetTotalVulnerabilities("npm", "github-advisories", 5000) + + // Scrape metrics handler + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + rec := httptest.NewRecorder() + promhttp.Handler().ServeHTTP(rec, req) + + resp := rec.Result() + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("failed to read response body: %v", err) + } + bodyStr := string(body) + + expectedStrings := []string{ + `osv_worker_tasks_processed_total{source="github-advisories",status="success"}`, + `osv_worker_tasks_processed_total{source="nvd",status="error"}`, + `osv_worker_task_duration_seconds_bucket{source="github-advisories",status="success"`, + `osv_pipeline_published_to_available_seconds_bucket{source="github-advisories"`, + `osv_dataset_total_vulnerabilities{ecosystem="npm",source="github-advisories"} 5000`, + } + + for _, exp := range expectedStrings { + if !strings.Contains(bodyStr, exp) { + t.Errorf("expected metrics output to contain %q, got:\n%s", exp, bodyStr) + } + } +} + +func TestMetricsServerLifecycle(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + srv := metrics.NewServer("127.0.0.1:0") // system assigns free port + srv.Start(ctx) + + // Test healthz handler directly + req := httptest.NewRequest(http.MethodGet, "/healthz", nil) + rec := httptest.NewRecorder() + srvHandler := metrics.NewServer(":0") + srvHandler.Handler().ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("expected status 200 from healthz, got %d", rec.Code) + } + + // Sleep briefly for server startup + time.Sleep(50 * time.Millisecond) + cancel() + time.Sleep(50 * time.Millisecond) +} diff --git a/go/internal/metrics/server.go b/go/internal/metrics/server.go new file mode 100644 index 00000000000..5c88cf0cd76 --- /dev/null +++ b/go/internal/metrics/server.go @@ -0,0 +1,76 @@ +// Package metrics provides Prometheus metric definitions and exposition for OSV services. +package metrics + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "time" + + "github.com/google/osv.dev/go/logger" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// DefaultPort is the standard metrics port scraped by GMP. +const DefaultPort = "9090" + +// Server wraps an HTTP server exposing Prometheus metrics. +type Server struct { + httpServer *http.Server +} + +// NewServer creates a new metrics server on the given address. +func NewServer(addr string) *Server { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + + return &Server{ + httpServer: &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + }, + } +} + +// Handler returns the underlying http.Handler. +func (s *Server) Handler() http.Handler { + return s.httpServer.Handler +} + +// Start runs the metrics server in a background goroutine and gracefully shuts down when ctx is done. +func (s *Server) Start(ctx context.Context) { + go func() { + logger.InfoContext(ctx, fmt.Sprintf("Starting Prometheus metrics server on %s", s.httpServer.Addr)) + if err := s.httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.ErrorContext(ctx, "Metrics server error", slog.Any("error", err)) + } + }() + + go func() { + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.httpServer.Shutdown(shutdownCtx); err != nil { + logger.ErrorContext(ctx, "Failed to gracefully shutdown metrics server", slog.Any("error", err)) + } + }() +} + +// StartMetricsServer is a convenience function that starts a metrics server on ":port" +// using DefaultPort if port is empty. +func StartMetricsServer(ctx context.Context, port string) *Server { + if port == "" { + port = DefaultPort + } + addr := ":" + port + srv := NewServer(addr) + srv.Start(ctx) + return srv +} diff --git a/go/internal/worker/subscriber.go b/go/internal/worker/subscriber.go index dbffdbbe4bc..7bc47264461 100644 --- a/go/internal/worker/subscriber.go +++ b/go/internal/worker/subscriber.go @@ -8,6 +8,7 @@ import ( "time" "cloud.google.com/go/pubsub/v2" + "github.com/google/osv.dev/go/internal/metrics" "github.com/google/osv.dev/go/logger" "github.com/klauspost/compress/zstd" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -28,6 +29,7 @@ func (s *Subscriber) Run(ctx context.Context) error { func (s *Subscriber) handleMessage(ctx context.Context, m *pubsub.Message) { if taskType := m.Attributes["type"]; taskType != "update" { logger.InfoContext(ctx, "Skipping message, not an update", slog.Any("task_type", taskType)) + metrics.RecordTaskProcessed("skipped", m.Attributes["source"]) m.Ack() return @@ -50,6 +52,7 @@ func (s *Subscriber) handleMessage(ctx context.Context, m *pubsub.Message) { task.Vuln, err = s.parseVuln(m) if err != nil { logger.ErrorContext(taskCtx, "Failed to parse vulnerability", append(logInfo, slog.Any("error", err))...) + metrics.RecordTaskProcessed("error", task.SourceID) m.Nack() return @@ -83,10 +86,20 @@ func (s *Subscriber) handleMessage(ctx context.Context, m *pubsub.Message) { task.SHA256 = m.Attributes["original_sha256"] } + startTime := time.Now() if err := s.Engine.RunTask(taskCtx, task); err != nil { + duration := time.Since(startTime).Seconds() + metrics.RecordTaskProcessed("error", task.SourceID) + metrics.RecordTaskDuration("error", task.SourceID, duration) logger.ErrorContext(taskCtx, "Failed to process task", append(logInfo, slog.Any("error", err))...) m.Nack() } else { + duration := time.Since(startTime).Seconds() + metrics.RecordTaskProcessed("success", task.SourceID) + metrics.RecordTaskDuration("success", task.SourceID, duration) + if task.SourceTime != nil { + metrics.RecordPipelinePublishedToAvailable(task.SourceID, time.Since(*task.SourceTime).Seconds()) + } logTaskLatency(taskCtx, task) m.Ack() } From 1ff4d9280084557f7126bb39cf2a93d9e050c968 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Tue, 1 Sep 2026 23:44:06 +0000 Subject: [PATCH 2/9] terraform files --- .../terraform/modules/osv_pipeline/iam.tf | 27 +++++++++++++++++++ .../terraform/modules/osv_pipeline/outputs.tf | 6 +++++ 2 files changed, 33 insertions(+) diff --git a/deployment/terraform/modules/osv_pipeline/iam.tf b/deployment/terraform/modules/osv_pipeline/iam.tf index a76144ed766..0a3fb2912e3 100644 --- a/deployment/terraform/modules/osv_pipeline/iam.tf +++ b/deployment/terraform/modules/osv_pipeline/iam.tf @@ -91,3 +91,30 @@ resource "google_pubsub_subscription_iam_member" "worker_recovery_subscriber" { role = "roles/pubsub.subscriber" member = "serviceAccount:${google_service_account.worker_sa.email}" } + +# Dedicated GKE Monitoring Service Account (Grafana) +resource "google_service_account" "monitoring_sa" { + project = var.project_id + account_id = "osv-monitoring-sa" + display_name = "OSV Monitoring Service Account" +} + +# Read-only telemetry roles for monitoring dashboards +resource "google_project_iam_member" "monitoring_sa_roles" { + for_each = toset([ + "roles/monitoring.viewer", + "roles/logging.viewer" + ]) + + project = var.project_id + role = each.value + member = "serviceAccount:${google_service_account.monitoring_sa.email}" +} + +# Workload Identity binding for Grafana K8s ServiceAccount +resource "google_service_account_iam_member" "grafana_workload_identity" { + service_account_id = google_service_account.monitoring_sa.name + role = "roles/iam.workloadIdentityUser" + member = "serviceAccount:${var.project_id}.svc.id.goog[default/grafana-sa]" +} + diff --git a/deployment/terraform/modules/osv_pipeline/outputs.tf b/deployment/terraform/modules/osv_pipeline/outputs.tf index bc7d6c74a9b..57ecc859798 100644 --- a/deployment/terraform/modules/osv_pipeline/outputs.tf +++ b/deployment/terraform/modules/osv_pipeline/outputs.tf @@ -67,3 +67,9 @@ output "affected_commits_backups_bucket_name" { value = google_storage_bucket.affected_commits_backups_bucket.name description = "The name of the AffectedCommits backups bucket" } + +output "monitoring_service_account_email" { + value = google_service_account.monitoring_sa.email + description = "The email of the GKE monitoring service account" +} + From 6cf7eb7da89eaeb61eb53bc05074f41bcefde1c5 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 2 Sep 2026 00:02:24 +0000 Subject: [PATCH 3/9] lint --- go/internal/metrics/server.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/internal/metrics/server.go b/go/internal/metrics/server.go index 5c88cf0cd76..ef8c4a7dd39 100644 --- a/go/internal/metrics/server.go +++ b/go/internal/metrics/server.go @@ -4,7 +4,6 @@ package metrics import ( "context" "errors" - "fmt" "log/slog" "net/http" "time" @@ -25,7 +24,7 @@ type Server struct { func NewServer(addr string) *Server { mux := http.NewServeMux() mux.Handle("/metrics", promhttp.Handler()) - mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("ok")) }) @@ -47,7 +46,7 @@ func (s *Server) Handler() http.Handler { // Start runs the metrics server in a background goroutine and gracefully shuts down when ctx is done. func (s *Server) Start(ctx context.Context) { go func() { - logger.InfoContext(ctx, fmt.Sprintf("Starting Prometheus metrics server on %s", s.httpServer.Addr)) + logger.InfoContext(ctx, "Starting Prometheus metrics server on "+s.httpServer.Addr) if err := s.httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { logger.ErrorContext(ctx, "Metrics server error", slog.Any("error", err)) } @@ -55,7 +54,7 @@ func (s *Server) Start(ctx context.Context) { go func() { <-ctx.Done() - shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) defer cancel() if err := s.httpServer.Shutdown(shutdownCtx); err != nil { logger.ErrorContext(ctx, "Failed to gracefully shutdown metrics server", slog.Any("error", err)) @@ -72,5 +71,6 @@ func StartMetricsServer(ctx context.Context, port string) *Server { addr := ":" + port srv := NewServer(addr) srv.Start(ctx) + return srv } From 913c1a575311ea8e1141f0f3cefe7677ffcb43fc Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 2 Sep 2026 02:43:37 +0000 Subject: [PATCH 4/9] even simpler --- .../dashboards/executive-overview.json | 694 +---------------- .../monitoring/grafana_dashboards.yaml | 702 +----------------- go/internal/metrics/metrics.go | 85 +-- go/internal/metrics/metrics_test.go | 22 +- go/internal/worker/subscriber.go | 16 +- 5 files changed, 75 insertions(+), 1444 deletions(-) diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json index 80419fbdab3..8728095b99e 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json @@ -1,166 +1,14 @@ { "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] + "list": [] }, "editable": false, "fiscalYearStartMonth": 0, - "graphTooltip": 1, + "graphTooltip": 0, "id": null, - "links": [ - { - "asDropdown": false, - "icon": "doc", - "includeVars": true, - "keepTime": true, - "tags": [], - "targetBlank": true, - "title": "On-Call Triage Runbook", - "tooltip": "Open OSV On-Call Runbook", - "type": "link", - "url": "https://github.com/google/osv.dev/blob/master/docs/observability.md" - } - ], + "links": [], "liveNow": false, "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 100, - "title": "🚀 Executive Service Level Objectives (SLOs) & Overall Health", - "type": "row" - }, - { - "datasource": { - "type": "googlecloud-monitoring-datasource", - "uid": "cloudmonitoring" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 95, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 99.5 - }, - { - "color": "green", - "value": 99.95 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 1, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "11.5.0", - "title": "API Availability (SLO > 99.95%)", - "type": "gauge" - }, - { - "datasource": { - "type": "googlecloud-monitoring-datasource", - "uid": "cloudmonitoring" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 100 - }, - { - "color": "red", - "value": 150 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "11.5.0", - "title": "API p99 Latency (SLO < 150ms)", - "type": "stat" - }, { "datasource": { "type": "prometheus", @@ -178,76 +26,6 @@ { "color": "green", "value": null - }, - { - "color": "yellow", - "value": 3600 - }, - { - "color": "red", - "value": 7200 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(osv_pipeline_published_to_available_seconds_bucket[1h])) by (le))", - "legendFormat": "p95 Lag", - "range": true, - "refId": "A" - } - ], - "title": "E2E Pipeline Lag (p95, SLO < 2h)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null } ] }, @@ -256,12 +34,12 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 1 + "h": 6, + "w": 8, + "x": 0, + "y": 0 }, - "id": 4, + "id": 1, "options": { "colorMode": "value", "graphMode": "area", @@ -284,125 +62,15 @@ "uid": "gmp" }, "editorMode": "code", - "expr": "sum(osv_dataset_total_vulnerabilities)", - "legendFormat": "Total Vulns", + "expr": "sum(osv_worker_tasks_processed_total)", + "legendFormat": "Total Tasks", "range": true, "refId": "A" } ], - "title": "Total Dataset Size", + "title": "Total Tasks Processed", "type": "stat" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 101, - "title": "🌐 Cloud Run Public API Serving Performance", - "type": "row" - }, - { - "datasource": { - "type": "googlecloud-monitoring-datasource", - "uid": "cloudmonitoring" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - } - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean", "max", "lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "title": "API Request Rate by HTTP Response Status Code", - "type": "timeseries" - }, - { - "datasource": { - "type": "googlecloud-monitoring-datasource", - "uid": "cloudmonitoring" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["mean", "max", "lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "title": "API Latency Percentiles (p50 / p95 / p99)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 102, - "title": "⚡ Ingestion Pipeline Throughput & Latency Lag", - "type": "row" - }, { "datasource": { "type": "prometheus", @@ -414,7 +82,7 @@ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, - "showPoints": "never", + "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", @@ -426,81 +94,18 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["mean", "max", "lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "sum(rate(osv_worker_tasks_processed_total[5m])) by (status, source)", - "legendFormat": "{{source}} ({{status}})", - "range": true, - "refId": "A" - } - ], - "title": "Worker Task Processing Rate (Velocity / Drain)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "showPoints": "never", - "spanNulls": false - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 7200 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 + "h": 6, + "w": 16, + "x": 8, + "y": 0 }, - "id": 8, + "id": 2, "options": { "legend": { - "calcs": ["mean", "max", "lastNotNull"], + "calcs": [ + "mean", + "lastNotNull" + ], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -517,260 +122,28 @@ "uid": "gmp" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(osv_pipeline_published_to_available_seconds_bucket[15m])) by (le, source))", - "legendFormat": "p95 {{source}}", + "expr": "sum(rate(osv_worker_tasks_processed_total[5m])) by (status)", + "legendFormat": "{{status}}", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(osv_pipeline_published_to_available_seconds_bucket[15m])) by (le, source))", - "legendFormat": "p50 {{source}}", - "range": true, - "refId": "B" } ], - "title": "E2E Published-to-Available Lag by Source", + "title": "Worker Task Processing Rate", "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 103, - "title": "🛡️ Subsystem Health & Multi-Window Error Budget Burn Rates", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 2 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 24 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "sum(rate(osv_worker_tasks_processed_total{status=\"error\"}[1h])) / (sum(rate(osv_worker_tasks_processed_total[1h])) > 0 or 1)", - "legendFormat": "1h Burn Rate", - "range": true, - "refId": "A" - } - ], - "title": "Worker Ingestion 1h Error Burn Rate", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 2 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 24 - }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "sum(rate(osv_worker_tasks_processed_total{status=\"error\"}[6h])) / (sum(rate(osv_worker_tasks_processed_total[6h])) > 0 or 1)", - "legendFormat": "6h Burn Rate", - "range": true, - "refId": "A" - } - ], - "title": "Worker Ingestion 6h Error Burn Rate", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 2 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 24 - }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "sum(rate(osv_worker_tasks_processed_total{status=\"error\"}[24h])) / (sum(rate(osv_worker_tasks_processed_total[24h])) > 0 or 1)", - "legendFormat": "24h Burn Rate", - "range": true, - "refId": "A" - } - ], - "title": "Worker Ingestion 24h Error Burn Rate", - "type": "stat" } ], - "refresh": "30s", + "refresh": "10s", "schemaVersion": 39, "tags": [ "osv", - "executive", - "sla", - "overview", - "gmp" + "gmp", + "worker" ], "templating": { "list": [] }, "time": { - "from": "now-24h", + "from": "now-1h", "to": "now" }, "timepicker": { @@ -779,16 +152,11 @@ "10s", "30s", "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" + "5m" ] }, "timezone": "utc", - "title": "Executive Overview & SLAs", - "uid": "osv-executive-overview", + "title": "OSV Worker - Test Dashboard", + "uid": "osv-worker-test", "version": 1 } diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards.yaml index 4494dd64f35..d6932bb59b7 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards.yaml @@ -9,167 +9,15 @@ data: executive-overview.json: | { "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] + "list": [] }, "editable": false, "fiscalYearStartMonth": 0, - "graphTooltip": 1, + "graphTooltip": 0, "id": null, - "links": [ - { - "asDropdown": false, - "icon": "doc", - "includeVars": true, - "keepTime": true, - "tags": [], - "targetBlank": true, - "title": "On-Call Triage Runbook", - "tooltip": "Open OSV On-Call Runbook", - "type": "link", - "url": "https://github.com/google/osv.dev/blob/master/docs/observability.md" - } - ], + "links": [], "liveNow": false, "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 100, - "title": "🚀 Executive Service Level Objectives (SLOs) & Overall Health", - "type": "row" - }, - { - "datasource": { - "type": "googlecloud-monitoring-datasource", - "uid": "cloudmonitoring" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 95, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 99.5 - }, - { - "color": "green", - "value": 99.95 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 1, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "11.5.0", - "title": "API Availability (SLO > 99.95%)", - "type": "gauge" - }, - { - "datasource": { - "type": "googlecloud-monitoring-datasource", - "uid": "cloudmonitoring" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 100 - }, - { - "color": "red", - "value": 150 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "11.5.0", - "title": "API p99 Latency (SLO < 150ms)", - "type": "stat" - }, { "datasource": { "type": "prometheus", @@ -187,76 +35,6 @@ data: { "color": "green", "value": null - }, - { - "color": "yellow", - "value": 3600 - }, - { - "color": "red", - "value": 7200 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(osv_pipeline_published_to_available_seconds_bucket[1h])) by (le))", - "legendFormat": "p95 Lag", - "range": true, - "refId": "A" - } - ], - "title": "E2E Pipeline Lag (p95, SLO < 2h)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null } ] }, @@ -265,12 +43,12 @@ data: "overrides": [] }, "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 1 + "h": 6, + "w": 8, + "x": 0, + "y": 0 }, - "id": 4, + "id": 1, "options": { "colorMode": "value", "graphMode": "area", @@ -293,133 +71,15 @@ data: "uid": "gmp" }, "editorMode": "code", - "expr": "sum(osv_dataset_total_vulnerabilities)", - "legendFormat": "Total Vulns", + "expr": "sum(osv_worker_tasks_processed_total)", + "legendFormat": "Total Tasks", "range": true, "refId": "A" } ], - "title": "Total Dataset Size", + "title": "Total Tasks Processed", "type": "stat" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 101, - "title": "🌐 Cloud Run Public API Serving Performance", - "type": "row" - }, - { - "datasource": { - "type": "googlecloud-monitoring-datasource", - "uid": "cloudmonitoring" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - } - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "title": "API Request Rate by HTTP Response Status Code", - "type": "timeseries" - }, - { - "datasource": { - "type": "googlecloud-monitoring-datasource", - "uid": "cloudmonitoring" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "title": "API Latency Percentiles (p50 / p95 / p99)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 102, - "title": "⚡ Ingestion Pipeline Throughput & Latency Lag", - "type": "row" - }, { "datasource": { "type": "prometheus", @@ -431,7 +91,7 @@ data: "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, - "showPoints": "never", + "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", @@ -443,87 +103,16 @@ data: "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "sum(rate(osv_worker_tasks_processed_total[5m])) by (status, source)", - "legendFormat": "{{source}} ({{status}})", - "range": true, - "refId": "A" - } - ], - "title": "Worker Task Processing Rate (Velocity / Drain)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "showPoints": "never", - "spanNulls": false - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 7200 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 + "h": 6, + "w": 16, + "x": 8, + "y": 0 }, - "id": 8, + "id": 2, "options": { "legend": { "calcs": [ "mean", - "max", "lastNotNull" ], "displayMode": "table", @@ -542,260 +131,28 @@ data: "uid": "gmp" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(osv_pipeline_published_to_available_seconds_bucket[15m])) by (le, source))", - "legendFormat": "p95 {{source}}", + "expr": "sum(rate(osv_worker_tasks_processed_total[5m])) by (status)", + "legendFormat": "{{status}}", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(osv_pipeline_published_to_available_seconds_bucket[15m])) by (le, source))", - "legendFormat": "p50 {{source}}", - "range": true, - "refId": "B" } ], - "title": "E2E Published-to-Available Lag by Source", + "title": "Worker Task Processing Rate", "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 103, - "title": "🛡️ Subsystem Health & Multi-Window Error Budget Burn Rates", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 2 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 24 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "sum(rate(osv_worker_tasks_processed_total{status=\"error\"}[1h])) / (sum(rate(osv_worker_tasks_processed_total[1h])) > 0 or 1)", - "legendFormat": "1h Burn Rate", - "range": true, - "refId": "A" - } - ], - "title": "Worker Ingestion 1h Error Burn Rate", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 2 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 24 - }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "sum(rate(osv_worker_tasks_processed_total{status=\"error\"}[6h])) / (sum(rate(osv_worker_tasks_processed_total[6h])) > 0 or 1)", - "legendFormat": "6h Burn Rate", - "range": true, - "refId": "A" - } - ], - "title": "Worker Ingestion 6h Error Burn Rate", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 2 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 24 - }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "sum(rate(osv_worker_tasks_processed_total{status=\"error\"}[24h])) / (sum(rate(osv_worker_tasks_processed_total[24h])) > 0 or 1)", - "legendFormat": "24h Burn Rate", - "range": true, - "refId": "A" - } - ], - "title": "Worker Ingestion 24h Error Burn Rate", - "type": "stat" } ], - "refresh": "30s", + "refresh": "10s", "schemaVersion": 39, "tags": [ "osv", - "executive", - "sla", - "overview", - "gmp" + "gmp", + "worker" ], "templating": { "list": [] }, "time": { - "from": "now-24h", + "from": "now-1h", "to": "now" }, "timepicker": { @@ -804,16 +161,11 @@ data: "10s", "30s", "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" + "5m" ] }, "timezone": "utc", - "title": "Executive Overview & SLAs", - "uid": "osv-executive-overview", + "title": "OSV Worker - Test Dashboard", + "uid": "osv-worker-test", "version": 1 } diff --git a/go/internal/metrics/metrics.go b/go/internal/metrics/metrics.go index 5956dc93e72..82d84fa832b 100644 --- a/go/internal/metrics/metrics.go +++ b/go/internal/metrics/metrics.go @@ -7,91 +7,20 @@ import ( ) var ( - // PipelinePublishedToAvailableSeconds tracks the end-to-end latency between - // the upstream publish/source timestamp and when the vulnerability is saved in the OSV database. - PipelinePublishedToAvailableSeconds = promauto.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "osv_pipeline_published_to_available_seconds", - Help: "Seconds between upstream vulnerability publish/source timestamp and OSV database write.", - Buckets: []float64{ - 60, // 1 min - 300, // 5 min - 600, // 10 min - 1800, // 30 min - 3600, // 1 hour - 7200, // 2 hours (SLO threshold) - 14400, // 4 hours - 28800, // 8 hours - 86400, // 24 hours - 172800, // 48 hours - 604800, // 7 days - }, - }, - []string{"source"}, - ) - - // WorkerTasksProcessedTotal tracks the total number of ingestion/enrichment tasks processed by the worker. + // WorkerTasksProcessedTotal tracks the total number of tasks processed by the worker. WorkerTasksProcessedTotal = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "osv_worker_tasks_processed_total", Help: "Total number of tasks processed by the worker.", }, - []string{"status", "source"}, - ) - - // WorkerTaskDurationSeconds tracks the duration taken by workers to process individual tasks. - WorkerTaskDurationSeconds = promauto.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "osv_worker_task_duration_seconds", - Help: "Duration in seconds to process a worker task.", - Buckets: []float64{ - 0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, - }, - }, - []string{"status", "source"}, - ) - - // DatasetTotalVulnerabilities tracks the total number of vulnerabilities in the OSV dataset. - DatasetTotalVulnerabilities = promauto.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "osv_dataset_total_vulnerabilities", - Help: "Total number of vulnerabilities in the OSV dataset.", - }, - []string{"ecosystem", "source"}, + []string{"status"}, ) ) -// RecordTaskProcessed increments the task processed counter for a given status and source. -func RecordTaskProcessed(status, source string) { - if source == "" { - source = "unknown" - } - WorkerTasksProcessedTotal.WithLabelValues(status, source).Inc() -} - -// RecordTaskDuration records the processing duration of a task in seconds. -func RecordTaskDuration(status, source string, durationSeconds float64) { - if source == "" { - source = "unknown" - } - WorkerTaskDurationSeconds.WithLabelValues(status, source).Observe(durationSeconds) -} - -// RecordPipelinePublishedToAvailable records the end-to-end lag from publication to database write. -func RecordPipelinePublishedToAvailable(source string, durationSeconds float64) { - if source == "" { - source = "unknown" - } - PipelinePublishedToAvailableSeconds.WithLabelValues(source).Observe(durationSeconds) -} - -// SetDatasetTotalVulnerabilities sets the vulnerability count for an ecosystem and source. -func SetDatasetTotalVulnerabilities(ecosystem, source string, count float64) { - if source == "" { - source = "unknown" - } - if ecosystem == "" { - ecosystem = "unknown" +// RecordTaskProcessed increments the task processed counter for a given status. +func RecordTaskProcessed(status string) { + if status == "" { + status = "unknown" } - DatasetTotalVulnerabilities.WithLabelValues(ecosystem, source).Set(count) + WorkerTasksProcessedTotal.WithLabelValues(status).Inc() } diff --git a/go/internal/metrics/metrics_test.go b/go/internal/metrics/metrics_test.go index bcdc70ad2d7..530552b4ba1 100644 --- a/go/internal/metrics/metrics_test.go +++ b/go/internal/metrics/metrics_test.go @@ -14,13 +14,9 @@ import ( ) func TestMetricsRecording(t *testing.T) { - metrics.RecordTaskProcessed("success", "github-advisories") - metrics.RecordTaskProcessed("error", "nvd") - metrics.RecordTaskDuration("success", "github-advisories", 0.42) - metrics.RecordPipelinePublishedToAvailable("github-advisories", 120.0) - metrics.SetDatasetTotalVulnerabilities("npm", "github-advisories", 5000) + metrics.RecordTaskProcessed("success") + metrics.RecordTaskProcessed("error") - // Scrape metrics handler req := httptest.NewRequest(http.MethodGet, "/metrics", nil) rec := httptest.NewRecorder() promhttp.Handler().ServeHTTP(rec, req) @@ -39,11 +35,8 @@ func TestMetricsRecording(t *testing.T) { bodyStr := string(body) expectedStrings := []string{ - `osv_worker_tasks_processed_total{source="github-advisories",status="success"}`, - `osv_worker_tasks_processed_total{source="nvd",status="error"}`, - `osv_worker_task_duration_seconds_bucket{source="github-advisories",status="success"`, - `osv_pipeline_published_to_available_seconds_bucket{source="github-advisories"`, - `osv_dataset_total_vulnerabilities{ecosystem="npm",source="github-advisories"} 5000`, + `osv_worker_tasks_processed_total{status="success"}`, + `osv_worker_tasks_processed_total{status="error"}`, } for _, exp := range expectedStrings { @@ -57,19 +50,16 @@ func TestMetricsServerLifecycle(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - srv := metrics.NewServer("127.0.0.1:0") // system assigns free port + srv := metrics.NewServer("127.0.0.1:0") srv.Start(ctx) - // Test healthz handler directly req := httptest.NewRequest(http.MethodGet, "/healthz", nil) rec := httptest.NewRecorder() - srvHandler := metrics.NewServer(":0") - srvHandler.Handler().ServeHTTP(rec, req) + srv.Handler().ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("expected status 200 from healthz, got %d", rec.Code) } - // Sleep briefly for server startup time.Sleep(50 * time.Millisecond) cancel() time.Sleep(50 * time.Millisecond) diff --git a/go/internal/worker/subscriber.go b/go/internal/worker/subscriber.go index 7bc47264461..5ec59368fcf 100644 --- a/go/internal/worker/subscriber.go +++ b/go/internal/worker/subscriber.go @@ -29,7 +29,7 @@ func (s *Subscriber) Run(ctx context.Context) error { func (s *Subscriber) handleMessage(ctx context.Context, m *pubsub.Message) { if taskType := m.Attributes["type"]; taskType != "update" { logger.InfoContext(ctx, "Skipping message, not an update", slog.Any("task_type", taskType)) - metrics.RecordTaskProcessed("skipped", m.Attributes["source"]) + metrics.RecordTaskProcessed("skipped") m.Ack() return @@ -52,7 +52,7 @@ func (s *Subscriber) handleMessage(ctx context.Context, m *pubsub.Message) { task.Vuln, err = s.parseVuln(m) if err != nil { logger.ErrorContext(taskCtx, "Failed to parse vulnerability", append(logInfo, slog.Any("error", err))...) - metrics.RecordTaskProcessed("error", task.SourceID) + metrics.RecordTaskProcessed("error") m.Nack() return @@ -86,20 +86,12 @@ func (s *Subscriber) handleMessage(ctx context.Context, m *pubsub.Message) { task.SHA256 = m.Attributes["original_sha256"] } - startTime := time.Now() if err := s.Engine.RunTask(taskCtx, task); err != nil { - duration := time.Since(startTime).Seconds() - metrics.RecordTaskProcessed("error", task.SourceID) - metrics.RecordTaskDuration("error", task.SourceID, duration) + metrics.RecordTaskProcessed("error") logger.ErrorContext(taskCtx, "Failed to process task", append(logInfo, slog.Any("error", err))...) m.Nack() } else { - duration := time.Since(startTime).Seconds() - metrics.RecordTaskProcessed("success", task.SourceID) - metrics.RecordTaskDuration("success", task.SourceID, duration) - if task.SourceTime != nil { - metrics.RecordPipelinePublishedToAvailable(task.SourceID, time.Since(*task.SourceTime).Seconds()) - } + metrics.RecordTaskProcessed("success") logTaskLatency(taskCtx, task) m.Ack() } From b693a37f44548caa41abdb3067b3a9cb448dc31d Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 2 Sep 2026 02:55:42 +0000 Subject: [PATCH 5/9] fix nits --- .../base/feeds/data-quality-dashboard.yaml | 44 ------------------- .../monitoring/grafana_ingress.yaml | 8 ++++ 2 files changed, 8 insertions(+), 44 deletions(-) delete mode 100644 deployment/clouddeploy/gke-workers/base/feeds/data-quality-dashboard.yaml diff --git a/deployment/clouddeploy/gke-workers/base/feeds/data-quality-dashboard.yaml b/deployment/clouddeploy/gke-workers/base/feeds/data-quality-dashboard.yaml deleted file mode 100644 index 9f16d77cb36..00000000000 --- a/deployment/clouddeploy/gke-workers/base/feeds/data-quality-dashboard.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: data-quality-dashboard -spec: - replicas: 1 - selector: - matchLabels: - app: data-quality-dashboard - template: - metadata: - labels: - app: data-quality-dashboard - spec: - containers: - - name: dashboard - image: data-quality-dashboard - imagePullPolicy: Always - ports: - - containerPort: 8080 - env: - - name: GCS_BUCKET - value: osv-test-cve-osv-conversion - - name: PORT - value: "8080" - resources: - requests: - cpu: "250m" - memory: "512Mi" - limits: - cpu: "1" - memory: "2Gi" ---- -apiVersion: v1 -kind: Service -metadata: - name: data-quality-dashboard -spec: - selector: - app: data-quality-dashboard - ports: - - protocol: TCP - port: 80 - targetPort: 8080 diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml index 179306a8a49..5787222b91c 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml @@ -10,6 +10,14 @@ spec: enabled: true oauthclientCredentials: secretName: grafana-iap-oauth + healthCheck: + checkIntervalSec: 15 + timeoutSec: 5 + healthyThreshold: 1 + unhealthyThreshold: 2 + type: HTTP + requestPath: /api/health + port: 3000 --- apiVersion: networking.gke.io/v1 kind: ManagedCertificate From b7a54229ad37accc9a42457eccb2cee72c3f603f Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 2 Sep 2026 03:03:06 +0000 Subject: [PATCH 6/9] fix some issues --- .../monitoring/grafana_dashboards.yaml | 171 ------------------ .../monitoring/grafana_ingress.yaml | 4 +- .../monitoring/kustomization.yaml | 6 +- 3 files changed, 7 insertions(+), 174 deletions(-) delete mode 100644 deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards.yaml diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards.yaml deleted file mode 100644 index d6932bb59b7..00000000000 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards.yaml +++ /dev/null @@ -1,171 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-dashboards - labels: - app.kubernetes.io/name: grafana - app.kubernetes.io/part-of: osv-monitoring -data: - executive-overview.json: | - { - "annotations": { - "list": [] - }, - "editable": false, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "sum(osv_worker_tasks_processed_total)", - "legendFormat": "Total Tasks", - "range": true, - "refId": "A" - } - ], - "title": "Total Tasks Processed", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - } - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 16, - "x": 8, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "gmp" - }, - "editorMode": "code", - "expr": "sum(rate(osv_worker_tasks_processed_total[5m])) by (status)", - "legendFormat": "{{status}}", - "range": true, - "refId": "A" - } - ], - "title": "Worker Task Processing Rate", - "type": "timeseries" - } - ], - "refresh": "10s", - "schemaVersion": 39, - "tags": [ - "osv", - "gmp", - "worker" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m" - ] - }, - "timezone": "utc", - "title": "OSV Worker - Test Dashboard", - "uid": "osv-worker-test", - "version": 1 - } diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml index 5787222b91c..875689952da 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml @@ -45,8 +45,8 @@ spec: - host: monitoring.osv.dev http: paths: - - path: /* - pathType: ImplementationSpecific + - path: / + pathType: Prefix backend: service: name: grafana diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml index 8a6cb3a8dd7..db4980bd3ee 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml @@ -3,7 +3,11 @@ resources: - grafana_service_account.yaml - grafana_datasources.yaml - grafana_dashboards_prov.yaml -- grafana_dashboards.yaml - grafana_deployment.yaml - grafana_service.yaml - grafana_ingress.yaml + +configMapGenerator: +- name: grafana-dashboards + files: + - executive-overview.json=dashboards/executive-overview.json From ae72dad15f1959aed446df49022976887d9ffff7 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 2 Sep 2026 03:48:02 +0000 Subject: [PATCH 7/9] add uid info --- .../monitoring/dashboards/executive-overview.json | 14 +++++++++++++- .../monitoring/grafana_datasources.yaml | 2 ++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json index 8728095b99e..b30958863db 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json @@ -1,6 +1,18 @@ { "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] }, "editable": false, "fiscalYearStartMonth": 0, diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_datasources.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_datasources.yaml index 031a613eabf..8418de1c4fe 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_datasources.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_datasources.yaml @@ -10,6 +10,7 @@ data: apiVersion: 1 datasources: - name: GMP (Prometheus) + uid: gmp type: prometheus access: proxy url: http://127.0.0.1:9090 @@ -19,6 +20,7 @@ data: httpMethod: POST timeInterval: 15s - name: Cloud Monitoring + uid: cloud-monitoring type: googlecloud-monitoring-datasource access: proxy editable: false From afb0f4f774772efaf5991e994c50c49014d588dd Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 2 Sep 2026 04:59:51 +0000 Subject: [PATCH 8/9] nits and stuff --- .../monitoring/grafana_deployment.yaml | 40 ++++++++++++++++--- .../monitoring/grafana_ingress.yaml | 4 +- go/internal/metrics/metrics.go | 16 ++++++-- go/internal/metrics/metrics_test.go | 4 +- go/internal/worker/subscriber.go | 8 ++-- 5 files changed, 55 insertions(+), 17 deletions(-) diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_deployment.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_deployment.yaml index 6464a5cfccc..790fed42089 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_deployment.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_deployment.yaml @@ -33,16 +33,28 @@ spec: value: "false" - name: GF_AUTH_BASIC_ENABLED value: "false" - - name: GF_AUTH_PROXY_ENABLED + - name: GF_AUTH_DISABLE_LOGIN_FORM value: "true" - - name: GF_AUTH_PROXY_HEADER_NAME - value: "X-Goog-Authenticated-User-Email" - - name: GF_AUTH_PROXY_HEADER_PROPERTY + - name: GF_AUTH_JWT_ENABLED + value: "true" + - name: GF_AUTH_JWT_HEADER_NAME + value: "X-Goog-IAP-JWT-Assertion" + - name: GF_AUTH_JWT_EMAIL_CLAIM + value: "email" + - name: GF_AUTH_JWT_USERNAME_CLAIM value: "email" - - name: GF_AUTH_PROXY_AUTO_SIGN_UP + - name: GF_AUTH_JWT_JWK_SET_URL + value: "https://www.gstatic.com/iap/verify/public_key-jwk" + - name: GF_AUTH_JWT_EXPECT_CLAIMS + value: '{"iss": "https://cloud.google.com/iap"}' + - name: GF_AUTH_JWT_AUTO_SIGN_UP + value: "true" + - name: GF_USERS_AUTO_ASSIGN_ORG value: "true" + - name: GF_USERS_AUTO_ASSIGN_ORG_ROLE + value: "Admin" - name: GF_SERVER_ROOT_URL - value: "https://monitoring.osv.dev" + value: "https://monitoring.test.osv.dev" resources: requests: cpu: 250m @@ -50,6 +62,22 @@ spec: limits: cpu: 1000m memory: 1Gi + livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 2 volumeMounts: - name: datasources mountPath: /etc/grafana/provisioning/datasources diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml index 875689952da..a433c522443 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml @@ -28,7 +28,7 @@ metadata: app.kubernetes.io/part-of: osv-monitoring spec: domains: - - monitoring.osv.dev + - monitoring.test.osv.dev --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -42,7 +42,7 @@ metadata: networking.gke.io/managed-certificates: grafana-managed-cert spec: rules: - - host: monitoring.osv.dev + - host: monitoring.test.osv.dev http: paths: - path: / diff --git a/go/internal/metrics/metrics.go b/go/internal/metrics/metrics.go index 82d84fa832b..1b8ed51e7a3 100644 --- a/go/internal/metrics/metrics.go +++ b/go/internal/metrics/metrics.go @@ -17,10 +17,20 @@ var ( ) ) +// TaskStatus represents the outcome of processing a worker task. +type TaskStatus string + +const ( + TaskStatusSuccess TaskStatus = "success" + TaskStatusError TaskStatus = "error" + TaskStatusSkipped TaskStatus = "skipped" + TaskStatusUnknown TaskStatus = "unknown" +) + // RecordTaskProcessed increments the task processed counter for a given status. -func RecordTaskProcessed(status string) { +func RecordTaskProcessed(status TaskStatus) { if status == "" { - status = "unknown" + status = TaskStatusUnknown } - WorkerTasksProcessedTotal.WithLabelValues(status).Inc() + WorkerTasksProcessedTotal.WithLabelValues(string(status)).Inc() } diff --git a/go/internal/metrics/metrics_test.go b/go/internal/metrics/metrics_test.go index 530552b4ba1..780c22ec6af 100644 --- a/go/internal/metrics/metrics_test.go +++ b/go/internal/metrics/metrics_test.go @@ -14,8 +14,8 @@ import ( ) func TestMetricsRecording(t *testing.T) { - metrics.RecordTaskProcessed("success") - metrics.RecordTaskProcessed("error") + metrics.RecordTaskProcessed(metrics.TaskStatusSuccess) + metrics.RecordTaskProcessed(metrics.TaskStatusError) req := httptest.NewRequest(http.MethodGet, "/metrics", nil) rec := httptest.NewRecorder() diff --git a/go/internal/worker/subscriber.go b/go/internal/worker/subscriber.go index 5ec59368fcf..1bcfefd8fc3 100644 --- a/go/internal/worker/subscriber.go +++ b/go/internal/worker/subscriber.go @@ -29,7 +29,7 @@ func (s *Subscriber) Run(ctx context.Context) error { func (s *Subscriber) handleMessage(ctx context.Context, m *pubsub.Message) { if taskType := m.Attributes["type"]; taskType != "update" { logger.InfoContext(ctx, "Skipping message, not an update", slog.Any("task_type", taskType)) - metrics.RecordTaskProcessed("skipped") + metrics.RecordTaskProcessed(metrics.TaskStatusSkipped) m.Ack() return @@ -52,7 +52,7 @@ func (s *Subscriber) handleMessage(ctx context.Context, m *pubsub.Message) { task.Vuln, err = s.parseVuln(m) if err != nil { logger.ErrorContext(taskCtx, "Failed to parse vulnerability", append(logInfo, slog.Any("error", err))...) - metrics.RecordTaskProcessed("error") + metrics.RecordTaskProcessed(metrics.TaskStatusError) m.Nack() return @@ -87,11 +87,11 @@ func (s *Subscriber) handleMessage(ctx context.Context, m *pubsub.Message) { } if err := s.Engine.RunTask(taskCtx, task); err != nil { - metrics.RecordTaskProcessed("error") + metrics.RecordTaskProcessed(metrics.TaskStatusError) logger.ErrorContext(taskCtx, "Failed to process task", append(logInfo, slog.Any("error", err))...) m.Nack() } else { - metrics.RecordTaskProcessed("success") + metrics.RecordTaskProcessed(metrics.TaskStatusSuccess) logTaskLatency(taskCtx, task) m.Ack() } From eb4b2b146e5c5c4a6703f9193dd456d7aa9c2065 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 2 Sep 2026 05:07:04 +0000 Subject: [PATCH 9/9] put the configmaps into a sub folder --- .../monitoring/{ => configmaps}/grafana_dashboards_prov.yaml | 0 .../monitoring/{ => configmaps}/grafana_datasources.yaml | 0 .../environments/oss-vdb-test/monitoring/kustomization.yaml | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/{ => configmaps}/grafana_dashboards_prov.yaml (100%) rename deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/{ => configmaps}/grafana_datasources.yaml (100%) diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards_prov.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/configmaps/grafana_dashboards_prov.yaml similarity index 100% rename from deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_dashboards_prov.yaml rename to deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/configmaps/grafana_dashboards_prov.yaml diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_datasources.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/configmaps/grafana_datasources.yaml similarity index 100% rename from deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_datasources.yaml rename to deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/configmaps/grafana_datasources.yaml diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml index db4980bd3ee..dd7ac9e0263 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml @@ -1,8 +1,8 @@ resources: - worker_pod_monitoring.yaml - grafana_service_account.yaml -- grafana_datasources.yaml -- grafana_dashboards_prov.yaml +- configmaps/grafana_datasources.yaml +- configmaps/grafana_dashboards_prov.yaml - grafana_deployment.yaml - grafana_service.yaml - grafana_ingress.yaml