diff --git a/deployment/clouddeploy/gke-workers/base/core/workers.yaml b/deployment/clouddeploy/gke-workers/base/core/workers.yaml index c38119dce61..4f90ab89d6d 100644 --- a/deployment/clouddeploy/gke-workers/base/core/workers.yaml +++ b/deployment/clouddeploy/gke-workers/base/core/workers.yaml @@ -41,6 +41,9 @@ spec: value: failed-tasks - name: NOTIFY_PYPI value: "false" + ports: + - name: metrics + containerPort: 9090 resources: requests: cpu: 1 diff --git a/deployment/clouddeploy/gke-workers/base/extra/workers-cves.yaml b/deployment/clouddeploy/gke-workers/base/extra/workers-cves.yaml index 9bba470bd7a..44699b0612d 100644 --- a/deployment/clouddeploy/gke-workers/base/extra/workers-cves.yaml +++ b/deployment/clouddeploy/gke-workers/base/extra/workers-cves.yaml @@ -75,6 +75,9 @@ spec: value: failed-tasks - name: NOTIFY_PYPI value: "false" + ports: + - name: metrics + containerPort: 9090 resources: requests: cpu: "0.9" diff --git a/deployment/clouddeploy/gke-workers/base/extra/workers-reimport.yaml b/deployment/clouddeploy/gke-workers/base/extra/workers-reimport.yaml index 765566c9bde..181e40b106a 100644 --- a/deployment/clouddeploy/gke-workers/base/extra/workers-reimport.yaml +++ b/deployment/clouddeploy/gke-workers/base/extra/workers-reimport.yaml @@ -75,6 +75,9 @@ spec: value: failed-tasks - name: NOTIFY_PYPI value: "false" + ports: + - name: metrics + containerPort: 9090 resources: requests: cpu: "0.9" diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml index 7ec61ca37bb..50f02560ae8 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml @@ -2,6 +2,7 @@ resources: - ../../base - staging-api-test.yaml - osv-linter.yaml +- monitoring patches: - path: workers.yaml - path: scaler.yaml diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/configmaps/grafana_dashboards_prov.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/configmaps/grafana_dashboards_prov.yaml new file mode 100644 index 00000000000..d62e3307d67 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/configmaps/grafana_dashboards_prov.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-prov + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring +data: + dashboards.yaml: | + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: false + options: + path: /etc/grafana/dashboards diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/configmaps/grafana_datasources.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/configmaps/grafana_datasources.yaml new file mode 100644 index 00000000000..8418de1c4fe --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/configmaps/grafana_datasources.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring +data: + datasources.yaml: | + apiVersion: 1 + datasources: + - name: GMP (Prometheus) + uid: gmp + type: prometheus + access: proxy + url: http://127.0.0.1:9090 + isDefault: true + editable: false + jsonData: + httpMethod: POST + timeInterval: 15s + - name: Cloud Monitoring + uid: cloud-monitoring + type: googlecloud-monitoring-datasource + access: proxy + editable: false + jsonData: + authenticationType: gce + defaultProject: oss-vdb-test diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json new file mode 100644 index 00000000000..b30958863db --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/dashboards/executive-overview.json @@ -0,0 +1,174 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "sum(osv_worker_tasks_processed_total)", + "legendFormat": "Total Tasks", + "range": true, + "refId": "A" + } + ], + "title": "Total Tasks Processed", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "gmp" + }, + "editorMode": "code", + "expr": "sum(rate(osv_worker_tasks_processed_total[5m])) by (status)", + "legendFormat": "{{status}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Task Processing Rate", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "osv", + "gmp", + "worker" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m" + ] + }, + "timezone": "utc", + "title": "OSV Worker - Test Dashboard", + "uid": "osv-worker-test", + "version": 1 +} diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_deployment.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_deployment.yaml new file mode 100644 index 00000000000..790fed42089 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_deployment.yaml @@ -0,0 +1,122 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: grafana + template: + metadata: + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring + spec: + serviceAccountName: grafana-sa + containers: + - name: grafana + image: grafana/grafana:11.5.0@sha256:4a0808cf7ca4b2569e5d4cb05ef2222a27ffb2a0c6499318b76c8cb81a6c4b28 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 3000 + env: + - name: GF_SECURITY_ALLOW_EMBEDDING + value: "true" + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "false" + - name: GF_AUTH_BASIC_ENABLED + value: "false" + - name: GF_AUTH_DISABLE_LOGIN_FORM + value: "true" + - name: GF_AUTH_JWT_ENABLED + value: "true" + - name: GF_AUTH_JWT_HEADER_NAME + value: "X-Goog-IAP-JWT-Assertion" + - name: GF_AUTH_JWT_EMAIL_CLAIM + value: "email" + - name: GF_AUTH_JWT_USERNAME_CLAIM + value: "email" + - name: GF_AUTH_JWT_JWK_SET_URL + value: "https://www.gstatic.com/iap/verify/public_key-jwk" + - name: GF_AUTH_JWT_EXPECT_CLAIMS + value: '{"iss": "https://cloud.google.com/iap"}' + - name: GF_AUTH_JWT_AUTO_SIGN_UP + value: "true" + - name: GF_USERS_AUTO_ASSIGN_ORG + value: "true" + - name: GF_USERS_AUTO_ASSIGN_ORG_ROLE + value: "Admin" + - name: GF_SERVER_ROOT_URL + value: "https://monitoring.test.osv.dev" + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 2 + volumeMounts: + - name: datasources + mountPath: /etc/grafana/provisioning/datasources + readOnly: true + - name: dashboards-prov + mountPath: /etc/grafana/provisioning/dashboards + readOnly: true + - name: dashboards + mountPath: /etc/grafana/dashboards + readOnly: true + - name: storage + mountPath: /var/lib/grafana + - name: gmp-frontend + image: gke.gcr.io/prometheus-engine/frontend:v0.18.2-gke.1@sha256:35ab356d969255dd7694ae371bf0991b97907451b4c7e1574b58a096d3be5977 + args: + - "--web.listen-address=127.0.0.1:9090" + - "--project-id=$(GOOGLE_CLOUD_PROJECT)" + env: + - name: GOOGLE_CLOUD_PROJECT + value: oss-vdb-test + ports: + - name: gmp-proxy + containerPort: 9090 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + volumes: + - name: datasources + configMap: + name: grafana-datasources + - name: dashboards-prov + configMap: + name: grafana-dashboards-prov + - name: dashboards + configMap: + name: grafana-dashboards + - name: storage + emptyDir: {} diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml new file mode 100644 index 00000000000..a433c522443 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_ingress.yaml @@ -0,0 +1,54 @@ +apiVersion: cloud.google.com/v1 +kind: BackendConfig +metadata: + name: grafana-backend-config + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring +spec: + iap: + enabled: true + oauthclientCredentials: + secretName: grafana-iap-oauth + healthCheck: + checkIntervalSec: 15 + timeoutSec: 5 + healthyThreshold: 1 + unhealthyThreshold: 2 + type: HTTP + requestPath: /api/health + port: 3000 +--- +apiVersion: networking.gke.io/v1 +kind: ManagedCertificate +metadata: + name: grafana-managed-cert + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring +spec: + domains: + - monitoring.test.osv.dev +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana-ingress + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring + annotations: + kubernetes.io/ingress.class: "gce" + networking.gke.io/managed-certificates: grafana-managed-cert +spec: + rules: + - host: monitoring.test.osv.dev + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana + port: + number: 3000 diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service.yaml new file mode 100644 index 00000000000..dd917314dfc --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: grafana + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring + annotations: + cloud.google.com/backend-config: '{"default": "grafana-backend-config"}' + cloud.google.com/neg: '{"ingress": true}' +spec: + type: ClusterIP + ports: + - name: http + port: 3000 + targetPort: 3000 + protocol: TCP + selector: + app.kubernetes.io/name: grafana diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service_account.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service_account.yaml new file mode 100644 index 00000000000..c36395c1421 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/grafana_service_account.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: grafana-sa + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: osv-monitoring + annotations: + iam.gke.io/gcp-service-account: osv-monitoring-sa@oss-vdb-test.iam.gserviceaccount.com diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml new file mode 100644 index 00000000000..dd7ac9e0263 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/kustomization.yaml @@ -0,0 +1,13 @@ +resources: +- worker_pod_monitoring.yaml +- grafana_service_account.yaml +- configmaps/grafana_datasources.yaml +- configmaps/grafana_dashboards_prov.yaml +- grafana_deployment.yaml +- grafana_service.yaml +- grafana_ingress.yaml + +configMapGenerator: +- name: grafana-dashboards + files: + - executive-overview.json=dashboards/executive-overview.json diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/worker_pod_monitoring.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/worker_pod_monitoring.yaml new file mode 100644 index 00000000000..54947f9e31d --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/monitoring/worker_pod_monitoring.yaml @@ -0,0 +1,19 @@ +apiVersion: monitoring.googleapis.com/v1 +kind: PodMonitoring +metadata: + name: worker-pod-monitoring + labels: + app.kubernetes.io/name: osv-worker + app.kubernetes.io/part-of: google-cloud-managed-prometheus +spec: + selector: + matchExpressions: + - key: name + operator: In + values: + - worker-private + - workers-reimport + - workers-cves + endpoints: + - port: metrics + interval: 15s diff --git a/deployment/terraform/modules/osv_pipeline/iam.tf b/deployment/terraform/modules/osv_pipeline/iam.tf index a76144ed766..0a3fb2912e3 100644 --- a/deployment/terraform/modules/osv_pipeline/iam.tf +++ b/deployment/terraform/modules/osv_pipeline/iam.tf @@ -91,3 +91,30 @@ resource "google_pubsub_subscription_iam_member" "worker_recovery_subscriber" { role = "roles/pubsub.subscriber" member = "serviceAccount:${google_service_account.worker_sa.email}" } + +# Dedicated GKE Monitoring Service Account (Grafana) +resource "google_service_account" "monitoring_sa" { + project = var.project_id + account_id = "osv-monitoring-sa" + display_name = "OSV Monitoring Service Account" +} + +# Read-only telemetry roles for monitoring dashboards +resource "google_project_iam_member" "monitoring_sa_roles" { + for_each = toset([ + "roles/monitoring.viewer", + "roles/logging.viewer" + ]) + + project = var.project_id + role = each.value + member = "serviceAccount:${google_service_account.monitoring_sa.email}" +} + +# Workload Identity binding for Grafana K8s ServiceAccount +resource "google_service_account_iam_member" "grafana_workload_identity" { + service_account_id = google_service_account.monitoring_sa.name + role = "roles/iam.workloadIdentityUser" + member = "serviceAccount:${var.project_id}.svc.id.goog[default/grafana-sa]" +} + diff --git a/deployment/terraform/modules/osv_pipeline/outputs.tf b/deployment/terraform/modules/osv_pipeline/outputs.tf index bc7d6c74a9b..57ecc859798 100644 --- a/deployment/terraform/modules/osv_pipeline/outputs.tf +++ b/deployment/terraform/modules/osv_pipeline/outputs.tf @@ -67,3 +67,9 @@ output "affected_commits_backups_bucket_name" { value = google_storage_bucket.affected_commits_backups_bucket.name description = "The name of the AffectedCommits backups bucket" } + +output "monitoring_service_account_email" { + value = google_service_account.monitoring_sa.email + description = "The email of the GKE monitoring service account" +} + diff --git a/go/cmd/worker/main.go b/go/cmd/worker/main.go index 2b9bbaf7ebb..4bf8f35ca04 100644 --- a/go/cmd/worker/main.go +++ b/go/cmd/worker/main.go @@ -17,6 +17,7 @@ import ( "cloud.google.com/go/pubsub/v2" "cloud.google.com/go/storage" db "github.com/google/osv.dev/go/internal/database/datastore" + "github.com/google/osv.dev/go/internal/metrics" "github.com/google/osv.dev/go/internal/worker" "github.com/google/osv.dev/go/internal/worker/pipeline/registry" "github.com/google/osv.dev/go/logger" @@ -59,6 +60,10 @@ func run() error { vulnBucket := envOrDefault("OSV_VULNERABILITIES_BUCKET", "osv-test-vulnerabilities") failTasksTopic := envOrDefault("FAILED_TASKS_TOPIC", "failed-tasks") notifyPyPI, _ := strconv.ParseBool(envOrDefault("NOTIFY_PYPI", "false")) // returns false on error + metricsPort := envOrDefault("METRICS_PORT", "9090") + + // Start Prometheus metrics server for GMP scraping + metrics.StartMetricsServer(ctx, metricsPort) // Plug in all the connections to the engine dsClient, err := datastore.NewClientWithDatabase(ctx, project, datastoreID) diff --git a/go/go.mod b/go/go.mod index fcd7d234234..ef294733d11 100644 --- a/go/go.mod +++ b/go/go.mod @@ -25,6 +25,7 @@ require ( github.com/ossf/osv-schema/bindings/go v0.0.0-20260806060209-f3f826310aec github.com/package-url/packageurl-go v0.1.6 github.com/pandatix/go-cvss v0.6.2 + github.com/prometheus/client_golang v1.23.2 github.com/redis/go-redis/v9 v9.22.0 github.com/tidwall/gjson v1.19.0 github.com/xeipuuv/gojsonschema v1.2.0 @@ -46,6 +47,7 @@ require ( k8s.io/apimachinery v0.36.3 mvdan.cc/sh/v3 v3.13.1 osv.dev/bindings/go v0.0.0-20260527044221-c1c72c1b0158 + sigs.k8s.io/yaml v1.6.0 ) require ( @@ -61,6 +63,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.4.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect github.com/charmbracelet/ultraviolet v0.0.0-20251205161215-1948445e3318 // indirect @@ -95,9 +98,13 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/muesli/cancelreader v0.2.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sergi/go-diff v1.4.0 // indirect github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect @@ -123,5 +130,4 @@ require ( google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go/go.sum b/go/go.sum index 4a67017042e..3ff99efaae3 100644 --- a/go/go.sum +++ b/go/go.sum @@ -52,6 +52,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -193,6 +195,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -211,6 +215,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nikolalohinski/gonja/v2 v2.9.0 h1:QICtNWj0siM3PF5xUjVod/l+C9XnGfI+wrfSIBGKQ6o= github.com/nikolalohinski/gonja/v2 v2.9.0/go.mod h1:UIzXPVuOsr5h7dZ5DUbqk3/Z7oFA/NLGQGMjqT4L2aU= github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= @@ -232,7 +238,15 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= diff --git a/go/internal/metrics/metrics.go b/go/internal/metrics/metrics.go new file mode 100644 index 00000000000..1b8ed51e7a3 --- /dev/null +++ b/go/internal/metrics/metrics.go @@ -0,0 +1,36 @@ +// Package metrics provides Prometheus metric definitions and exposition for OSV services. +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + // WorkerTasksProcessedTotal tracks the total number of tasks processed by the worker. + WorkerTasksProcessedTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "osv_worker_tasks_processed_total", + Help: "Total number of tasks processed by the worker.", + }, + []string{"status"}, + ) +) + +// TaskStatus represents the outcome of processing a worker task. +type TaskStatus string + +const ( + TaskStatusSuccess TaskStatus = "success" + TaskStatusError TaskStatus = "error" + TaskStatusSkipped TaskStatus = "skipped" + TaskStatusUnknown TaskStatus = "unknown" +) + +// RecordTaskProcessed increments the task processed counter for a given status. +func RecordTaskProcessed(status TaskStatus) { + if status == "" { + status = TaskStatusUnknown + } + WorkerTasksProcessedTotal.WithLabelValues(string(status)).Inc() +} diff --git a/go/internal/metrics/metrics_test.go b/go/internal/metrics/metrics_test.go new file mode 100644 index 00000000000..780c22ec6af --- /dev/null +++ b/go/internal/metrics/metrics_test.go @@ -0,0 +1,66 @@ +package metrics_test + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/google/osv.dev/go/internal/metrics" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func TestMetricsRecording(t *testing.T) { + metrics.RecordTaskProcessed(metrics.TaskStatusSuccess) + metrics.RecordTaskProcessed(metrics.TaskStatusError) + + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + rec := httptest.NewRecorder() + promhttp.Handler().ServeHTTP(rec, req) + + resp := rec.Result() + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("failed to read response body: %v", err) + } + bodyStr := string(body) + + expectedStrings := []string{ + `osv_worker_tasks_processed_total{status="success"}`, + `osv_worker_tasks_processed_total{status="error"}`, + } + + for _, exp := range expectedStrings { + if !strings.Contains(bodyStr, exp) { + t.Errorf("expected metrics output to contain %q, got:\n%s", exp, bodyStr) + } + } +} + +func TestMetricsServerLifecycle(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + srv := metrics.NewServer("127.0.0.1:0") + srv.Start(ctx) + + req := httptest.NewRequest(http.MethodGet, "/healthz", nil) + rec := httptest.NewRecorder() + srv.Handler().ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("expected status 200 from healthz, got %d", rec.Code) + } + + time.Sleep(50 * time.Millisecond) + cancel() + time.Sleep(50 * time.Millisecond) +} diff --git a/go/internal/metrics/server.go b/go/internal/metrics/server.go new file mode 100644 index 00000000000..ef8c4a7dd39 --- /dev/null +++ b/go/internal/metrics/server.go @@ -0,0 +1,76 @@ +// Package metrics provides Prometheus metric definitions and exposition for OSV services. +package metrics + +import ( + "context" + "errors" + "log/slog" + "net/http" + "time" + + "github.com/google/osv.dev/go/logger" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// DefaultPort is the standard metrics port scraped by GMP. +const DefaultPort = "9090" + +// Server wraps an HTTP server exposing Prometheus metrics. +type Server struct { + httpServer *http.Server +} + +// NewServer creates a new metrics server on the given address. +func NewServer(addr string) *Server { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + + return &Server{ + httpServer: &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + }, + } +} + +// Handler returns the underlying http.Handler. +func (s *Server) Handler() http.Handler { + return s.httpServer.Handler +} + +// Start runs the metrics server in a background goroutine and gracefully shuts down when ctx is done. +func (s *Server) Start(ctx context.Context) { + go func() { + logger.InfoContext(ctx, "Starting Prometheus metrics server on "+s.httpServer.Addr) + if err := s.httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.ErrorContext(ctx, "Metrics server error", slog.Any("error", err)) + } + }() + + go func() { + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + if err := s.httpServer.Shutdown(shutdownCtx); err != nil { + logger.ErrorContext(ctx, "Failed to gracefully shutdown metrics server", slog.Any("error", err)) + } + }() +} + +// StartMetricsServer is a convenience function that starts a metrics server on ":port" +// using DefaultPort if port is empty. +func StartMetricsServer(ctx context.Context, port string) *Server { + if port == "" { + port = DefaultPort + } + addr := ":" + port + srv := NewServer(addr) + srv.Start(ctx) + + return srv +} diff --git a/go/internal/worker/subscriber.go b/go/internal/worker/subscriber.go index dbffdbbe4bc..1bcfefd8fc3 100644 --- a/go/internal/worker/subscriber.go +++ b/go/internal/worker/subscriber.go @@ -8,6 +8,7 @@ import ( "time" "cloud.google.com/go/pubsub/v2" + "github.com/google/osv.dev/go/internal/metrics" "github.com/google/osv.dev/go/logger" "github.com/klauspost/compress/zstd" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -28,6 +29,7 @@ func (s *Subscriber) Run(ctx context.Context) error { func (s *Subscriber) handleMessage(ctx context.Context, m *pubsub.Message) { if taskType := m.Attributes["type"]; taskType != "update" { logger.InfoContext(ctx, "Skipping message, not an update", slog.Any("task_type", taskType)) + metrics.RecordTaskProcessed(metrics.TaskStatusSkipped) m.Ack() return @@ -50,6 +52,7 @@ func (s *Subscriber) handleMessage(ctx context.Context, m *pubsub.Message) { task.Vuln, err = s.parseVuln(m) if err != nil { logger.ErrorContext(taskCtx, "Failed to parse vulnerability", append(logInfo, slog.Any("error", err))...) + metrics.RecordTaskProcessed(metrics.TaskStatusError) m.Nack() return @@ -84,9 +87,11 @@ func (s *Subscriber) handleMessage(ctx context.Context, m *pubsub.Message) { } if err := s.Engine.RunTask(taskCtx, task); err != nil { + metrics.RecordTaskProcessed(metrics.TaskStatusError) logger.ErrorContext(taskCtx, "Failed to process task", append(logInfo, slog.Any("error", err))...) m.Nack() } else { + metrics.RecordTaskProcessed(metrics.TaskStatusSuccess) logTaskLatency(taskCtx, task) m.Ack() }