HYPERFLEET-1410 - feat: expose operator metrics, health probes, and ServiceMonitor

README.md

@@ -49,6 +49,19 @@ kubectl apply -k config/samples/
 
 >**NOTE**: Ensure that the samples has default values to test it out.
 
+### Observability endpoints
+
+The manager exposes the standard HyperFleet observability endpoints (defaults):
+
+- **Liveness probe:**  `http://localhost:8080/healthz`
+- **Readiness probe:** `http://localhost:8080/readyz`
+- **Metrics:**         `http://localhost:9090/metrics`
+
+Metrics are served as plain HTTP under the `hyperfleet_operator_*` namespace. Ports
+are configurable via `--health-probe-bind-address` and `--metrics-bind-address`.
+See [docs/metrics.md](docs/metrics.md) for the full metric catalogue, labels, and
+example PromQL queries.
+
 ### To Uninstall
 **Delete the instances (CRs) from the cluster:**
 

bundle/manifests/hyperfleet-operator-controller-manager-metrics-service_v1_service.yaml

@@ -9,10 +9,10 @@ metadata:
   name: hyperfleet-operator-controller-manager-metrics-service
 spec:
   ports:
-  - name: https
-    port: 8443
+  - name: metrics
+    port: 9090
     protocol: TCP
-    targetPort: 8443
+    targetPort: metrics
   selector:
     app.kubernetes.io/name: hyperfleet-operator
     control-plane: controller-manager

bundle/manifests/hyperfleet-operator.clusterserviceversion.yaml

@@ -33,7 +33,7 @@ metadata:
         }
       ]
     capabilities: Basic Install
-    createdAt: "2026-09-01T18:11:57Z"
+    createdAt: "2026-09-03T22:06:02Z"
     operators.operatorframework.io/builder: operator-sdk-v1.42.3
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v4
   name: hyperfleet-operator.v0.0.1
@@ -158,9 +158,10 @@ spec:
             spec:
               containers:
               - args:
-                - --metrics-bind-address=:8443
                 - --leader-elect
-                - --health-probe-bind-address=:8081
+                - --health-probe-bind-address=:8080
+                - --metrics-bind-address=:9090
+                - --metrics-secure=false
                 command:
                 - /manager
                 env:
@@ -172,18 +173,29 @@ spec:
                   value: quay.io/redhat-services-prod/hyperfleet-tenant/hyperfleet/hyperfleet-api:latest
                 image: quay.io/redhat-services-prod/hyperfleet-tenant/hyperfleet/hyperfleet-operator:latest
                 livenessProbe:
+                  failureThreshold: 3
                   httpGet:
                     path: /healthz
-                    port: 8081
+                    port: 8080
                   initialDelaySeconds: 15
                   periodSeconds: 20
+                  timeoutSeconds: 5
                 name: manager
+                ports:
+                - containerPort: 9090
+                  name: metrics
+                  protocol: TCP
+                - containerPort: 8080
+                  name: health
+                  protocol: TCP
                 readinessProbe:
+                  failureThreshold: 3
                   httpGet:
                     path: /readyz
-                    port: 8081
+                    port: 8080
                   initialDelaySeconds: 5
                   periodSeconds: 10
+                  timeoutSeconds: 3
                 resources:
                   limits:
                     cpu: 500m
@@ -201,7 +213,7 @@ spec:
                 seccompProfile:
                   type: RuntimeDefault
               serviceAccountName: hyperfleet-operator-controller-manager
-              terminationGracePeriodSeconds: 10
+              terminationGracePeriodSeconds: 30
       permissions:
       - rules:
         - apiGroups:
@@ -243,6 +255,17 @@ spec:
           - get
           - list
           - watch
+        - apiGroups:
+          - monitoring.coreos.com
+          resources:
+          - servicemonitors
+          verbs:
+          - create
+          - get
+          - list
+          - patch
+          - update
+          - watch
         serviceAccountName: hyperfleet-operator-controller-manager
     strategy: deployment
   installModes:

cmd/main.go

@@ -43,6 +43,8 @@ import (
 
 	hyperfleetv1alpha1 "github.com/openshift-hyperfleet/hyperfleet-operator/api/v1alpha1"
 	"github.com/openshift-hyperfleet/hyperfleet-operator/internal/controller"
+	"github.com/openshift-hyperfleet/hyperfleet-operator/internal/metrics"
+	"github.com/openshift-hyperfleet/hyperfleet-operator/internal/servicemonitor"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -68,14 +70,18 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 	var tlsOpts []func(*tls.Config)
-	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
-		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
-	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	// Defaults follow the HyperFleet health-endpoints / metrics standards, matching
+	// the API, Sentinel and Adapter components: metrics on :9090 over plain HTTP at
+	// /metrics, health/readiness on :8080. Set 0 on metrics-bind-address to disable.
+	flag.StringVar(&metricsAddr, "metrics-bind-address", ":9090", "The address the metrics endpoint binds to. "+
+		"Defaults to :9090 (HyperFleet standard). Set to 0 to disable the metrics service.")
+	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8080", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	flag.BoolVar(&secureMetrics, "metrics-secure", true,
-		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	flag.BoolVar(&secureMetrics, "metrics-secure", false,
+		"If set, the metrics endpoint is served securely via HTTPS with authn/authz. "+
+			"The HyperFleet standard scrapes plain HTTP, so this defaults to false.")
 	flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.")
 	flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.")
 	flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.")
@@ -93,6 +99,10 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// Publish the build-info and up metrics into controller-runtime's registry so
+	// they are exposed on the same /metrics endpoint as the reconcile metrics.
+	metrics.Init()
+
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
 	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
@@ -256,6 +266,19 @@ func main() {
 	}
 	// +kubebuilder:scaffold:builder
 
+	// Create the operator's own ServiceMonitor at runtime, but only when the
+	// Prometheus Operator API is present. Shipping it in the OLM bundle would fail
+	// the InstallPlan on clusters without the monitoring.coreos.com CRD and block
+	// the operator install; this bootstrapper degrades gracefully instead. See the
+	// servicemonitor package doc for the full rationale.
+	if err := mgr.Add(&servicemonitor.Bootstrapper{
+		Config:    mgr.GetConfig(),
+		Namespace: operatorNamespace,
+	}); err != nil {
+		setupLog.Error(err, "unable to add ServiceMonitor bootstrapper")
+		os.Exit(1)
+	}
+
 	if metricsCertWatcher != nil {
 		setupLog.Info("Adding metrics certificate watcher to manager")
 		if err := mgr.Add(metricsCertWatcher); err != nil {

config/default/kustomization.yaml

@@ -38,13 +38,9 @@ resources:
 # be able to communicate with the Webhook Server.
 #- ../network-policy
 
-# Uncomment the patches line if you enable Metrics
-patches:
-# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
-# More info: https://book.kubebuilder.io/reference/metrics
-- path: manager_metrics_patch.yaml
-  target:
-    kind: Deployment
+# The metrics endpoint (:9090, plain HTTP, /metrics) and health probes (:8080)
+# are configured directly on the manager Deployment (config/manager/manager.yaml)
+# per the HyperFleet standard, so no metrics args patch is needed here.
 
 # Uncomment the patches line if you enable Metrics and CertManager
 # [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.

config/default/metrics_service.yaml

@@ -9,10 +9,10 @@ metadata:
   namespace: system
 spec:
   ports:
-  - name: https
-    port: 8443
+  - name: metrics
+    port: 9090
     protocol: TCP
-    targetPort: 8443
+    targetPort: metrics
   selector:
     control-plane: controller-manager
     app.kubernetes.io/name: hyperfleet-operator

config/manager/manager.yaml

@@ -62,7 +62,11 @@ spec:
         - /manager
         args:
           - --leader-elect
-          - --health-probe-bind-address=:8081
+          # HyperFleet standard: metrics on :9090 (plain HTTP, /metrics) and
+          # health/readiness on :8080, matching the other components.
+          - --health-probe-bind-address=:8080
+          - --metrics-bind-address=:9090
+          - --metrics-secure=false
         image: controller:latest
         name: manager
         env:
@@ -72,24 +76,35 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        ports: []
+        ports:
+        - name: metrics
+          containerPort: 9090
+          protocol: TCP
+        - name: health
+          containerPort: 8080
+          protocol: TCP
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
             drop:
             - "ALL"
+        # Probe timing follows the HyperFleet health-endpoints standard.
         livenessProbe:
           httpGet:
             path: /healthz
-            port: 8081
+            port: 8080
           initialDelaySeconds: 15
           periodSeconds: 20
+          timeoutSeconds: 5
+          failureThreshold: 3
         readinessProbe:
           httpGet:
             path: /readyz
-            port: 8081
+            port: 8080
           initialDelaySeconds: 5
           periodSeconds: 10
+          timeoutSeconds: 3
+          failureThreshold: 3
         # TODO(user): Configure the resources accordingly based on the project requirements.
         # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
         resources:
@@ -102,4 +117,4 @@ spec:
         volumeMounts: []
       volumes: []
       serviceAccountName: controller-manager
-      terminationGracePeriodSeconds: 10
+      terminationGracePeriodSeconds: 30

config/manifests/kustomization.yaml

@@ -5,6 +5,14 @@ resources:
 - ../default
 - ../samples
 - ../scorecard
+# The Prometheus ServiceMonitor is deliberately NOT included in the bundle. OLM
+# applies a bundle's arbitrary manifests but does not install the CRDs they need,
+# so bundling the ServiceMonitor would fail the InstallPlan — and block the whole
+# operator install — on clusters without the Prometheus Operator CRD. HyperFleet
+# targets generic Kubernetes, so the operator creates the ServiceMonitor itself at
+# runtime only when the monitoring.coreos.com/v1 API is present (see the
+# internal/servicemonitor package). config/prometheus remains an optional GitOps
+# overlay for users who prefer to apply it statically.
 
 # [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix.
 # Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager.

config/network-policy/allow-metrics-traffic.yaml

@@ -23,5 +23,5 @@ spec:
           matchLabels:
             metrics: enabled  # Only from namespaces with this label
       ports:
-        - port: 8443
+        - port: 9090
           protocol: TCP

config/prometheus/monitor.yaml

@@ -1,4 +1,6 @@
 # Prometheus Monitor Service (Metrics)
+# Scrapes the operator's plain-HTTP metrics endpoint on :9090, per the HyperFleet
+# metrics standard (same exposition as the API, Sentinel and Adapter components).
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -11,16 +13,9 @@ metadata:
 spec:
   endpoints:
     - path: /metrics
-      port: https # Ensure this is the name of the port that exposes HTTPS metrics
-      scheme: https
-      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-      tlsConfig:
-        # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
-        # certificate verification, exposing the system to potential man-in-the-middle attacks.
-        # For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
-        # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
-        # which securely references the certificate from the 'metrics-server-cert' secret.
-        insecureSkipVerify: true
+      port: metrics # matches the metrics Service port name
+      scheme: http
+      interval: 30s
   selector:
     matchLabels:
       control-plane: controller-manager

config/rbac/role.yaml

@@ -81,3 +81,14 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - servicemonitors
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch