chore(stack/observability): Update and remove TLS from internal components for now

* chore(stack/observability): Update and remove TLS from internal components for now
  Also move from the loki exporter to the generic otlp exporter for shipping logs to Loki
* fix(stack/observability): Fix name of exporter
* chore(stack/observability): Move Loki OTLP exporter from gRPC to HTTP
  Loki doesn't support gRPC ingestion.
* chore(stack/observability): Disable Loki Canery
  It only generates noise which is not needed for the demo.
* chore(stack/obsevability): Remove commented out config for TLS
* chore(stack/obsevability): Add link for Loki schemaConfig

Author: NickLarsenNZ

stacks/observability/grafana-loki.yaml

@@ -5,21 +5,64 @@ name: loki
 repo:
   name: loki
   url: https://grafana.github.io/helm-charts
-version: 6.44.0 # 3.5.7
+# To find the latest version:
+# helm repo add grafana https://grafana.github.io/helm-charts --force-update
+# helm search repo grafana/loki
+version: 6.45.2 # 3.5.7
 options:
+  deploymentMode: SingleBinary
+  singleBinary:
+    replicas: 1
+
+  # Zero out replica counts of other deployment modes
+  backend:
+    replicas: 0
+  read:
+    replicas: 0
+  write:
+    replicas: 0
+
+  ingester:
+    replicas: 0
+  querier:
+    replicas: 0
+  queryFrontend:
+    replicas: 0
+  queryScheduler:
+    replicas: 0
+  distributor:
+    replicas: 0
+  compactor:
+    replicas: 0
+  indexGateway:
+    replicas: 0
+  bloomCompactor:
+    replicas: 0
+  bloomGateway:
+    replicas: 0
+
   loki:
     auth_enabled: false
     commonConfig:
       replication_factor: 1
     storage:
       type: 'filesystem'
-    server:
-      http_tls_config:
-        cert_file: /etc/loki/certs/tls.crt
-        key_file: /etc/loki/certs/tls.key
-    readinessProbe:
-      httpGet:
-        scheme: HTTPS
+      bucketNames:
+        # Buckets names from: https://github.com/grafana/loki/blob/015be223bcd2c29952db3f5cf0748414acf57fd3/production/helm/loki/values.yaml#L3677-L3686
+        chunks: chunks
+        ruler: ruler
+        admin: admin
+    schemaConfig:
+      # See: https://grafana.com/docs/loki/latest/operations/storage/schema/
+      configs:
+        - from: "2024-04-01"
+          store: tsdb
+          object_store: filesystem
+          schema: v13
+          index:
+            prefix: loki_index_
+            period: 24h
+
   monitoring:
     dashboards:
       enabled: false
@@ -35,31 +78,7 @@ options:
         installOperator: false
   test:
     enabled: false
+  lokiCanary:
+    enabled: false
   gateway:
     enabled: false
-  singleBinary:
-    replicas: 1
-    extraVolumeMounts:
-      # Mount the certificate generated by the secret-operator
-      - name: tls
-        mountPath: /etc/loki/certs/
-    extraVolumes:
-      # Request a TLS certificate from the secret-operator
-      - name: tls
-        ephemeral:
-          volumeClaimTemplate:
-            metadata:
-              annotations:
-                secrets.stackable.tech/class: tls
-                # Add the service loki to the
-                # distinguished names because this service is used
-                # by opentelemetry-collector.
-                secrets.stackable.tech/scope: |-
-                  service=loki
-            spec:
-              storageClassName: secrets.stackable.tech
-              accessModes:
-                - ReadWriteOnce
-              resources:
-                requests:
-                  storage: 1

stacks/observability/grafana-tempo.yaml

@@ -6,49 +6,17 @@ repo:
   name: tempo
   url: https://grafana.github.io/helm-charts
 # Watch out for https://github.com/grafana/helm-charts/issues/3417
-# and https://github.com/grafana/helm-charts/pull/3503!
+# Workaround at: https://github.com/grafana/helm-charts/issues/3660#issuecomment-3007113458
+# To find the latest version:
+# helm repo add grafana https://grafana.github.io/helm-charts --force-update
+# helm search repo grafana/tempo
 version: 1.24.0 # 2.9.0
 options:
   tempo:
-    server:
-      http_tls_config:
-        cert_file: /etc/tempo/certs/tls.crt
-        key_file: /etc/tempo/certs/tls.key
     receivers:
       otlp:
         protocols:
           grpc:
             endpoint: "0.0.0.0:4317"
-            tls:
-              cert_file: /etc/tempo/certs/tls.crt
-              key_file: /etc/tempo/certs/tls.key
-    extraVolumeMounts:
-      # Mount the certificate generated by the secret-operator
-      - name: tls
-        mountPath: /etc/tempo/certs/
   tempoQuery:
     enabled: true
-    extraVolumeMounts:
-      # Mount the certificate generated by the secret-operator
-      - name: tls
-        mountPath: /etc/tempo/certs/
-  extraVolumes:
-    # Request a TLS certificate from the secret-operator
-    - name: tls
-      ephemeral:
-        volumeClaimTemplate:
-          metadata:
-            annotations:
-              secrets.stackable.tech/class: tls
-              # Add the service loki to the
-              # distinguished names because this service is used
-              # by opentelemetry-collector.
-              secrets.stackable.tech/scope: |-
-                service=tempo
-          spec:
-            storageClassName: secrets.stackable.tech
-            accessModes:
-              - ReadWriteOnce
-            resources:
-              requests:
-                storage: 1

stacks/observability/grafana.yaml

@@ -6,34 +6,34 @@ name: grafana
 repo:
   name: grafana
   url: https://grafana.github.io/helm-charts
-version: 10.1.2 # 12.2.0
+# To find the latest version:
+# helm repo add grafana https://grafana.github.io/helm-charts --force-update
+# helm search repo grafana/grafana
+version: 10.1.4 # 12.2.1
 options:
   admin:
     existingSecret: grafana-admin-credentials
   service:
     type: NodePort
+    port: 443
   datasources:
     datasources.yaml:
       apiVersion: 1
       datasources:
         - name: Loki
           type: loki
-          url: https://loki.default.svc.cluster.local:3100
+          url: http://loki:3100
           access: proxy
           isDefault: false
           jsonData:
-            tlsAuthWithCACert: true
-          secureJsonData:
-            tlsCACert: $__file{/etc/grafana/certs/ca.crt}
+            tlsAuthWithCACert: false
         - name: Tempo
           type: tempo
-          url: https://tempo.default.svc.cluster.local:3100
+          url: http://tempo:3100
           access: proxy
           isDefault: false
           jsonData:
-            tlsAuthWithCACert: true
-          secureJsonData:
-            tlsCACert: $__file{/etc/grafana/certs/ca.crt}
+            tlsAuthWithCACert: false
   readinessProbe:
     httpGet:
       scheme: HTTPS
@@ -74,16 +74,3 @@ options:
         volumeAttributes:
           secrets.stackable.tech/class: tls
           secrets.stackable.tech/scope: node,pod,service=grafana
-      # ephemeral:
-      #   volumeClaimTemplate:
-      #     metadata:
-      #       annotations:
-      #         secrets.stackable.tech/class: tls
-      #         secrets.stackable.tech/scope: pod #,service=grafana
-      #     spec:
-      #       storageClassName: secrets.stackable.tech
-      #       accessModes:
-      #         - ReadWriteOnce
-      #       resources:
-      #         requests:
-      #           storage: 1

stacks/observability/jaeger.yaml

@@ -6,6 +6,9 @@ name: jaeger
 repo:
   name: jaeger
   url: https://jaegertracing.github.io/helm-charts
+# To find the latest version:
+# helm repo add jaegertracing https://jaegertracing.github.io/helm-charts --force-update
+# helm search repo jaegertracing/jaeger
 version: 3.4.1 # 1.53.0
 options:
   # labels:

stacks/observability/opentelemetry-collector-deployment.yaml

@@ -9,45 +9,46 @@ metadata:
 spec:
   mode: deployment
   # See: https://github.com/open-telemetry/opentelemetry-collector-releases/pkgs/container/opentelemetry-collector-releases%2Fopentelemetry-collector-contrib/versions?filters%5Bversion_type%5D=tagged
+  # See: https://github.com/open-telemetry/opentelemetry-collector-releases/pkgs/container/opentelemetry-collector-releases%2Fopentelemetry-collector-contrib/550448680?tag=latest
   # Using the contrib image so we get all of the useful plugins
-  image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.97.0
+  image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.138.0
   config:
-    # Configure the collector to receove via OTLP, and export to various "sinks".
+    # Configure the collector to receive via OTLP, and export to various "sinks".
     # For configuration structure, see: https://opentelemetry.io/docs/collector/configuration/
     #
     # A receiver for logs/metrics/traces via OTLP
     # See all available receivers:
     # - https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
     # - https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver
     receivers:
-      otlp/insecure:
+      otlp:
         protocols:
           grpc:
             # In sidecar mode, we don't want, nor need the sidecar to be
             # accessible from outside the pod. For that reason, we do not enable
             # TLS on the receiver(s) because only containers in the same pod can
             # connect, and so network traffic will not leave the host.
             endpoint: 0.0.0.0:4317
-    # Exporters to separate logs/metrics/taces
+    # Exporters to separate logs/metrics/traces
     # See all available exporters:
     # - https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter
     # - https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter
     exporters:
       # When added to a logs/metrics/traces pipeline, the collector will print to stdout.
-      debug:
+      debug: {}
         # verbosity: detailed
-      otlp/jaeger-insecure:
+      otlp/jaeger:
         endpoint: jaeger-collector:4317
         tls:
           insecure: true
       otlp/tempo:
-        endpoint: tempo.default.svc.cluster.local:4317
+        endpoint: tempo:4317
         tls:
-          ca_file: /certs/ca.crt
-      loki:
-        endpoint: https://loki.default.svc.cluster.local:3100/loki/api/v1/push
+          insecure: true
+      otlphttp/loki:
+        endpoint: http://loki:3100/otlp
         tls:
-          ca_file: /certs/ca.crt
+          insecure: true
       #     auth:
       #       authenticator: basicauth/loki
     # Processors to enrich logs/metrics/traces. For example, adding Kubernetes node/pod/deployment information
@@ -98,7 +99,7 @@ spec:
     # See all available extensions:
     # - https://github.com/open-telemetry/opentelemetry-collector/tree/main/extension
     # - https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension
-    extensions:
+    extensions: {}
       # basicauth/loki:
       #   client_auth:
       #     username: loki
@@ -108,28 +109,29 @@ spec:
       # extensions: [basicauth/loki]
       pipelines:
         traces:
-          receivers: [otlp/insecure]
+          receivers: [otlp]
           processors:
           - memory_limiter
           - batch/every-second
           - resourcedetection/env
           exporters:
+          # - otlp/jaeger
           - otlp/tempo
           # - debug
         logs:
-          receivers: [otlp/insecure]
+          receivers: [otlp]
           processors:
           - memory_limiter
           - batch/every-second
           - resourcedetection/env
           - resource/logs
           - attributes/logs
           exporters:
-          - loki
+          - otlphttp/loki
           # - debug
       # Set the log level of the opentelemetry-collector application.
       # You can also expose the collector metrcics (separate from the metrics receivers/exporters).
-      telemetry:
+      telemetry: {}
         # logs:
         #   level: DEBUG
         # metrics:
@@ -142,22 +144,3 @@ spec:
     requests:
       cpu: 250m
       memory: 64Mi
-  volumeMounts:
-    # We mount a certificate from the secret-operator so that we have the CA
-    # cert for exporters to trust what they connect to.
-    - name: tls
-      mountPath: /certs
-  volumes:
-    - name: tls
-      ephemeral:
-        volumeClaimTemplate:
-          metadata:
-            annotations:
-              secrets.stackable.tech/class: tls
-          spec:
-            storageClassName: secrets.stackable.tech
-            accessModes:
-              - ReadWriteOnce
-            resources:
-              requests:
-                storage: 1