Add tracing to clients

Author: yorickdowne

besu.yml

@@ -28,6 +28,7 @@ services:
       - NODE_TYPE=${EL_NODE_TYPE:-pre-merge-expiry}
       - NETWORK=${NETWORK}
       - IPV6=${IPV6:-false}
+      - COMPOSE_FILE=${COMPOSE_FILE}
     volumes:
       - besu-el-data:/var/lib/besu
       - besu-eth1-data:/var/lib/besu-og

besu/docker-entrypoint.sh

@@ -123,6 +123,14 @@ fi
 __strip_empty_args "$@"
 set -- "${__args[@]}"
 
+# Traces
+if [[ "${COMPOSE_FILE}" =~ (grafana\.ymlr|grafana-rootless\.yml) ]]; then
+  export OTEL_EXPORTER_OTLP_PROTOCOL=grpc
+  export OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317
+  export OTEL_EXPORTER_OTLP_INSECURE=true
+  export OTEL_SERVICE_NAME=besu
+fi
+
 if [[ -f /var/lib/besu/prune-history-marker ]]; then
   rm -f /var/lib/besu/prune-history-marker
   if [[ "${NODE_TYPE}" = "archive" ]]; then

lighthouse-cl-only.yml

@@ -41,6 +41,7 @@ services:
       - CL_P2P_PORT=${CL_P2P_PORT:-9000}
       - CL_QUIC_PORT=${CL_QUIC_PORT:-9001}
       - NETWORK=${NETWORK}
+      - COMPOSE_FILE=${COMPOSE_FILE}
     ports:
       - ${HOST_IP:-}:${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/tcp
       - ${HOST_IP:-}:${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/udp

lighthouse.yml

@@ -51,6 +51,7 @@ services:
       - CL_P2P_PORT=${CL_P2P_PORT:-9000}
       - CL_QUIC_PORT=${CL_QUIC_PORT:-9001}
       - NETWORK=${NETWORK}
+      - COMPOSE_FILE=${COMPOSE_FILE}
     ports:
       - ${HOST_IP:-}:${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/tcp
       - ${HOST_IP:-}:${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/udp

lighthouse/docker-entrypoint.sh

@@ -116,6 +116,17 @@ fi
 __strip_empty_args "$@"
 set -- "${__args[@]}"
 
+# Traces
+if [[ "${COMPOSE_FILE}" =~ (grafana\.ymlr|grafana-rootless\.yml) ]]; then
+  __trace="--telemetry-collector-url http://tempo:4317 --telemetry-service-name lighthouse"
+# These may become default in future. Here so Lighthouse doesn't murder itself in the meantime
+  export OTEL_TRACES_SAMPLER=parentbased_traceidratio
+  export OTEL_TRACES_SAMPLER_ARG=0.01
+  export OTEL_EXPORTER_OTLP_INSECURE=true
+else
+  __trace=""
+fi
+
 if [[ -f /var/lib/lighthouse/beacon/prune-marker ]]; then
   rm -f /var/lib/lighthouse/beacon/prune-marker
   if [[ "${NODE_TYPE}" = "archive" ]]; then
@@ -128,5 +139,5 @@ if [[ -f /var/lib/lighthouse/beacon/prune-marker ]]; then
 else
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086
-  exec "$@" ${__network} ${__mev_boost} ${__checkpoint_sync} ${__prune} ${__beacon_stats} ${__ipv6} ${CL_EXTRAS}
+  exec "$@" ${__network} ${__mev_boost} ${__checkpoint_sync} ${__prune} ${__beacon_stats} ${__trace} ${__ipv6} ${CL_EXTRAS}
 fi

reth.yml

@@ -29,6 +29,7 @@ services:
       - NODE_TYPE=${EL_NODE_TYPE:-pre-merge-expiry}
       - NETWORK=${NETWORK}
       - STATIC_DIR=${ANCIENT_DIR}
+      - COMPOSE_FILE=${COMPOSE_FILE}
       # Make this RUST_LOG=${LOG_LEVEL:-info},engine=trace when requiring deep debug
       # RPC debug can be done with jsonrpsee=trace or jsonrpsee::target=trace for a specific target
       # - RUST_LOG=${LOG_LEVEL:-info},engine=trace

reth/docker-entrypoint.sh

@@ -139,6 +139,15 @@ fi
 __strip_empty_args "$@"
 set -- "${__args[@]}"
 
+# Traces
+if [[ "${COMPOSE_FILE}" =~ (grafana\.ymlr|grafana-rootless\.yml) ]]; then
+  __trace="--tracing-otlp=http://tempo:4318/v1/traces"
+  export OTEL_EXPORTER_OTLP_INSECURE=true
+  export OTEL_SERVICE_NAME=reth
+else
+  __trace=""
+fi
+
 if [[ -f /var/lib/reth/prune-marker ]]; then
   rm -f /var/lib/reth/prune-marker
   if [[ "${NODE_TYPE}" = "archive" ]]; then
@@ -151,5 +160,5 @@ if [[ -f /var/lib/reth/prune-marker ]]; then
 else
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086
-  exec "$@" ${__network} ${__verbosity} ${__static} ${__prune} ${EL_EXTRAS}
+  exec "$@" ${__network} ${__verbosity} ${__static} ${__prune} ${__trace} ${EL_EXTRAS}
 fi

ssv.yml

@@ -23,6 +23,11 @@ services:
       - CONFIG_PATH=/config/config.yaml
       - HOME=/tmp
       - ENABLE_DOPPELGANGER_PROTECTION=${DOPPELGANGER:-false}
+# This will throw errors if tempo is not running. An alternative would be an entrypoint script for ssv node
+      - OTEL_EXPORTER_OTLP_PROTOCOL=grpc
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317
+      - OTEL_EXPORTER_OTLP_INSECURE=true
+      - OTEL_SERVICE_NAME=ssv-node
     command: make BUILD_PATH=/go/bin/ssvnode start-node
     labels:
       - metrics.scrape=true

vero-vc-only.yml

@@ -36,6 +36,7 @@ services:
       - NETWORK=${NETWORK}
       - CL_NODE=${CL_NODE}
       - LOG_LEVEL=${LOG_LEVEL}
+      - COMPOSE_FILE=${COMPOSE_FILE}
     volumes:
       - vero-data:/var/lib/vero
       - /etc/localtime:/etc/localtime:ro

vero/docker-entrypoint.sh

@@ -100,6 +100,14 @@ fi
 # Uppercase log level
 __log_level="--log-level ${LOG_LEVEL^^}"
 
+# Traces
+if [[ "${COMPOSE_FILE}" =~ (grafana\.ymlr|grafana-rootless\.yml) ]]; then
+  export OTEL_EXPORTER_OTLP_PROTOCOL=grpc
+  export OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317
+  export OTEL_EXPORTER_OTLP_INSECURE=true
+  export OTEL_SERVICE_NAME=vero
+fi
+
 if [[ "${DEFAULT_GRAFFITI}" = "true" ]]; then
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086