feat(tracing): enrich OTel resource, compress OTLP, annotate spans

cmd/bee/cmd/cmd.go

@@ -40,10 +40,10 @@ const (
 	optionWelcomeMessage                   = "welcome-message"
 	optionCORSAllowedOrigins               = "cors-allowed-origins"
 	optionNameTracingEnabled               = "tracing-enable"
-	optionNameTracingOTLPEndpoint          = "tracing-otlp-endpoint"
-	optionNameTracingOTLPInsecure          = "tracing-otlp-insecure"
-	optionNameTracingOTLPCAFile            = "tracing-otlp-ca-file"
-	optionNameTracingOTLPProtocol          = "tracing-otlp-protocol"
+	optionNameTracingEndpoint              = "tracing-endpoint"
+	optionNameTracingInsecure              = "tracing-insecure"
+	optionNameTracingCAFile                = "tracing-ca-file"
+	optionNameTracingProtocol              = "tracing-protocol"
 	optionNameTracingSamplingRatio         = "tracing-sampling-ratio"
 	optionNameTracingServiceName           = "tracing-service-name"
 	optionNameVerbosity                    = "verbosity"
@@ -108,10 +108,10 @@ const (
 
 	// tracing
 	configKeyTracingEnabled       = "tracing.enable"
-	configKeyTracingOTLPEndpoint  = "tracing.otlp-endpoint"
-	configKeyTracingOTLPInsecure  = "tracing.otlp-insecure"
-	configKeyTracingOTLPCAFile    = "tracing.otlp-ca-file"
-	configKeyTracingOTLPProtocol  = "tracing.otlp-protocol"
+	configKeyTracingEndpoint      = "tracing.endpoint"
+	configKeyTracingInsecure      = "tracing.insecure"
+	configKeyTracingCAFile        = "tracing.ca-file"
+	configKeyTracingProtocol      = "tracing.protocol"
 	configKeyTracingSamplingRatio = "tracing.sampling-ratio"
 	configKeyTracingServiceName   = "tracing.service-name"
 )
@@ -126,28 +126,30 @@ var blockchainRpcConfigPairs = []struct{ flat, dotted string }{
 
 var tracingConfigPairs = []struct{ flat, dotted string }{
 	{optionNameTracingEnabled, configKeyTracingEnabled},
-	{optionNameTracingOTLPEndpoint, configKeyTracingOTLPEndpoint},
-	{optionNameTracingOTLPInsecure, configKeyTracingOTLPInsecure},
-	{optionNameTracingOTLPCAFile, configKeyTracingOTLPCAFile},
-	{optionNameTracingOTLPProtocol, configKeyTracingOTLPProtocol},
+	{optionNameTracingEndpoint, configKeyTracingEndpoint},
+	{optionNameTracingInsecure, configKeyTracingInsecure},
+	{optionNameTracingCAFile, configKeyTracingCAFile},
+	{optionNameTracingProtocol, configKeyTracingProtocol},
 	{optionNameTracingSamplingRatio, configKeyTracingSamplingRatio},
 	{optionNameTracingServiceName, configKeyTracingServiceName},
 }
 
 // Deprecated tracing options, removed in the OpenTelemetry migration. They are
 // kept only as hidden no-op flags so that existing configs do not break node
 // startup; their values are ignored (see deprecatedTracingKeys).
+//
+// Note: tracing-endpoint is intentionally NOT here — it is reused as the active
+// OTLP endpoint flag (see optionNameTracingEndpoint). Its meaning changed from a
+// Jaeger agent address to an OTLP collector endpoint.
 const (
-	optionNameTracingEndpoint = "tracing-endpoint"
-	optionNameTracingHost     = "tracing-host"
-	optionNameTracingPort     = "tracing-port"
+	optionNameTracingHost = "tracing-host"
+	optionNameTracingPort = "tracing-port"
 )
 
 // deprecatedTracingKeys are the pre-OpenTelemetry tracing options. They are
 // registered as hidden no-op flags so stale configs still start the node, but
-// their values are ignored; operators should migrate to the tracing-otlp-* keys.
+// their values are ignored; operators should migrate to the tracing-* keys.
 var deprecatedTracingKeys = []string{
-	optionNameTracingEndpoint,
 	optionNameTracingHost,
 	optionNameTracingPort,
 }
@@ -326,17 +328,17 @@ func (c *command) setAllFlags(cmd *cobra.Command) {
 	cmd.Flags().Uint64(optionNameNetworkID, chaincfg.Mainnet.NetworkID, "ID of the Swarm network")
 	cmd.Flags().StringSlice(optionCORSAllowedOrigins, []string{}, "origins with CORS headers enabled")
 	cmd.Flags().Bool(optionNameTracingEnabled, false, "enable tracing")
-	cmd.Flags().String(optionNameTracingOTLPEndpoint, "127.0.0.1:4318", "OTLP endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc")
-	cmd.Flags().Bool(optionNameTracingOTLPInsecure, false, "disable TLS for the OTLP exporter (useful for a local collector); when false, set --tracing-otlp-ca-file to verify the collector certificate against a private CA")
-	cmd.Flags().String(optionNameTracingOTLPCAFile, "", "path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when --tracing-otlp-insecure=true")
-	cmd.Flags().String(optionNameTracingOTLPProtocol, "http", "OTLP exporter transport: http or grpc")
+	cmd.Flags().String(optionNameTracingEndpoint, "127.0.0.1:4318", "OTLP collector endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc")
+	cmd.Flags().Bool(optionNameTracingInsecure, false, "disable TLS for the OTLP exporter (useful for a local collector); when false, set --tracing-ca-file to verify the collector certificate against a private CA")
+	cmd.Flags().String(optionNameTracingCAFile, "", "path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when --tracing-insecure=true")
+	cmd.Flags().String(optionNameTracingProtocol, "http", "OTLP exporter transport: http or grpc")
 	cmd.Flags().Float64(optionNameTracingSamplingRatio, 1.0, "head-based sampling ratio in [0,1]; 0 samples nothing, 1 samples everything")
 	cmd.Flags().String(optionNameTracingServiceName, "bee", "service name identifier for tracing")
 	// Deprecated, no-op tracing flags kept for backward compatibility so that
 	// existing configs do not break node startup. Their values are ignored.
 	for _, name := range deprecatedTracingKeys {
-		cmd.Flags().String(name, "", "deprecated and ignored; use the tracing-otlp-* options")
-		_ = cmd.Flags().MarkDeprecated(name, "no longer used after the OpenTelemetry migration; use the tracing-otlp-* options")
+		cmd.Flags().String(name, "", "deprecated and ignored; use the tracing-* options")
+		_ = cmd.Flags().MarkDeprecated(name, "no longer used after the OpenTelemetry migration; use the tracing-* options")
 	}
 	cmd.Flags().String(optionNameVerbosity, "info", "log verbosity level 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace")
 	cmd.Flags().String(optionWelcomeMessage, "", "send a welcome message string during handshakes")
@@ -423,8 +425,8 @@ func (c *command) bindBlockchainRpcConfig(cmd *cobra.Command) {
 	c.bindNestedConfig(cmd, blockchainRpcConfigPairs)
 }
 
-// bindTracingConfig supports both flat (tracing-otlp-endpoint) and nested
-// (tracing.otlp-endpoint) YAML forms, with nested taking precedence.
+// bindTracingConfig supports both flat (tracing-endpoint) and nested
+// (tracing.endpoint) YAML forms, with nested taking precedence.
 func (c *command) bindTracingConfig(cmd *cobra.Command) {
 	c.bindNestedConfig(cmd, tracingConfigPairs)
 }
@@ -436,7 +438,7 @@ func (c *command) bindTracingConfig(cmd *cobra.Command) {
 func (c *command) warnDeprecatedTracingKeys() {
 	for _, key := range deprecatedTracingKeys {
 		if c.config.InConfig(key) {
-			c.logger.Warning("deprecated tracing config key is ignored; use the tracing-otlp-* options", "key", key)
+			c.logger.Warning("deprecated tracing config key is ignored; use the tracing-* options", "key", key)
 		}
 	}
 }
@@ -445,9 +447,9 @@ func (c *command) warnDeprecatedTracingKeys() {
 // CA bundle, in which case the OTLP exporter falls back to the system root CAs.
 func (c *command) warnTracingTLSWithoutCA() {
 	if c.config.GetBool(configKeyTracingEnabled) &&
-		!c.config.GetBool(configKeyTracingOTLPInsecure) &&
-		c.config.GetString(configKeyTracingOTLPCAFile) == "" {
-		c.logger.Warning("tracing: TLS is enabled but no CA bundle is configured; the OTLP exporter will rely on the system root CAs. Provide --tracing-otlp-ca-file, set --tracing-otlp-insecure=true for a plaintext local collector, or disable tracing.")
+		!c.config.GetBool(configKeyTracingInsecure) &&
+		c.config.GetString(configKeyTracingCAFile) == "" {
+		c.logger.Warning("tracing: TLS is enabled but no CA bundle is configured; the OTLP exporter will rely on the system root CAs. Provide --tracing-ca-file, set --tracing-insecure=true for a plaintext local collector, or disable tracing.")
 	}
 }
 

cmd/bee/cmd/start.go

@@ -343,10 +343,10 @@ func buildBeeNode(ctx context.Context, c *command, cmd *cobra.Command, logger lo
 		SwapInitialDeposit:            c.config.GetString(optionNameSwapInitialDeposit),
 		TargetNeighborhood:            c.config.GetString(optionNameTargetNeighborhood),
 		TracingEnabled:                c.config.GetBool(configKeyTracingEnabled),
-		TracingEndpoint:               c.config.GetString(configKeyTracingOTLPEndpoint),
-		TracingInsecure:               c.config.GetBool(configKeyTracingOTLPInsecure),
-		TracingCAFile:                 c.config.GetString(configKeyTracingOTLPCAFile),
-		TracingProtocol:               c.config.GetString(configKeyTracingOTLPProtocol),
+		TracingEndpoint:               c.config.GetString(configKeyTracingEndpoint),
+		TracingInsecure:               c.config.GetBool(configKeyTracingInsecure),
+		TracingCAFile:                 c.config.GetString(configKeyTracingCAFile),
+		TracingProtocol:               c.config.GetString(configKeyTracingProtocol),
 		TracingSamplingRatio:          c.config.GetFloat64(configKeyTracingSamplingRatio),
 		TracingServiceName:            c.config.GetString(configKeyTracingServiceName),
 		TrxDebugMode:                  c.config.GetBool(optionNameTransactionDebugMode),

packaging/bee.yaml

@@ -111,14 +111,14 @@ password-file: "/var/lib/bee/password"
 # tracing:
 #   ## enable tracing
 #   enable: false
-#   ## OTLP endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
-#   otlp-endpoint: 127.0.0.1:4318
+#   ## OTLP collector endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
+#   endpoint: 127.0.0.1:4318
 #   ## disable TLS for the OTLP exporter (useful for a local collector)
-#   otlp-insecure: false
-#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when otlp-insecure is true
-#   otlp-ca-file: ""
+#   insecure: false
+#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when insecure is true
+#   ca-file: ""
 #   ## OTLP exporter transport: http or grpc
-#   otlp-protocol: http
+#   protocol: http
 #   ## head-based sampling ratio in [0,1]; 0 samples nothing, 1 samples everything
 #   sampling-ratio: 1.0
 #   ## service name identifier for tracing

packaging/homebrew-amd64/bee.yaml

@@ -111,14 +111,14 @@ password-file: "/usr/local/var/lib/swarm-bee/password"
 # tracing:
 #   ## enable tracing
 #   enable: false
-#   ## OTLP endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
-#   otlp-endpoint: 127.0.0.1:4318
+#   ## OTLP collector endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
+#   endpoint: 127.0.0.1:4318
 #   ## disable TLS for the OTLP exporter (useful for a local collector)
-#   otlp-insecure: false
-#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when otlp-insecure is true
-#   otlp-ca-file: ""
+#   insecure: false
+#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when insecure is true
+#   ca-file: ""
 #   ## OTLP exporter transport: http or grpc
-#   otlp-protocol: http
+#   protocol: http
 #   ## head-based sampling ratio in [0,1]; 0 samples nothing, 1 samples everything
 #   sampling-ratio: 1.0
 #   ## service name identifier for tracing

packaging/homebrew-arm64/bee.yaml

@@ -111,14 +111,14 @@ password-file: "/opt/homebrew/var/lib/swarm-bee/password"
 # tracing:
 #   ## enable tracing
 #   enable: false
-#   ## OTLP endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
-#   otlp-endpoint: 127.0.0.1:4318
+#   ## OTLP collector endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
+#   endpoint: 127.0.0.1:4318
 #   ## disable TLS for the OTLP exporter (useful for a local collector)
-#   otlp-insecure: false
-#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when otlp-insecure is true
-#   otlp-ca-file: ""
+#   insecure: false
+#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when insecure is true
+#   ca-file: ""
 #   ## OTLP exporter transport: http or grpc
-#   otlp-protocol: http
+#   protocol: http
 #   ## head-based sampling ratio in [0,1]; 0 samples nothing, 1 samples everything
 #   sampling-ratio: 1.0
 #   ## service name identifier for tracing

packaging/scoop/bee.yaml

@@ -111,14 +111,14 @@ password-file: "./password"
 # tracing:
 #   ## enable tracing
 #   enable: false
-#   ## OTLP endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
-#   otlp-endpoint: 127.0.0.1:4318
+#   ## OTLP collector endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
+#   endpoint: 127.0.0.1:4318
 #   ## disable TLS for the OTLP exporter (useful for a local collector)
-#   otlp-insecure: false
-#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when otlp-insecure is true
-#   otlp-ca-file: ""
+#   insecure: false
+#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when insecure is true
+#   ca-file: ""
 #   ## OTLP exporter transport: http or grpc
-#   otlp-protocol: http
+#   protocol: http
 #   ## head-based sampling ratio in [0,1]; 0 samples nothing, 1 samples everything
 #   sampling-ratio: 1.0
 #   ## service name identifier for tracing

pkg/api/api.go

@@ -63,6 +63,9 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/hashicorp/go-multierror"
 	"github.com/prometheus/client_golang/prometheus"
+	"go.opentelemetry.io/otel/codes"
+	semconv "go.opentelemetry.io/otel/semconv/v1.25.0"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/semaphore"
 )
 
@@ -462,7 +465,7 @@ func (s *Service) newTracingHandler(spanName string) func(h http.Handler) http.H
 				// ignore
 			}
 
-			span, _, ctx := s.tracer.StartSpanFromContext(ctx, spanName, s.logger)
+			span, _, ctx := s.tracer.StartSpanFromContext(ctx, spanName, s.logger, trace.WithSpanKind(trace.SpanKindServer))
 			defer span.End()
 
 			err = s.tracer.AddContextHTTPHeader(ctx, r.Header)
@@ -472,6 +475,23 @@ func (s *Service) newTracingHandler(spanName string) func(h http.Handler) http.H
 			}
 
 			h.ServeHTTP(w, r.WithContext(ctx))
+
+			// The response status code is captured upstream by responseCodeMetricsHandler's
+			// *responseWriter, which exposes Status(). Read it to annotate the span with the
+			// standard HTTP server attributes so the request is queryable in Tempo/Grafana.
+			if sw, ok := w.(interface{ Status() int }); ok {
+				code := sw.Status()
+				span.SetAttributes(
+					semconv.HTTPRequestMethodKey.String(r.Method),
+					semconv.HTTPRoute(spanName),
+					semconv.HTTPResponseStatusCode(code),
+				)
+				// OTel server-span convention: 5xx marks the span as Error; 4xx is a client
+				// error and stays Unset, still queryable via http.response.status_code.
+				if code >= 500 {
+					span.SetStatus(codes.Error, http.StatusText(code))
+				}
+			}
 		})
 	}
 }

pkg/api/api_test.go

@@ -90,6 +90,7 @@ func init() {
 
 type testServerOptions struct {
 	Storer             api.Storer
+	Tracer             *tracing.Tracer
 	StateStorer        storage.StateStorer
 	Resolver           resolver.Interface
 	Pss                pss.Interface
@@ -231,12 +232,16 @@ func newTestServer(t *testing.T, o testServerOptions) (*http.Client, *websocket.
 	s.SetSwarmAddress(&o.Overlay)
 	s.SetProbe(o.Probe)
 
-	noOpTracer, tracerCloser, _ := tracing.NewTracer(&tracing.Options{
-		Enabled: false,
-	})
-	testutil.CleanupCloser(t, tracerCloser)
+	tracer := o.Tracer
+	if tracer == nil {
+		noOpTracer, tracerCloser, _ := tracing.NewTracer(&tracing.Options{
+			Enabled: false,
+		})
+		testutil.CleanupCloser(t, tracerCloser)
+		tracer = noOpTracer
+	}
 
-	s.Configure(signer, noOpTracer, api.Options{
+	s.Configure(signer, tracer, api.Options{
 		CORSAllowedOrigins: o.CORSAllowedOrigins,
 		WsPingPeriod:       o.WsPingPeriod,
 	}, extraOpts, 1, erc20APIService)

pkg/api/bytes.go

@@ -30,7 +30,7 @@ type bytesPostResponse struct {
 
 // bytesUploadHandler handles upload of raw binary data of arbitrary length.
 func (s *Service) bytesUploadHandler(w http.ResponseWriter, r *http.Request) {
-	span, logger, ctx := s.tracer.StartSpanFromContext(r.Context(), "post_bytes", s.logger.WithName("post_bytes").Build())
+	span, logger, ctx := s.tracer.StartSpanFromContext(r.Context(), "bytes-post", s.logger.WithName("post_bytes").Build())
 	defer span.End()
 
 	headers := struct {
@@ -73,7 +73,7 @@ func (s *Service) bytesUploadHandler(w http.ResponseWriter, r *http.Request) {
 			tracing.RecordError(span, err, attribute.String("action", "tag.create"))
 			return
 		}
-		span.SetAttributes(attribute.Int64("tagID", int64(tag)))
+		span.SetAttributes(attribute.Int64("tag_id", int64(tag)))
 	}
 
 	defer s.observeUploadSpeed(w, r, time.Now(), "bytes", deferred)

pkg/api/bzz.go

@@ -65,7 +65,7 @@ func lookaheadBufferSize(size int64) int {
 }
 
 func (s *Service) bzzUploadHandler(w http.ResponseWriter, r *http.Request) {
-	span, logger, ctx := s.tracer.StartSpanFromContext(r.Context(), "post_bzz", s.logger.WithName("post_bzz").Build())
+	span, logger, ctx := s.tracer.StartSpanFromContext(r.Context(), "bzz-post", s.logger.WithName("post_bzz").Build())
 	defer span.End()
 
 	headers := struct {
@@ -107,7 +107,7 @@ func (s *Service) bzzUploadHandler(w http.ResponseWriter, r *http.Request) {
 			tracing.RecordError(span, err, attribute.String("action", "tag.create"))
 			return
 		}
-		span.SetAttributes(attribute.Int64("tagID", int64(tag)))
+		span.SetAttributes(attribute.Int64("tag_id", int64(tag)))
 	}
 
 	putter, err := s.newStamperPutter(ctx, putterOptions{
@@ -340,7 +340,7 @@ func (s *Service) fileUploadHandler(
 
 	if tagID != 0 {
 		w.Header().Set(SwarmTagHeader, fmt.Sprint(tagID))
-		span.SetAttributes(attribute.Int64("tagID", int64(tagID)))
+		span.SetAttributes(attribute.Int64("tag_id", int64(tagID)))
 	}
 	w.Header().Set(ETagHeader, fmt.Sprintf("%q", reference.String()))
 	w.Header().Set(AccessControlExposeHeaders, SwarmTagHeader)