From 178c1080ed4c174d3d02216a57ea0f880e0add71 Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Thu, 26 Mar 2026 17:11:19 -0400 Subject: [PATCH 1/4] implement trace stats for serverless compat --- Cargo.lock | 29 ++- crates/datadog-serverless-compat/src/main.rs | 40 +++- crates/datadog-trace-agent/Cargo.toml | 7 +- crates/datadog-trace-agent/src/config.rs | 9 + crates/datadog-trace-agent/src/lib.rs | 2 + crates/datadog-trace-agent/src/mini_agent.rs | 7 +- .../src/stats_concentrator_service.rs | 194 ++++++++++++++++++ .../datadog-trace-agent/src/stats_flusher.rs | 30 ++- .../src/stats_generator.rs | 49 +++++ .../src/trace_processor.rs | 30 ++- .../tests/integration_test.rs | 16 +- 11 files changed, 389 insertions(+), 24 deletions(-) create mode 100644 crates/datadog-trace-agent/src/stats_concentrator_service.rs create mode 100644 crates/datadog-trace-agent/src/stats_generator.rs diff --git a/Cargo.lock b/Cargo.lock index 9a7c7be8..4339e732 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -546,6 +546,7 @@ dependencies = [ "libdd-common 3.0.1", "libdd-trace-obfuscation", "libdd-trace-protobuf 3.0.0", + "libdd-trace-stats 1.0.4", "libdd-trace-utils 3.0.0", "reqwest", "rmp-serde", @@ -554,6 +555,7 @@ dependencies = [ "serial_test", "temp-env", "tempfile", + "thiserror 1.0.69", "tokio", "tracing", ] @@ -1487,12 +1489,12 @@ dependencies = [ "http", "http-body-util", "libdd-common 2.0.1", - "libdd-ddsketch", + "libdd-ddsketch 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "libdd-dogstatsd-client", "libdd-telemetry", "libdd-tinybytes 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "libdd-trace-protobuf 2.0.0", - "libdd-trace-stats", + "libdd-trace-stats 1.0.3", "libdd-trace-utils 2.0.2", "rmp-serde", "serde", @@ -1513,6 +1515,14 @@ dependencies = [ "prost 0.14.3", ] +[[package]] +name = "libdd-ddsketch" +version = "1.0.1" +source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +dependencies = [ + "prost 0.14.3", +] + [[package]] name = "libdd-dogstatsd-client" version = "1.0.1" @@ -1541,7 +1551,7 @@ dependencies = [ "http-body-util", "libc", "libdd-common 2.0.1", - "libdd-ddsketch", + "libdd-ddsketch 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde", "serde_json", "sys-info", @@ -1633,11 +1643,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea447dc8a5d84c6b5eb6ea877c4fea4149fd29f6b45fcfc5cfd7edf82a18e056" dependencies = [ "hashbrown 0.15.5", - "libdd-ddsketch", + "libdd-ddsketch 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "libdd-trace-protobuf 2.0.0", "libdd-trace-utils 2.0.2", ] +[[package]] +name = "libdd-trace-stats" +version = "1.0.4" +source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +dependencies = [ + "hashbrown 0.15.5", + "libdd-ddsketch 1.0.1 (git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad)", + "libdd-trace-protobuf 3.0.0", + "libdd-trace-utils 3.0.0", +] + [[package]] name = "libdd-trace-utils" version = "2.0.2" diff --git a/crates/datadog-serverless-compat/src/main.rs b/crates/datadog-serverless-compat/src/main.rs index d50798f0..28603972 100644 --- a/crates/datadog-serverless-compat/src/main.rs +++ b/crates/datadog-serverless-compat/src/main.rs @@ -18,7 +18,8 @@ use zstd::zstd_safe::CompressionLevel; use datadog_trace_agent::{ aggregator::TraceAggregator, - config, env_verifier, mini_agent, proxy_flusher, stats_flusher, stats_processor, + config, env_verifier, mini_agent, proxy_flusher, stats_concentrator_service, stats_flusher, + stats_generator, stats_processor, trace_flusher::{self, TraceFlusher}, trace_processor, }; @@ -107,6 +108,12 @@ pub async fn main() { let https_proxy = env::var("DD_PROXY_HTTPS") .or_else(|_| env::var("HTTPS_PROXY")) .ok(); + + let dd_serverless_stats_computation_enabled = + env::var("DD_SERVERLESS_STATS_COMPUTATION_ENABLED") + .map(|val| val.to_lowercase() != "false") + .unwrap_or(true); + debug!("Starting serverless trace mini agent"); let env_filter = format!("h2=off,hyper=off,rustls=off,{}", log_level); @@ -132,11 +139,6 @@ pub async fn main() { let env_verifier = Arc::new(env_verifier::ServerlessEnvVerifier::default()); - let trace_processor = Arc::new(trace_processor::ServerlessTraceProcessor {}); - - let stats_flusher = Arc::new(stats_flusher::ServerlessStatsFlusher {}); - let stats_processor = Arc::new(stats_processor::ServerlessStatsProcessor {}); - let config = match config::Config::new() { Ok(c) => Arc::new(c), Err(e) => { @@ -145,6 +147,30 @@ pub async fn main() { } }; + // Initialize stats concentrator service and generator conditionally + let (stats_concentrator_handle, stats_generator) = if dd_serverless_stats_computation_enabled { + info!("Serverless stats computation enabled"); + let (stats_concentrator_service, stats_concentrator_handle) = + stats_concentrator_service::StatsConcentratorService::new(config.clone()); + tokio::spawn(stats_concentrator_service.run()); + let stats_generator = Arc::new(stats_generator::StatsGenerator::new( + stats_concentrator_handle.clone(), + )); + (Some(stats_concentrator_handle), Some(stats_generator)) + } else { + info!("Serverless stats computation disabled"); + (None, None) + }; + + let trace_processor = Arc::new(trace_processor::ServerlessTraceProcessor { + stats_generator: stats_generator.clone(), + }); + + let stats_flusher = Arc::new(stats_flusher::ServerlessStatsFlusher { + stats_concentrator: stats_concentrator_handle.clone(), + }); + let stats_processor = Arc::new(stats_processor::ServerlessStatsProcessor {}); + let trace_aggregator = Arc::new(TokioMutex::new(TraceAggregator::default())); let trace_flusher = Arc::new(trace_flusher::ServerlessTraceFlusher::new( trace_aggregator, @@ -161,6 +187,8 @@ pub async fn main() { stats_processor, stats_flusher, proxy_flusher, + stats_concentrator: stats_concentrator_handle, + stats_generator, }); tokio::spawn(async move { diff --git a/crates/datadog-trace-agent/Cargo.toml b/crates/datadog-trace-agent/Cargo.toml index 1a69733a..88221059 100644 --- a/crates/datadog-trace-agent/Cargo.toml +++ b/crates/datadog-trace-agent/Cargo.toml @@ -24,14 +24,19 @@ async-trait = "0.1.64" tracing = { version = "0.1", default-features = false } serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0" +thiserror = { version = "1.0.58", default-features = false } libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } +libdd-trace-stats = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad", features = [ "mini_agent", ] } libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } datadog-fips = { path = "../datadog-fips" } -reqwest = { version = "0.12.23", features = ["json", "http2"], default-features = false } +reqwest = { version = "0.12.23", features = [ + "json", + "http2", +], default-features = false } bytes = "1.10.1" [dev-dependencies] diff --git a/crates/datadog-trace-agent/src/config.rs b/crates/datadog-trace-agent/src/config.rs index 5a7b8a8c..bf1d8445 100644 --- a/crates/datadog-trace-agent/src/config.rs +++ b/crates/datadog-trace-agent/src/config.rs @@ -109,6 +109,9 @@ pub struct Config { /// timeout for environment verification, in milliseconds pub verify_env_timeout_ms: u64, pub proxy_url: Option, + pub service: Option, + pub env: Option, + pub version: Option, } impl Config { @@ -251,6 +254,9 @@ impl Config { .or_else(|_| env::var("HTTPS_PROXY")) .ok(), tags, + service: env::var("DD_SERVICE").ok(), + env: env::var("DD_ENV").ok(), + version: env::var("DD_VERSION").ok(), }) } } @@ -721,6 +727,9 @@ pub mod test_helpers { proxy_request_retry_backoff_base_ms: 100, verify_env_timeout_ms: 1000, proxy_url: None, + service: None, + env: None, + version: None, } } } diff --git a/crates/datadog-trace-agent/src/lib.rs b/crates/datadog-trace-agent/src/lib.rs index a87bf56b..daeed742 100644 --- a/crates/datadog-trace-agent/src/lib.rs +++ b/crates/datadog-trace-agent/src/lib.rs @@ -13,7 +13,9 @@ pub mod env_verifier; pub mod http_utils; pub mod mini_agent; pub mod proxy_flusher; +pub mod stats_concentrator_service; pub mod stats_flusher; +pub mod stats_generator; pub mod stats_processor; pub mod trace_flusher; pub mod trace_processor; diff --git a/crates/datadog-trace-agent/src/mini_agent.rs b/crates/datadog-trace-agent/src/mini_agent.rs index 7fa5a92f..50ecb1ad 100644 --- a/crates/datadog-trace-agent/src/mini_agent.rs +++ b/crates/datadog-trace-agent/src/mini_agent.rs @@ -19,7 +19,10 @@ use crate::proxy_flusher::{ProxyFlusher, ProxyRequest}; #[cfg(all(windows, feature = "windows-pipes"))] use tokio::net::windows::named_pipe::ServerOptions; -use crate::{config, env_verifier, stats_flusher, stats_processor, trace_flusher, trace_processor}; +use crate::{ + config, env_verifier, stats_concentrator_service, stats_flusher, stats_generator, + stats_processor, trace_flusher, trace_processor, +}; use libdd_trace_protobuf::pb; use libdd_trace_utils::trace_utils; use libdd_trace_utils::trace_utils::SendData; @@ -47,6 +50,8 @@ pub struct MiniAgent { pub stats_flusher: Arc, pub env_verifier: Arc, pub proxy_flusher: Arc, + pub stats_concentrator: Option, + pub stats_generator: Option>, } impl MiniAgent { diff --git a/crates/datadog-trace-agent/src/stats_concentrator_service.rs b/crates/datadog-trace-agent/src/stats_concentrator_service.rs new file mode 100644 index 00000000..6523c856 --- /dev/null +++ b/crates/datadog-trace-agent/src/stats_concentrator_service.rs @@ -0,0 +1,194 @@ +use tokio::sync::{mpsc, oneshot}; + +use crate::config::Config; +use libdd_trace_protobuf::pb; +use libdd_trace_protobuf::pb::{ClientStatsPayload, TracerPayload}; +use libdd_trace_stats::span_concentrator::SpanConcentrator; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, SystemTime}; +use tracing::error; + +const S_TO_NS: u64 = 1_000_000_000; +const BUCKET_DURATION_NS: u64 = 10 * S_TO_NS; // 10 seconds + +#[derive(Debug, thiserror::Error)] +pub enum StatsError { + #[error("Failed to send command to concentrator: {0}")] + SendError(mpsc::error::SendError), + #[error("Failed to receive response from concentrator: {0}")] + RecvError(oneshot::error::RecvError), +} + +#[derive(Clone, Debug, Default)] +pub struct TracerMetadata { + // e.g. "python" + pub language: String, + // e.g. "3.11.0" + pub tracer_version: String, + // e.g. "f45568ad09d5480b99087d86ebda26e6" + pub runtime_id: String, + pub container_id: String, +} + +pub enum ConcentratorCommand { + SetTracerMetadata(TracerMetadata), + // Use a box to reduce the size of the command enum + Add(Box), + Flush(bool, oneshot::Sender>), +} + +pub struct StatsConcentratorHandle { + tx: mpsc::UnboundedSender, + is_tracer_metadata_set: AtomicBool, +} + +impl Clone for StatsConcentratorHandle { + fn clone(&self) -> Self { + Self { + tx: self.tx.clone(), + // Cloning this may cause trace metadata to be set multiple times, + // but it's okay because it's the same for all traces and we don't need to be perfect on dedup. + is_tracer_metadata_set: AtomicBool::new( + self.is_tracer_metadata_set.load(Ordering::Acquire), + ), + } + } +} + +impl StatsConcentratorHandle { + #[must_use] + pub fn new(tx: mpsc::UnboundedSender) -> Self { + Self { + tx, + is_tracer_metadata_set: AtomicBool::new(false), + } + } + + pub fn set_tracer_metadata(&self, trace: &TracerPayload) -> Result<(), StatsError> { + // Set tracer metadata only once for the first trace because + // it is the same for all traces. + if !self.is_tracer_metadata_set.load(Ordering::Acquire) { + self.is_tracer_metadata_set.store(true, Ordering::Release); + let tracer_metadata = TracerMetadata { + language: trace.language_name.clone(), + tracer_version: trace.tracer_version.clone(), + runtime_id: trace.runtime_id.clone(), + container_id: trace.container_id.clone(), + }; + self.tx + .send(ConcentratorCommand::SetTracerMetadata(tracer_metadata)) + .map_err(StatsError::SendError)?; + } + Ok(()) + } + + pub fn add(&self, span: &pb::Span) -> Result<(), StatsError> { + self.tx + .send(ConcentratorCommand::Add(Box::new(span.clone()))) + .map_err(StatsError::SendError)?; + Ok(()) + } + + pub async fn flush(&self, force_flush: bool) -> Result, StatsError> { + let (response_tx, response_rx) = oneshot::channel(); + self.tx + .send(ConcentratorCommand::Flush(force_flush, response_tx)) + .map_err(StatsError::SendError)?; + response_rx.await.map_err(StatsError::RecvError) + } +} + +pub struct StatsConcentratorService { + concentrator: SpanConcentrator, + rx: mpsc::UnboundedReceiver, + tracer_metadata: TracerMetadata, + config: Arc, +} + +// A service that handles add() and flush() requests in the same queue, +// to avoid using mutex, which may cause lock contention. +impl StatsConcentratorService { + #[must_use] + pub fn new(config: Arc) -> (Self, StatsConcentratorHandle) { + let (tx, rx) = mpsc::unbounded_channel(); + let handle = StatsConcentratorHandle::new(tx); + // TODO: set span_kinds_stats_computed and peer_tag_keys + let concentrator = SpanConcentrator::new( + Duration::from_nanos(BUCKET_DURATION_NS), + SystemTime::now(), + vec![], + vec![], + ); + let service: StatsConcentratorService = Self { + concentrator, + rx, + // To be set when the first trace is received + tracer_metadata: TracerMetadata::default(), + config, + }; + (service, handle) + } + + pub async fn run(mut self) { + while let Some(command) = self.rx.recv().await { + match command { + ConcentratorCommand::SetTracerMetadata(tracer_metadata) => { + self.tracer_metadata = tracer_metadata; + } + ConcentratorCommand::Add(span) => self.concentrator.add_span(&*span), + ConcentratorCommand::Flush(force_flush, response_tx) => { + self.handle_flush(force_flush, response_tx); + } + } + } + } + + fn handle_flush( + &mut self, + force_flush: bool, + response_tx: oneshot::Sender>, + ) { + let stats_buckets = self.concentrator.flush(SystemTime::now(), force_flush); + let stats = if stats_buckets.is_empty() { + None + } else { + Some(ClientStatsPayload { + // Do not set hostname so the trace stats backend can aggregate stats properly + hostname: String::new(), + env: self.config.env.clone().unwrap_or("unknown-env".to_string()), + // Version is not in the trace payload. Need to read it from config. + version: self.config.version.clone().unwrap_or_default(), + lang: self.tracer_metadata.language.clone(), + tracer_version: self.tracer_metadata.tracer_version.clone(), + runtime_id: self.tracer_metadata.runtime_id.clone(), + // Not supported yet + sequence: 0, + // Not supported yet + agent_aggregation: String::new(), + service: self + .config + .service + .clone() + .unwrap_or_default() + .to_lowercase(), + container_id: self.tracer_metadata.container_id.clone(), + // Not supported yet + tags: vec![], + // Not supported yet + git_commit_sha: String::new(), + // Not supported yet + image_tag: String::new(), + stats: stats_buckets, + // Not supported yet + process_tags: String::new(), + // Not supported yet + process_tags_hash: 0, + }) + }; + let response = response_tx.send(stats); + if let Err(e) = response { + error!("Failed to return trace stats: {e:?}"); + } + } +} diff --git a/crates/datadog-trace-agent/src/stats_flusher.rs b/crates/datadog-trace-agent/src/stats_flusher.rs index 6c6e5805..e187a8d2 100644 --- a/crates/datadog-trace-agent/src/stats_flusher.rs +++ b/crates/datadog-trace-agent/src/stats_flusher.rs @@ -10,6 +10,7 @@ use libdd_trace_protobuf::pb; use libdd_trace_utils::stats_utils; use crate::config::Config; +use crate::stats_concentrator_service::StatsConcentratorHandle; #[async_trait] pub trait StatsFlusher { @@ -25,7 +26,9 @@ pub trait StatsFlusher { } #[derive(Clone)] -pub struct ServerlessStatsFlusher {} +pub struct ServerlessStatsFlusher { + pub stats_concentrator: Option, +} #[async_trait] impl StatsFlusher for ServerlessStatsFlusher { @@ -50,14 +53,31 @@ impl StatsFlusher for ServerlessStatsFlusher { tokio::time::sleep(time::Duration::from_secs(config.stats_flush_interval_secs)).await; let mut buffer = buffer_consumer.lock().await; - if !buffer.is_empty() { - self.flush_stats(config.clone(), buffer.to_vec()).await; - buffer.clear(); + let channel_stats = buffer.to_vec(); + buffer.clear(); + drop(buffer); + + let should_flush = !channel_stats.is_empty() || self.stats_concentrator.is_some(); + if should_flush { + self.flush_stats(config.clone(), channel_stats).await; } } } - async fn flush_stats(&self, config: Arc, stats: Vec) { + async fn flush_stats(&self, config: Arc, mut stats: Vec) { + if let Some(ref concentrator) = self.stats_concentrator { + match concentrator.flush(false).await { + Ok(Some(payload)) => { + debug!("Merged agent-generated stats from concentrator into flush batch"); + stats.push(payload); + } + Ok(None) => {} + Err(e) => { + error!("Failed to flush stats concentrator: {e}"); + } + } + } + if stats.is_empty() { return; } diff --git a/crates/datadog-trace-agent/src/stats_generator.rs b/crates/datadog-trace-agent/src/stats_generator.rs new file mode 100644 index 00000000..2bd730e1 --- /dev/null +++ b/crates/datadog-trace-agent/src/stats_generator.rs @@ -0,0 +1,49 @@ +use crate::stats_concentrator_service::{StatsConcentratorHandle, StatsError}; +use libdd_trace_utils::tracer_payload::TracerPayloadCollection; +use tracing::error; + +pub struct StatsGenerator { + stats_concentrator: StatsConcentratorHandle, +} + +#[derive(Debug, thiserror::Error)] +pub enum StatsGeneratorError { + #[error("Error sending trace stats to the stats concentrator: {0}")] + ConcentratorCommandError(StatsError), + #[error("Unsupported trace payload version. Failed to send trace stats.")] + TracePayloadVersionError, +} + +// Extracts information from traces related to stats and sends it to the stats concentrator +impl StatsGenerator { + #[must_use] + pub fn new(stats_concentrator: StatsConcentratorHandle) -> Self { + Self { stats_concentrator } + } + + pub fn send(&self, traces: &TracerPayloadCollection) -> Result<(), StatsGeneratorError> { + if let TracerPayloadCollection::V07(traces) = traces { + for trace in traces { + // Set tracer metadata + if let Err(err) = self.stats_concentrator.set_tracer_metadata(trace) { + error!("Failed to set tracer metadata: {err}"); + return Err(StatsGeneratorError::ConcentratorCommandError(err)); + } + + // Generate stats for each span in the trace + for chunk in &trace.chunks { + for span in &chunk.spans { + if let Err(err) = self.stats_concentrator.add(span) { + error!("Failed to send trace stats: {err}"); + return Err(StatsGeneratorError::ConcentratorCommandError(err)); + } + } + } + } + Ok(()) + } else { + error!("Unsupported trace payload version. Failed to send trace stats."); + Err(StatsGeneratorError::TracePayloadVersionError) + } + } +} diff --git a/crates/datadog-trace-agent/src/trace_processor.rs b/crates/datadog-trace-agent/src/trace_processor.rs index 96f82098..afd5d5cd 100644 --- a/crates/datadog-trace-agent/src/trace_processor.rs +++ b/crates/datadog-trace-agent/src/trace_processor.rs @@ -7,7 +7,7 @@ use async_trait::async_trait; use hyper::{StatusCode, http}; use libdd_common::http_common; use tokio::sync::mpsc::Sender; -use tracing::debug; +use tracing::{debug, error}; use libdd_trace_obfuscation::obfuscate::obfuscate_span; use libdd_trace_protobuf::pb; @@ -18,6 +18,7 @@ use libdd_trace_utils::tracer_payload::{TraceChunkProcessor, TracerPayloadCollec use crate::{ config::Config, http_utils::{self, log_and_create_http_response, log_and_create_traces_success_http_response}, + stats_generator::StatsGenerator, }; const TRACER_PAYLOAD_FUNCTION_TAGS_TAG_KEY: &str = "_dd.tags.function"; @@ -65,7 +66,11 @@ impl TraceChunkProcessor for ChunkProcessor { } } #[derive(Clone)] -pub struct ServerlessTraceProcessor {} +pub struct ServerlessTraceProcessor { + /// The [`StatsGenerator`] to use for generating stats and sending them to + /// the stats concentrator. + pub stats_generator: Option>, +} #[async_trait] impl TraceProcessor for ServerlessTraceProcessor { @@ -139,6 +144,16 @@ impl TraceProcessor for ServerlessTraceProcessor { } } + if let Some(stats_generator) = self.stats_generator.as_ref() { + if tracer_header_tags.client_computed_stats { + debug!( + "Skipping agent-side stats generation: trace payload has Datadog-Client-Computed-Stats" + ); + } else if let Err(e) = stats_generator.send(&payload) { + error!("Stats generator error: {e}"); + } + } + let send_data = SendData::new(body_size, payload, tracer_header_tags, &config.trace_intake); // send trace payload to our trace flusher @@ -219,6 +234,9 @@ mod tests { ..Default::default() }, tags: Tags::from_env_string("env:test,service:my-service"), + service: Some("test-service".to_string()), + env: Some("test-env".to_string()), + version: Some("1.0.0".to_string()), } } @@ -254,7 +272,9 @@ mod tests { .body(http_common::Body::from(bytes)) .unwrap(); - let trace_processor = trace_processor::ServerlessTraceProcessor {}; + let trace_processor = trace_processor::ServerlessTraceProcessor { + stats_generator: None, + }; let res = trace_processor .process_traces( Arc::new(create_test_config()), @@ -326,7 +346,9 @@ mod tests { .body(http_common::Body::from(bytes)) .unwrap(); - let trace_processor = trace_processor::ServerlessTraceProcessor {}; + let trace_processor = trace_processor::ServerlessTraceProcessor { + stats_generator: None, + }; let res = trace_processor .process_traces( Arc::new(create_test_config()), diff --git a/crates/datadog-trace-agent/tests/integration_test.rs b/crates/datadog-trace-agent/tests/integration_test.rs index bf28d4f8..35f39a70 100644 --- a/crates/datadog-trace-agent/tests/integration_test.rs +++ b/crates/datadog-trace-agent/tests/integration_test.rs @@ -54,15 +54,21 @@ pub fn create_mini_agent_with_real_flushers(config: Arc) -> MiniAgent { let aggregator = Arc::new(tokio::sync::Mutex::new(TraceAggregator::default())); MiniAgent { config: config.clone(), - trace_processor: Arc::new(ServerlessTraceProcessor {}), + trace_processor: Arc::new(ServerlessTraceProcessor { + stats_generator: None, + }), trace_flusher: Arc::new(ServerlessTraceFlusher::new( aggregator.clone(), config.clone(), )), stats_processor: Arc::new(ServerlessStatsProcessor {}), - stats_flusher: Arc::new(ServerlessStatsFlusher {}), + stats_flusher: Arc::new(ServerlessStatsFlusher { + stats_concentrator: None, + }), env_verifier: Arc::new(MockEnvVerifier), proxy_flusher: Arc::new(ProxyFlusher::new(config.clone())), + stats_concentrator: None, + stats_generator: None, } } @@ -110,12 +116,16 @@ async fn test_mini_agent_tcp_handles_requests() { let test_port = config.dd_apm_receiver_port; let mini_agent = MiniAgent { config: config.clone(), - trace_processor: Arc::new(ServerlessTraceProcessor {}), + trace_processor: Arc::new(ServerlessTraceProcessor { + stats_generator: None, + }), trace_flusher: Arc::new(MockTraceFlusher), stats_processor: Arc::new(MockStatsProcessor), stats_flusher: Arc::new(MockStatsFlusher), env_verifier: Arc::new(MockEnvVerifier), proxy_flusher: Arc::new(ProxyFlusher::new(config)), + stats_concentrator: None, + stats_generator: None, }; // Start the mini agent From 41429151b867717985a1ce6b55e904fe350a7ff4 Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Fri, 27 Mar 2026 09:18:56 -0400 Subject: [PATCH 2/4] remove references where not needed --- crates/datadog-serverless-compat/src/main.rs | 2 -- crates/datadog-trace-agent/src/mini_agent.rs | 7 +------ crates/datadog-trace-agent/tests/integration_test.rs | 8 +++----- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/crates/datadog-serverless-compat/src/main.rs b/crates/datadog-serverless-compat/src/main.rs index 28603972..e0cea89f 100644 --- a/crates/datadog-serverless-compat/src/main.rs +++ b/crates/datadog-serverless-compat/src/main.rs @@ -187,8 +187,6 @@ pub async fn main() { stats_processor, stats_flusher, proxy_flusher, - stats_concentrator: stats_concentrator_handle, - stats_generator, }); tokio::spawn(async move { diff --git a/crates/datadog-trace-agent/src/mini_agent.rs b/crates/datadog-trace-agent/src/mini_agent.rs index 50ecb1ad..7fa5a92f 100644 --- a/crates/datadog-trace-agent/src/mini_agent.rs +++ b/crates/datadog-trace-agent/src/mini_agent.rs @@ -19,10 +19,7 @@ use crate::proxy_flusher::{ProxyFlusher, ProxyRequest}; #[cfg(all(windows, feature = "windows-pipes"))] use tokio::net::windows::named_pipe::ServerOptions; -use crate::{ - config, env_verifier, stats_concentrator_service, stats_flusher, stats_generator, - stats_processor, trace_flusher, trace_processor, -}; +use crate::{config, env_verifier, stats_flusher, stats_processor, trace_flusher, trace_processor}; use libdd_trace_protobuf::pb; use libdd_trace_utils::trace_utils; use libdd_trace_utils::trace_utils::SendData; @@ -50,8 +47,6 @@ pub struct MiniAgent { pub stats_flusher: Arc, pub env_verifier: Arc, pub proxy_flusher: Arc, - pub stats_concentrator: Option, - pub stats_generator: Option>, } impl MiniAgent { diff --git a/crates/datadog-trace-agent/tests/integration_test.rs b/crates/datadog-trace-agent/tests/integration_test.rs index 35f39a70..5ea8f71d 100644 --- a/crates/datadog-trace-agent/tests/integration_test.rs +++ b/crates/datadog-trace-agent/tests/integration_test.rs @@ -67,8 +67,6 @@ pub fn create_mini_agent_with_real_flushers(config: Arc) -> MiniAgent { }), env_verifier: Arc::new(MockEnvVerifier), proxy_flusher: Arc::new(ProxyFlusher::new(config.clone())), - stats_concentrator: None, - stats_generator: None, } } @@ -124,8 +122,6 @@ async fn test_mini_agent_tcp_handles_requests() { stats_flusher: Arc::new(MockStatsFlusher), env_verifier: Arc::new(MockEnvVerifier), proxy_flusher: Arc::new(ProxyFlusher::new(config)), - stats_concentrator: None, - stats_generator: None, }; // Start the mini agent @@ -216,7 +212,9 @@ async fn test_mini_agent_named_pipe_handles_requests() { let mini_agent = MiniAgent { config: config.clone(), - trace_processor: Arc::new(ServerlessTraceProcessor {}), + trace_processor: Arc::new(ServerlessTraceProcessor { + stats_generator: None, + }), trace_flusher: Arc::new(MockTraceFlusher), stats_processor: Arc::new(MockStatsProcessor), stats_flusher: Arc::new(MockStatsFlusher), From 4ecd9fd383719475153df025bb8bba0d750324a4 Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Fri, 27 Mar 2026 14:57:58 -0400 Subject: [PATCH 3/4] support span derived primary tags --- Cargo.lock | 21 ++++++++------- crates/datadog-agent-config/Cargo.toml | 4 +-- crates/datadog-serverless-compat/Cargo.toml | 2 +- crates/datadog-trace-agent/Cargo.toml | 12 ++++----- crates/datadog-trace-agent/src/config.rs | 26 +++++++++++++++++++ .../src/stats_concentrator_service.rs | 1 + .../src/trace_processor.rs | 1 + 7 files changed, 48 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4339e732..a9816bdc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1445,7 +1445,7 @@ dependencies = [ [[package]] name = "libdd-common" version = "3.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +source = "git+https://github.com/DataDog/libdatadog?rev=49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49#49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" dependencies = [ "anyhow", "bytes", @@ -1518,7 +1518,7 @@ dependencies = [ [[package]] name = "libdd-ddsketch" version = "1.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +source = "git+https://github.com/DataDog/libdatadog?rev=49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49#49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" dependencies = [ "prost 0.14.3", ] @@ -1574,7 +1574,7 @@ dependencies = [ [[package]] name = "libdd-tinybytes" version = "1.1.0" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +source = "git+https://github.com/DataDog/libdatadog?rev=49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49#49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" dependencies = [ "serde", ] @@ -1592,7 +1592,7 @@ dependencies = [ [[package]] name = "libdd-trace-normalization" version = "1.0.3" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +source = "git+https://github.com/DataDog/libdatadog?rev=49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49#49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" dependencies = [ "anyhow", "libdd-trace-protobuf 3.0.0", @@ -1601,7 +1601,7 @@ dependencies = [ [[package]] name = "libdd-trace-obfuscation" version = "1.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +source = "git+https://github.com/DataDog/libdatadog?rev=49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49#49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" dependencies = [ "anyhow", "fluent-uri", @@ -1629,7 +1629,7 @@ dependencies = [ [[package]] name = "libdd-trace-protobuf" version = "3.0.0" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +source = "git+https://github.com/DataDog/libdatadog?rev=49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49#49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" dependencies = [ "prost 0.14.3", "serde", @@ -1651,10 +1651,10 @@ dependencies = [ [[package]] name = "libdd-trace-stats" version = "1.0.4" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +source = "git+https://github.com/DataDog/libdatadog?rev=49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49#49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" dependencies = [ "hashbrown 0.15.5", - "libdd-ddsketch 1.0.1 (git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad)", + "libdd-ddsketch 1.0.1 (git+https://github.com/DataDog/libdatadog?rev=49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49)", "libdd-trace-protobuf 3.0.0", "libdd-trace-utils 3.0.0", ] @@ -1690,9 +1690,10 @@ dependencies = [ [[package]] name = "libdd-trace-utils" version = "3.0.0" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +source = "git+https://github.com/DataDog/libdatadog?rev=49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49#49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" dependencies = [ "anyhow", + "base64 0.22.1", "bytes", "cargo-platform", "cargo_metadata", @@ -1705,7 +1706,7 @@ dependencies = [ "hyper", "indexmap", "libdd-common 3.0.1", - "libdd-tinybytes 1.1.0 (git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad)", + "libdd-tinybytes 1.1.0 (git+https://github.com/DataDog/libdatadog?rev=49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49)", "libdd-trace-normalization 1.0.3", "libdd-trace-protobuf 3.0.0", "prost 0.14.3", diff --git a/crates/datadog-agent-config/Cargo.toml b/crates/datadog-agent-config/Cargo.toml index 222d7265..a2d24835 100644 --- a/crates/datadog-agent-config/Cargo.toml +++ b/crates/datadog-agent-config/Cargo.toml @@ -9,8 +9,8 @@ path = "mod.rs" [dependencies] figment = { version = "0.10", default-features = false, features = ["yaml", "env"] } -libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } +libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" } log = { version = "0.4", default-features = false } serde = { version = "1.0", default-features = false, features = ["derive"] } serde-aux = { version = "4.7", default-features = false } diff --git a/crates/datadog-serverless-compat/Cargo.toml b/crates/datadog-serverless-compat/Cargo.toml index b41de18a..18875f58 100644 --- a/crates/datadog-serverless-compat/Cargo.toml +++ b/crates/datadog-serverless-compat/Cargo.toml @@ -11,7 +11,7 @@ windows-pipes = ["datadog-trace-agent/windows-pipes", "dogstatsd/windows-pipes"] [dependencies] datadog-trace-agent = { path = "../datadog-trace-agent" } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" } datadog-fips = { path = "../datadog-fips", default-features = false } dogstatsd = { path = "../dogstatsd", default-features = true } reqwest = { version = "0.12.4", default-features = false } diff --git a/crates/datadog-trace-agent/Cargo.toml b/crates/datadog-trace-agent/Cargo.toml index 88221059..5c19bfc3 100644 --- a/crates/datadog-trace-agent/Cargo.toml +++ b/crates/datadog-trace-agent/Cargo.toml @@ -25,13 +25,13 @@ tracing = { version = "0.1", default-features = false } serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0" thiserror = { version = "1.0.58", default-features = false } -libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } -libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } -libdd-trace-stats = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad", features = [ +libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" } +libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" } +libdd-trace-stats = { git = "https://github.com/DataDog/libdatadog", rev = "49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49", features = [ "mini_agent", ] } -libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } +libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49" } datadog-fips = { path = "../datadog-fips" } reqwest = { version = "0.12.23", features = [ "json", @@ -45,6 +45,6 @@ serial_test = "2.0.0" duplicate = "0.4.1" temp-env = "0.3.6" tempfile = "3.3.0" -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad", features = [ +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "49ba1c9d33734ec3bd8cc5e7e57bb63a1f832c49", features = [ "test-utils", ] } diff --git a/crates/datadog-trace-agent/src/config.rs b/crates/datadog-trace-agent/src/config.rs index bf1d8445..f0af9d1a 100644 --- a/crates/datadog-trace-agent/src/config.rs +++ b/crates/datadog-trace-agent/src/config.rs @@ -14,6 +14,7 @@ use libdd_trace_utils::config_utils::{ trace_stats_url_prefixed, }; use libdd_trace_utils::trace_utils; +use tracing::debug; const DEFAULT_APM_RECEIVER_PORT: u16 = 8126; const DEFAULT_DOGSTATSD_PORT: u16 = 8125; @@ -112,6 +113,8 @@ pub struct Config { pub service: Option, pub env: Option, pub version: Option, + /// Span tag keys used as second primary tags + pub span_derived_primary_tags: Vec, } impl Config { @@ -215,6 +218,17 @@ impl Config { Tags::new() }; + let span_derived_primary_tags = match env::var("DD_APM_SPAN_DERIVED_PRIMARY_TAGS") { + Ok(env_tags) => parse_json_string_list(&env_tags)?, + Err(_) => vec![], + }; + if !span_derived_primary_tags.is_empty() { + debug!( + "span_derived_primary_tags configured: [{}]", + span_derived_primary_tags.join(" ") + ); + } + #[allow(clippy::unwrap_used)] Ok(Config { app_name: Some(app_name), @@ -257,10 +271,21 @@ impl Config { service: env::var("DD_SERVICE").ok(), env: env::var("DD_ENV").ok(), version: env::var("DD_VERSION").ok(), + span_derived_primary_tags, }) } } +/// Parses a JSON array of strings. Invalid JSON is an error and returns []. +fn parse_json_string_list(env_tags: &str) -> Result, Box> { + serde_json::from_str::>(env_tags).map_err(|e| { + anyhow::anyhow!( + "expected a JSON array of strings, e.g. [] or [\"http.url\",\"db.name\"]: {e}" + ) + .into() + }) +} + #[cfg(test)] mod tests { use duplicate::duplicate_item; @@ -730,6 +755,7 @@ pub mod test_helpers { service: None, env: None, version: None, + span_derived_primary_tags: vec![], } } } diff --git a/crates/datadog-trace-agent/src/stats_concentrator_service.rs b/crates/datadog-trace-agent/src/stats_concentrator_service.rs index 6523c856..5ea4ea65 100644 --- a/crates/datadog-trace-agent/src/stats_concentrator_service.rs +++ b/crates/datadog-trace-agent/src/stats_concentrator_service.rs @@ -119,6 +119,7 @@ impl StatsConcentratorService { SystemTime::now(), vec![], vec![], + config.span_derived_primary_tags.clone(), ); let service: StatsConcentratorService = Self { concentrator, diff --git a/crates/datadog-trace-agent/src/trace_processor.rs b/crates/datadog-trace-agent/src/trace_processor.rs index afd5d5cd..5faf5de2 100644 --- a/crates/datadog-trace-agent/src/trace_processor.rs +++ b/crates/datadog-trace-agent/src/trace_processor.rs @@ -237,6 +237,7 @@ mod tests { service: Some("test-service".to_string()), env: Some("test-env".to_string()), version: Some("1.0.0".to_string()), + span_derived_primary_tags: vec![], } } From 5a64507be703d6b9999ef4c8083c409d28f9c0b8 Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Fri, 27 Mar 2026 14:59:53 -0400 Subject: [PATCH 4/4] address clippy warnings --- crates/datadog-trace-agent/src/mini_agent.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/datadog-trace-agent/src/mini_agent.rs b/crates/datadog-trace-agent/src/mini_agent.rs index 7fa5a92f..ae074810 100644 --- a/crates/datadog-trace-agent/src/mini_agent.rs +++ b/crates/datadog-trace-agent/src/mini_agent.rs @@ -11,7 +11,7 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Instant; use tokio::sync::mpsc::{self, Receiver, Sender}; -use tracing::{debug, error, warn}; +use tracing::{debug, error}; use crate::http_utils::{log_and_create_http_response, verify_request_content_length}; use crate::proxy_flusher::{ProxyFlusher, ProxyRequest}; @@ -191,14 +191,14 @@ impl MiniAgent { let sentinel = std::path::Path::new(LAMBDA_LITE_SENTINEL_PATH); // SAFETY: LAMBDA_LITE_SENTINEL_PATH is a hard-coded absolute path, // so .parent() always returns Some. - if let Some(parent) = sentinel.parent() { - if let Err(e) = tokio::fs::create_dir_all(parent).await { - error!( - "Could not create parent directory for Lambda Lite sentinel \ + if let Some(parent) = sentinel.parent() + && let Err(e) = tokio::fs::create_dir_all(parent).await + { + error!( + "Could not create parent directory for Lambda Lite sentinel \ file at {}: {}.", - LAMBDA_LITE_SENTINEL_PATH, e - ); - } + LAMBDA_LITE_SENTINEL_PATH, e + ); } if let Err(e) = tokio::fs::write(sentinel, b"").await { error!(