From cf170244a2738d7731fa32765fade20dfb9d10d5 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Thu, 7 May 2026 15:59:39 -0400 Subject: [PATCH] fix(aws-lambda): force placeholder resource so the Lambda Extension drops dd-tracer-serverless-span The Datadog Lambda Extension filters the placeholder invocation span by comparing `span.resource == "dd-tracer-serverless-span"` (see `bottlecap/src/traces/trace_processor.rs::filter_span_from_lambda_library_or_runtime`). The intent is that the extension drops the placeholder so the inferred `aws.lambda` span (which the end-invocation handshake gives the same `span_id`) is the surviving record under that key in the trace store and remains parented to the inferred apigateway root. In practice the HTTP/JAX-RS instrumentation overwrites the placeholder's resource with the request route (e.g. "POST /") at `HTTP_FRAMEWORK_ROUTE` priority during the invocation, so the extension's filter no longer matches at end-of-invocation. Both records (the placeholder with `parent_id=0`, and the inferred `aws.lambda` with `parent_id=apigateway.span_id`) reach the backend under the same `(trace_id, span_id)` key. The trace store keeps the placeholder, and the rest of the trace is detached from the inferred apigateway root. Fix: in `LambdaHandlerInstrumentation.exit`, force the resource name back to the literal placeholder marker right before `span.finish()`, using `ResourceNamePriorities.MANUAL_INSTRUMENTATION` so the override beats whatever HTTP-framework priority the in-flight instrumentation has written. Verified end-to-end on Quarkus 3.15.4 / Java 21 / Lambda Extension v96: pre-fix the placeholder leaks to the backend with `parent_id=0` and the trace store reports orphans; post-fix zero `dd_tracer_serverless_span` rows reach the backend and the trace store reports a single 11-span chunk with no orphans. Refs: SLES-2837 --- .../v1/lambda/LambdaHandlerInstrumentation.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java b/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java index d7407b47b5f..a945fabf647 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java +++ b/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java @@ -24,6 +24,7 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; +import datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities; import datadog.trace.config.inversion.ConfigHelper; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.type.TypeDescription; @@ -126,6 +127,21 @@ static void exit( String lambdaRequestId = awsContext.getAwsRequestId(); AgentTracer.get().notifyAppSecEnd(span); + // Force the resource name back to the literal placeholder marker right + // before finish so that the Datadog Lambda Extension's filter + // (filter_span_from_lambda_library_or_runtime in + // bottlecap/src/traces/trace_processor.rs, which compares + // span.resource == "dd-tracer-serverless-span") drops the placeholder. + // Other instrumentation (HTTP/JAX-RS) may have overwritten it with the + // route ("POST /") during the invocation, in which case the extension + // would fail to dedup, leading to the placeholder leaking to the backend + // with parent_id=0 and detaching the inferred apigateway root from the + // rest of the trace. + // Use MANUAL_INSTRUMENTATION priority because DDSpanContext.setResourceName + // ignores writes whose priority is below the current resource priority, + // and the HTTP/JAX-RS instrumentation will already have written + // HTTP_FRAMEWORK_ROUTE (3) by this point. + span.setResourceName(INVOCATION_SPAN_NAME, ResourceNamePriorities.MANUAL_INSTRUMENTATION); span.finish(); AgentTracer.get().notifyExtensionEnd(span, result, null != throwable, lambdaRequestId); } finally {