From acc4558be415e51c873e6e69405f5d474980abab Mon Sep 17 00:00:00 2001 From: bm1549 Date: Wed, 18 Mar 2026 10:52:48 -0400 Subject: [PATCH] fix: fix Synapse 3.0 instrumentation bugs and harden span propagation Three deterministic bugs fixed: 1. Key mismatch in PassthruInstrumentation: PR #9422 renamed the context key constant but forgot to update the hardcoded string literal in SynapsePassthruInstrumentation, breaking parent span propagation across the passthru mechanism. 2. http.url test assertion: QueryObfuscator recombines http.url with the query string after obfuscation. The test expected path-only but the actual value correctly includes the query string. 3. Missing peer.service in V1 client spans: SynapseClientDecorator returns relative URIs (no host), so peer.hostname was never set. Added peer hostname/IP/port extraction from NHttpClientConnection. Additionally, hardened the passthru span propagation to read the server span directly from the source connection context rather than relying solely on activeSpan(). SourceHandler.requestReceived() dispatches to a worker thread pool, and while java-concurrent instrumentation normally handles context propagation across ThreadPoolExecutor, the connection- based lookup is more robust. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../SynapseClientInstrumentation.java | 15 +++++++++++ .../SynapsePassthruInstrumentation.java | 26 ++++++++++++++++--- .../synapse3/SynapseTest.groovy | 7 ++--- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/instrumentation/synapse-3.0/src/main/java/datadog/trace/instrumentation/synapse3/SynapseClientInstrumentation.java b/dd-java-agent/instrumentation/synapse-3.0/src/main/java/datadog/trace/instrumentation/synapse3/SynapseClientInstrumentation.java index b6abf97dd45..898dcd313af 100644 --- a/dd-java-agent/instrumentation/synapse-3.0/src/main/java/datadog/trace/instrumentation/synapse3/SynapseClientInstrumentation.java +++ b/dd-java-agent/instrumentation/synapse-3.0/src/main/java/datadog/trace/instrumentation/synapse3/SynapseClientInstrumentation.java @@ -22,8 +22,10 @@ import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.agent.tooling.annotation.AppliesOn; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; import net.bytebuddy.asm.Advice; import org.apache.axis2.context.MessageContext; +import org.apache.http.HttpInetConnection; import org.apache.http.HttpResponse; import org.apache.http.nio.NHttpClientConnection; import org.apache.synapse.transport.passthru.TargetContext; @@ -107,6 +109,19 @@ public static void requestSubmitted( // populate span using details from the submitted HttpRequest (resolved URI, etc.) AgentSpan span = spanFromContext(scope.context()); DECORATE.onRequest(span, TargetContext.getRequest(connection).getRequest()); + + // set peer hostname from the connection since request URIs are relative paths + if (connection instanceof HttpInetConnection) { + java.net.InetAddress remoteAddress = ((HttpInetConnection) connection).getRemoteAddress(); + if (remoteAddress != null) { + span.setTag(Tags.PEER_HOSTNAME, remoteAddress.getHostName()); + span.setTag(Tags.PEER_HOST_IPV4, remoteAddress.getHostAddress()); + int remotePort = ((HttpInetConnection) connection).getRemotePort(); + if (remotePort > 0) { + span.setTag(Tags.PEER_PORT, remotePort); + } + } + } scope.close(); } } diff --git a/dd-java-agent/instrumentation/synapse-3.0/src/main/java/datadog/trace/instrumentation/synapse3/SynapsePassthruInstrumentation.java b/dd-java-agent/instrumentation/synapse-3.0/src/main/java/datadog/trace/instrumentation/synapse3/SynapsePassthruInstrumentation.java index c4203348ab6..c27d9bf71fb 100644 --- a/dd-java-agent/instrumentation/synapse-3.0/src/main/java/datadog/trace/instrumentation/synapse3/SynapsePassthruInstrumentation.java +++ b/dd-java-agent/instrumentation/synapse-3.0/src/main/java/datadog/trace/instrumentation/synapse3/SynapsePassthruInstrumentation.java @@ -2,10 +2,13 @@ import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; +import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.spanFromContext; +import static datadog.trace.instrumentation.synapse3.SynapseClientDecorator.SYNAPSE_CONTEXT_KEY; import static net.bytebuddy.matcher.ElementMatchers.isMethod; import static net.bytebuddy.matcher.ElementMatchers.takesArgument; import com.google.auto.service.AutoService; +import datadog.context.Context; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -14,6 +17,7 @@ import java.util.Map; import net.bytebuddy.asm.Advice; import org.apache.axis2.context.MessageContext; +import org.apache.http.nio.NHttpServerConnection; /** Helps propagate parent spans over 'passthru' mechanism to synapse-client instrumentation. */ @AutoService(InstrumenterModule.class) @@ -53,10 +57,26 @@ public static void submit(@Advice.Argument(0) final MessageContext message) { } } - // use message context to propagate active spans across Synapse's 'passthru' mechanism - AgentSpan span = activeSpan(); + // Propagate the server span to the client via the message context. + // Prefer reading the span directly from the source connection's context (where + // SynapseServerInstrumentation stored it) over activeSpan(). SourceHandler dispatches + // request processing to a worker thread pool, and while java-concurrent instrumentation + // normally propagates context across ThreadPoolExecutor, the connection-based lookup is + // more robust as it doesn't depend on automatic context propagation. + AgentSpan span = null; + Object sourceConn = message.getProperty("pass-through.Source-Connection"); + if (sourceConn instanceof NHttpServerConnection) { + Object ctx = + ((NHttpServerConnection) sourceConn).getContext().getAttribute(SYNAPSE_CONTEXT_KEY); + if (ctx instanceof Context) { + span = spanFromContext((Context) ctx); + } + } + if (null == span) { + span = activeSpan(); + } if (null != span) { - message.setNonReplicableProperty("dd.trace.synapse.span", span); + message.setNonReplicableProperty(SYNAPSE_CONTEXT_KEY, span); } } } diff --git a/dd-java-agent/instrumentation/synapse-3.0/src/test/groovy/datadog/trace/instrumentation/synapse3/SynapseTest.groovy b/dd-java-agent/instrumentation/synapse-3.0/src/test/groovy/datadog/trace/instrumentation/synapse3/SynapseTest.groovy index caf7be64d54..80d445b1b61 100644 --- a/dd-java-agent/instrumentation/synapse-3.0/src/test/groovy/datadog/trace/instrumentation/synapse3/SynapseTest.groovy +++ b/dd-java-agent/instrumentation/synapse-3.0/src/test/groovy/datadog/trace/instrumentation/synapse3/SynapseTest.groovy @@ -247,7 +247,7 @@ abstract class SynapseTest extends VersionedNamingTestBase { "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER "$Tags.PEER_HOST_IPV4" "127.0.0.1" "$Tags.PEER_PORT" Integer - "$Tags.HTTP_URL" "/services/SimpleStockQuoteService" + "$Tags.HTTP_URL" query ? "/services/SimpleStockQuoteService?${query}" : "/services/SimpleStockQuoteService" "$DDTags.HTTP_QUERY" query "$Tags.HTTP_METHOD" method "$Tags.HTTP_STATUS" statusCode @@ -302,6 +302,9 @@ abstract class SynapseTest extends VersionedNamingTestBase { tags { "$Tags.COMPONENT" "synapse-client" "$Tags.SPAN_KIND" Tags.SPAN_KIND_CLIENT + "$Tags.PEER_HOSTNAME" String + "$Tags.PEER_HOST_IPV4" "127.0.0.1" + "$Tags.PEER_PORT" Integer "$Tags.HTTP_URL" "/services/SimpleStockQuoteService" "$Tags.HTTP_METHOD" method "$Tags.HTTP_STATUS" statusCode @@ -311,7 +314,6 @@ abstract class SynapseTest extends VersionedNamingTestBase { } } -@Flaky("Occasionally times out when receiving traces") class SynapseV0ForkedTest extends SynapseTest implements TestingGenericHttpNamingConventions.ClientV0 { @@ -321,7 +323,6 @@ class SynapseV0ForkedTest extends SynapseTest implements TestingGenericHttpNamin } } -@Flaky("Occasionally times out when receiving traces") class SynapseV1ForkedTest extends SynapseTest implements TestingGenericHttpNamingConventions.ClientV1 { @Override