From 97f99331ec1179cfeafc19c3ba2263b325bb9908 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Thu, 7 May 2026 15:58:35 -0400 Subject: [PATCH] fix(vertx-web): finish vertx.route-handler via RoutingContext.addEndHandler fallback Vert.x's `Http1xServerResponse.end(Buffer, PromiseInternal)` invokes the registered `endHandler` only when `closed == false` at the moment the response body has been written. In synthetic transports such as quarkus-amazon-lambda-rest's `VirtualClientConnection` (in-memory Netty channel) the writes and the connection close happen synchronously inside `responseComplete()`, so by the time the `!closed` guard runs `closed` is already `true` and `endHandler` is silently skipped. Symptom: `RouteHandlerWrapper` starts a `vertx.route-handler` span for every route in the chain (e.g. Quarkus's AuthenticationHandler) but `EndHandlerWrapper.handle` is never called, so the span is never finished. The span dies in PendingTrace and is not enqueued on the writer. All children parented to that span (`jakarta-rs.request`, `netty.client.request`, downstream `aws.http`/`aws.apigateway` inferred spans) end up orphaned in the trace UI. Fix: also register a finish via `RoutingContext.addEndHandler`, which fires on routing-context completion regardless of underlying connection state and on both success and failure. Both paths funnel through a shared idempotent `finishHandlerSpan` so the second one to fire on real-network transports is a no-op. Verified end-to-end against a Quarkus 3.15.4 / Java 21 Lambda chain (caller -> netty.client.request -> callee) on Datadog Lambda Extension v96. Pre-fix: 5/5 invocations Started, 0/5 Finished. Post-fix: 5/5 Started, 5/5 Finished, single connected trace tree in the UI. Refs: SLES-2837 --- .../vertx_4_0/server/EndHandlerWrapper.java | 10 +------- .../vertx_4_0/server/RouteHandlerWrapper.java | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/EndHandlerWrapper.java b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/EndHandlerWrapper.java index a71584b11a7..fd4a042edf8 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/EndHandlerWrapper.java +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/EndHandlerWrapper.java @@ -1,9 +1,5 @@ package datadog.trace.instrumentation.vertx_4_0.server; -import static datadog.trace.instrumentation.vertx_4_0.server.RouteHandlerWrapper.HANDLER_SPAN_CONTEXT_KEY; -import static datadog.trace.instrumentation.vertx_4_0.server.VertxDecorator.DECORATE; - -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import io.vertx.core.Handler; import io.vertx.ext.web.RoutingContext; @@ -18,16 +14,12 @@ public class EndHandlerWrapper implements Handler { @Override public void handle(final Void event) { - AgentSpan span = routingContext.get(HANDLER_SPAN_CONTEXT_KEY); try { if (actual != null) { actual.handle(event); } } finally { - if (span != null) { - DECORATE.onResponse(span, routingContext.response()); - span.finish(); - } + RouteHandlerWrapper.finishHandlerSpan(routingContext); } } } diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/RouteHandlerWrapper.java b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/RouteHandlerWrapper.java index 8706f816e1c..19b344b75f6 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/RouteHandlerWrapper.java +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/RouteHandlerWrapper.java @@ -44,6 +44,16 @@ public void handle(final RoutingContext routingContext) { routingContext.put(HANDLER_SPAN_CONTEXT_KEY, span); routingContext.response().endHandler(new EndHandlerWrapper(routingContext)); + // Fallback finish path: HttpServerResponse.endHandler is silently skipped + // by Vert.x's Http1xServerResponse.end() when the underlying connection + // has already closed (Http1xServerResponse#end gates `endHandler.handle()` + // behind `!closed`). This happens in synthetic transports such as + // quarkus-amazon-lambda-rest's virtual Netty channel, where writes and + // close are synchronous in-memory, leaving the route-handler span unfinished + // and orphaning all jakarta-rs.request / aws.http child spans in the trace. + // RoutingContext#addEndHandler fires on routing-context completion regardless + // of underlying connection state and on both success and failure. + routingContext.addEndHandler(ar -> finishHandlerSpan(routingContext)); DECORATE.afterStart(span); span.setResourceName(DECORATE.className(actual.getClass())); } @@ -60,6 +70,19 @@ public void handle(final RoutingContext routingContext) { } } + // Idempotently finish the route-handler span. Both EndHandlerWrapper (the + // response.endHandler path) and the routingContext.addEndHandler fallback may call + // this; the first one to win clears HANDLER_SPAN_CONTEXT_KEY so the second is a no-op. + static void finishHandlerSpan(final RoutingContext routingContext) { + final AgentSpan span = routingContext.get(HANDLER_SPAN_CONTEXT_KEY); + if (span == null) { + return; + } + routingContext.put(HANDLER_SPAN_CONTEXT_KEY, null); + DECORATE.onResponse(span, routingContext.response()); + span.finish(); + } + private void setRoute(RoutingContext routingContext) { final AgentSpan parentSpan = routingContext.get(PARENT_SPAN_CONTEXT_KEY); if (parentSpan == null) {