diff --git a/api/src/main/java/io/grpc/ClientStreamTracer.java b/api/src/main/java/io/grpc/ClientStreamTracer.java index d654fd33358..dfc4c5d8820 100644 --- a/api/src/main/java/io/grpc/ClientStreamTracer.java +++ b/api/src/main/java/io/grpc/ClientStreamTracer.java @@ -67,7 +67,7 @@ public void createPendingStream() { * * @param delayType canonical low-cardinality label categorizing the delay (e.g., "connecting") * @param delayReason high-cardinality diagnostic string describing granular runtime conditions - * @since 1.82.0 + * @since 1.84.0 */ public void recordAttemptDelayStart(String delayType, String delayReason) { } @@ -80,7 +80,7 @@ public void recordAttemptDelayStart(String delayType, String delayReason) { * on the active delay span without recreating the span or resetting cumulative timers. * * @param delayReason updated high-cardinality diagnostic string describing new conditions - * @since 1.82.0 + * @since 1.84.0 */ public void recordAttemptDelayReasonChanged(String delayReason) { } @@ -91,7 +91,7 @@ public void recordAttemptDelayReasonChanged(String delayReason) { *

Implementations should simultaneously close active child tracing spans and record elapsed * duration to the {@code grpc.client.attempt.delay.duration} histogram. * - * @since 1.82.0 + * @since 1.84.0 */ public void recordAttemptDelayEnd() { } @@ -156,6 +156,44 @@ public abstract static class Factory { public ClientStreamTracer newClientStreamTracer(StreamInfo info, Metadata headers) { throw new UnsupportedOperationException("Not implemented"); } + + /** + * Called when a call-level delay segment (such as waiting for name resolution or service + * configuration parsing) starts before any individual RPC attempt is created. + * + *

Implementations should start logical timers and create child tracing spans (named strictly + * {@code "Call Delay"}) carrying the canonical {@code grpc.delay_type} attribute. + * + * @param delayType canonical low-cardinality label categorizing the delay (e.g., "resolving") + * @param delayReason high-cardinality diagnostic string describing granular runtime conditions + * @since 1.84.0 + */ + public void recordCallDelayStart(String delayType, String delayReason) { + } + + /** + * Called when a call-level delay reason changes while the active delay segment continues. + * + *

Implementations should emit structured events (such as {@code "Delay state transition"}) + * on the active call delay span without recreating the span or resetting timers. + * + * @param delayReason updated high-cardinality diagnostic string describing new conditions + * @since 1.84.0 + */ + public void recordCallDelayReasonChanged(String delayReason) { + } + + /** + * Called when a call-level delay segment ends upon successful name resolution or when an RPC + * is cancelled before resolution completes. + * + *

Implementations should close active call delay spans and record elapsed duration to the + * {@code grpc.client.call.delay.duration} histogram. + * + * @since 1.84.0 + */ + public void recordCallDelayEnd() { + } } /** diff --git a/api/src/main/java/io/grpc/LoadBalancer.java b/api/src/main/java/io/grpc/LoadBalancer.java index e5c3d053ee7..c9dce0f2e3d 100644 --- a/api/src/main/java/io/grpc/LoadBalancer.java +++ b/api/src/main/java/io/grpc/LoadBalancer.java @@ -739,7 +739,7 @@ public static PickResult withNoResult() { * * @param delayType low-cardinality root cause label (e.g., "connecting") * @param delayReason high-cardinality diagnostic string for trace events - * @since 1.82.0 + * @since 1.84.0 */ public static PickResult withNoResult(String delayType, String delayReason) { Preconditions.checkNotNull(delayType, "delayType"); diff --git a/api/src/test/java/io/grpc/ClientStreamTracerTest.java b/api/src/test/java/io/grpc/ClientStreamTracerTest.java index 5ddee77f5c0..1d605a4f707 100644 --- a/api/src/test/java/io/grpc/ClientStreamTracerTest.java +++ b/api/src/test/java/io/grpc/ClientStreamTracerTest.java @@ -57,4 +57,17 @@ public void streamInfo_toBuilder() { StreamInfo info2 = info1.toBuilder().build(); assertThat(info2.getCallOptions()).isSameInstanceAs(callOptions); } + + @Test + public void defaultDelayMethodsNoOp() { + ClientStreamTracer tracer = new ClientStreamTracer() {}; + tracer.recordAttemptDelayStart("connecting", "test"); + tracer.recordAttemptDelayReasonChanged("test2"); + tracer.recordAttemptDelayEnd(); + + ClientStreamTracer.Factory factory = new ClientStreamTracer.Factory() {}; + factory.recordCallDelayStart("resolving", "test"); + factory.recordCallDelayReasonChanged("test2"); + factory.recordCallDelayEnd(); + } } diff --git a/core/src/main/java/io/grpc/internal/DelayedClientTransport.java b/core/src/main/java/io/grpc/internal/DelayedClientTransport.java index aa1d820a570..0b302a9f3f7 100644 --- a/core/src/main/java/io/grpc/internal/DelayedClientTransport.java +++ b/core/src/main/java/io/grpc/internal/DelayedClientTransport.java @@ -396,7 +396,7 @@ private static String determineQueuingDelayReason(@Nullable PickResult pickResul return "client channel: waiting for picker"; } - private class PendingStream extends DelayedStream { + class PendingStream extends DelayedStream { private final PickSubchannelArgs args; private final Context context = Context.current(); private final ClientStreamTracer[] tracers; @@ -405,6 +405,8 @@ private class PendingStream extends DelayedStream { @Nullable private String activeDelayType; @GuardedBy("this") @Nullable private String activeDelayReason; + @GuardedBy("this") + private boolean delayEnded; private PendingStream(PickSubchannelArgs args, ClientStreamTracer[] tracers, @Nullable String initialType, @Nullable String initialReason) { @@ -428,11 +430,11 @@ private PendingStream(PickSubchannelArgs args, ClientStreamTracer[] tracers, * structured transition event is appended to the active span without span re-creation. */ synchronized void updateDelay(@Nullable String newType, @Nullable String newReason) { - if (getRealStream() != null) { + if (getRealStream() != null || delayEnded) { return; } + // Delay type changed (e.g., from RLS lookup to connecting). End the previous delay. if (!Objects.equals(activeDelayType, newType)) { - // Delay type changed (e.g., from RLS lookup to connecting). End the previous delay. if (activeDelayType != null) { for (ClientStreamTracer tracer : tracers) { tracer.recordAttemptDelayEnd(); @@ -446,8 +448,9 @@ synchronized void updateDelay(@Nullable String newType, @Nullable String newReas } } } - if (newType != null && newReason != null && !Objects.equals(activeDelayReason, newReason)) { - // Delay type is unchanged, but the reason changed (e.g., priority failover). + // Delay type is unchanged, but the reason changed (e.g., priority failover). + if (newType != null && newReason != null + && !Objects.equals(activeDelayReason, newReason)) { activeDelayReason = newReason; for (ClientStreamTracer tracer : tracers) { tracer.recordAttemptDelayReasonChanged(newReason); @@ -459,6 +462,10 @@ synchronized void updateDelay(@Nullable String newType, @Nullable String newReas * Ends active attempt delay segment telemetry upon stream creation or stream cancellation. */ synchronized void endDelay() { + if (delayEnded) { + return; + } + delayEnded = true; if (activeDelayType != null) { for (ClientStreamTracer tracer : tracers) { tracer.recordAttemptDelayEnd(); @@ -469,8 +476,9 @@ synchronized void endDelay() { } Runnable setStreamAndEndDelay(ClientStream stream) { + Runnable runnable = setStream(stream); endDelay(); - return setStream(stream); + return runnable; } /** Runnable may be null. */ diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java index 00df05a0c00..a55fe08b3de 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java @@ -918,6 +918,7 @@ public void run() { inUseStateAggregator.updateObjectInUse(pendingCallsInUseObject, true); } pendingCalls.add(pendingCall); + pendingCall.notifyQueuedForNameResolution(); } else { pendingCall.reprocess(); } @@ -997,6 +998,9 @@ private final class PendingCall extends DelayedClientCall method; final CallOptions callOptions; private final long callCreationTime; + private volatile boolean queuedForResolution; + private volatile boolean callCancelled; + private final AtomicBoolean delayEnded = new AtomicBoolean(); PendingCall(Context context, MethodDescriptor method, CallOptions callOptions) { super( @@ -1010,14 +1014,42 @@ private final class PendingCall extends DelayedClientCall realCall; Context previous = context.attach(); try { - CallOptions delayResolutionOption = callOptions.withOption(NAME_RESOLUTION_DELAYED, - ticker.nanoTime() - callCreationTime); - realCall = newClientCall(method, delayResolutionOption); + CallOptions effectiveOptions = callOptions; + if (queuedForResolution) { + effectiveOptions = callOptions.withOption(NAME_RESOLUTION_DELAYED, + ticker.nanoTime() - callCreationTime); + } + realCall = newClientCall(method, effectiveOptions); } finally { context.detach(previous); } @@ -1037,6 +1069,10 @@ public void run() { @Override protected void callCancelled() { + synchronized (this) { + callCancelled = true; + } + endDelayIfNeeded(); super.callCancelled(); syncContext.execute(new PendingCallRemoval()); } diff --git a/core/src/test/java/io/grpc/internal/DelayedClientTransportTest.java b/core/src/test/java/io/grpc/internal/DelayedClientTransportTest.java index afcce806f2b..0c47fd8447d 100644 --- a/core/src/test/java/io/grpc/internal/DelayedClientTransportTest.java +++ b/core/src/test/java/io/grpc/internal/DelayedClientTransportTest.java @@ -972,6 +972,86 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { assertEquals(1, fakeTracer.delayEndedCount); } + @Test + public void streamDelayMetrics_allBranchCombinations() { + FakeStreamTracer fakeTracer = new FakeStreamTracer(); + ClientStreamTracer[] customTracers = new ClientStreamTracer[] { fakeTracer }; + + // 1. Initial stream with no picker set (initialType defaults to "connecting") + ClientStream stream = delayedTransport.newStream(method, headers, callOptions, customTracers); + stream.start(streamListener); + + assertEquals(Collections.singletonList("connecting"), fakeTracer.startedDelayTypes); + assertEquals(Collections.singletonList("client channel: waiting for picker"), + fakeTracer.startedDelayReasons); + + // 2. Reprocess with same delay type ("connecting") but new reason ("attempting to connect") + delayedTransport.reprocess(fakePicker( + PickResult.withNoResult("connecting", "attempting to connect"))); + assertEquals(Collections.singletonList("attempting to connect"), + fakeTracer.changedDelayReasons); + + // 3. Reprocess with a different delay type ("rls_lookup_pending") + delayedTransport.reprocess(fakePicker( + PickResult.withNoResult("rls_lookup_pending", "RLS pending"))); + assertEquals(1, fakeTracer.delayEndedCount); + assertEquals(Arrays.asList("connecting", "rls_lookup_pending"), fakeTracer.startedDelayTypes); + + // 4. Reprocess with default PickResult.withNoResult() (defaults to "connecting") + delayedTransport.reprocess(fakePicker(PickResult.withNoResult())); + assertEquals(2, fakeTracer.delayEndedCount); + + // 5. Cancel stream ends active delay + stream.cancel(Status.CANCELLED); + assertEquals(3, fakeTracer.delayEndedCount); + + // 6. Direct updateDelay with null type and null reason + fakeTracer = new FakeStreamTracer(); + customTracers = new ClientStreamTracer[] { fakeTracer }; + ClientStream stream2 = delayedTransport.newStream(method, headers, callOptions, customTracers); + fakeTracer.startedDelayTypes.clear(); + stream2.start(streamListener); + + // Explicitly update delay to null + delayedTransport.reprocess(fakePicker(PickResult.withNoResult())); + + // 7. Cancel stream2 (endDelay when activeDelayType is null) + stream2.cancel(Status.CANCELLED); + } + + @Test + public void streamDelayMetrics_emptyTracersAndNullInitialReason() { + ClientStreamTracer[] emptyTracers = new ClientStreamTracer[0]; + delayedTransport.reprocess(fakePicker(PickResult.withNoResult("resolving", ""))); + ClientStream stream = delayedTransport.newStream(method, headers, callOptions, emptyTracers); + stream.start(streamListener); + + // Update delay with empty tracers and empty reason + delayedTransport.reprocess(fakePicker(PickResult.withNoResult("connecting", ""))); + stream.cancel(Status.CANCELLED); + } + + @Test + public void streamDelayMetrics_delayEndedWhileRealStreamNull_updateDelayReturnsEarly() { + FakeStreamTracer fakeTracer = new FakeStreamTracer(); + ClientStreamTracer[] customTracers = new ClientStreamTracer[] { fakeTracer }; + + DelayedClientTransport.PendingStream pendingStream = + (DelayedClientTransport.PendingStream) delayedTransport.newStream( + method, headers, callOptions, customTracers); + pendingStream.start(streamListener); + + // End delay directly while realStream is still null (sets delayEnded = true) + pendingStream.endDelay(); + assertEquals(1, fakeTracer.delayEndedCount); + + // updateDelay after delayEnded when getRealStream() == null should return early + // without starting a new delay segment + pendingStream.updateDelay("connecting", "new reason"); + assertEquals(Collections.singletonList("connecting"), fakeTracer.startedDelayTypes); + assertEquals(1, fakeTracer.delayEndedCount); + } + private static TransportProvider newTransportProvider(final ClientTransport transport) { return new TransportProvider() { @Override diff --git a/core/src/test/java/io/grpc/internal/ForwardingClientStreamTracerTest.java b/core/src/test/java/io/grpc/internal/ForwardingClientStreamTracerTest.java index 5eb5b49fa19..87b5b8f87b4 100644 --- a/core/src/test/java/io/grpc/internal/ForwardingClientStreamTracerTest.java +++ b/core/src/test/java/io/grpc/internal/ForwardingClientStreamTracerTest.java @@ -17,6 +17,7 @@ package io.grpc.internal; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import io.grpc.ClientStreamTracer; import io.grpc.ForwardingTestUtil; @@ -40,6 +41,19 @@ public void allMethodsForwarded() throws Exception { Collections.emptyList()); } + @Test + public void attemptDelayMethodsForwarded() { + TestClientStreamTracer tracer = new TestClientStreamTracer(); + tracer.recordAttemptDelayStart("connecting", "test"); + verify(mockDelegate).recordAttemptDelayStart("connecting", "test"); + + tracer.recordAttemptDelayReasonChanged("test2"); + verify(mockDelegate).recordAttemptDelayReasonChanged("test2"); + + tracer.recordAttemptDelayEnd(); + verify(mockDelegate).recordAttemptDelayEnd(); + } + private final class TestClientStreamTracer extends ForwardingClientStreamTracer { @Override protected ClientStreamTracer delegate() { diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java index 42fb374a68b..a5fbc827cdf 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java @@ -1894,6 +1894,10 @@ public void oobChannelHasNoChannelCallCredentials() { same(method), same(headers), ArgumentMatchers.any(), ArgumentMatchers.any()); assertThat(headers.getAll(metadataKey)).containsExactly(callCredValue); + if (oob instanceof ManagedChannelImpl.OobChannel) { + ((ManagedChannelImpl.OobChannel) oob) + .updateAddresses(Collections.singletonList(addressGroup)); + } oob.shutdownNow(); // Verify that resolving oob channel does not @@ -4522,6 +4526,171 @@ public void validAuthorityTarget_overrideAuthority() throws Exception { assertThat(nameResolver.getServiceAuthority()).isEqualTo(overrideAuthority); } + @Test + public void nameResolutionDelay_recordsStartAndEndOnTracerFactory() { + FakeNameResolverFactory nameResolverFactory = + new FakeNameResolverFactory.Builder(expectedUri) + .setResolvedAtStart(false) + .build(); + channelBuilder.nameResolverFactory(nameResolverFactory); + createChannel(); + + ClientStreamTracer.Factory mockTracerFactory = mock(ClientStreamTracer.Factory.class); + when(mockTracerFactory.newClientStreamTracer(any(), any())) + .thenReturn(new ClientStreamTracer() {}); + ClientCall call = channel.newCall(method, + CallOptions.DEFAULT.withStreamTracerFactory(mockTracerFactory)); + call.start(mockCallListener, new Metadata()); + + verify(mockTracerFactory).recordCallDelayStart( + eq("resolving"), eq("waiting for name resolution or service config")); + verify(mockTracerFactory, never()).recordCallDelayEnd(); + + nameResolverFactory.allResolved(); + timer.runDueTasks(); + executor.runDueTasks(); + verify(mockTracerFactory).recordCallDelayEnd(); + + call.cancel("Cleanup test", null); + timer.runDueTasks(); + executor.runDueTasks(); + } + + @Test + public void nameResolutionDelay_cancelledBeforeResolution_recordsEndOnTracerFactory() { + FakeNameResolverFactory nameResolverFactory = + new FakeNameResolverFactory.Builder(expectedUri) + .setResolvedAtStart(false) + .build(); + channelBuilder.nameResolverFactory(nameResolverFactory); + createChannel(); + + ClientStreamTracer.Factory mockTracerFactory = mock(ClientStreamTracer.Factory.class); + when(mockTracerFactory.newClientStreamTracer(any(), any())) + .thenReturn(new ClientStreamTracer() {}); + ClientCall call = channel.newCall(method, + CallOptions.DEFAULT.withStreamTracerFactory(mockTracerFactory)); + call.start(mockCallListener, new Metadata()); + + verify(mockTracerFactory).recordCallDelayStart( + eq("resolving"), eq("waiting for name resolution or service config")); + + call.cancel("Cancelled during name resolution", null); + timer.runDueTasks(); + executor.runDueTasks(); + verify(mockTracerFactory).recordCallDelayEnd(); + } + + @Test + public void nameResolutionDelay_nameResolutionError_recordsEndOnTracerFactory() { + FakeNameResolverFactory nameResolverFactory = + new FakeNameResolverFactory.Builder(expectedUri) + .setResolvedAtStart(false) + .setError(Status.UNAVAILABLE.withDescription("Name resolution failed")) + .build(); + channelBuilder.nameResolverFactory(nameResolverFactory); + createChannel(); + + ClientStreamTracer.Factory mockTracerFactory = mock(ClientStreamTracer.Factory.class); + when(mockTracerFactory.newClientStreamTracer(any(), any())) + .thenReturn(new ClientStreamTracer() {}); + ClientCall call = channel.newCall(method, + CallOptions.DEFAULT.withStreamTracerFactory(mockTracerFactory)); + call.start(mockCallListener, new Metadata()); + + verify(mockTracerFactory).recordCallDelayStart( + eq("resolving"), eq("waiting for name resolution or service config")); + + nameResolverFactory.allResolved(); + timer.runDueTasks(); + executor.runDueTasks(); + verify(mockTracerFactory).recordCallDelayEnd(); + + call.cancel("Cleanup test", null); + timer.runDueTasks(); + executor.runDueTasks(); + } + + @Test + public void nameResolutionDelay_alreadyResolved_doesNotRecordDelay() { + FakeNameResolverFactory nameResolverFactory = + new FakeNameResolverFactory.Builder(expectedUri) + .setResolvedAtStart(true) + .build(); + channelBuilder.nameResolverFactory(nameResolverFactory); + createChannel(); + + ClientStreamTracer.Factory mockTracerFactory = mock(ClientStreamTracer.Factory.class); + when(mockTracerFactory.newClientStreamTracer(any(), any())) + .thenReturn(new ClientStreamTracer() {}); + ClientCall call = channel.newCall(method, + CallOptions.DEFAULT.withStreamTracerFactory(mockTracerFactory)); + call.start(mockCallListener, new Metadata()); + timer.runDueTasks(); + executor.runDueTasks(); + + verify(mockTracerFactory, never()).recordCallDelayStart(any(), any()); + verify(mockTracerFactory, never()).recordCallDelayEnd(); + + call.cancel("Cleanup test", null); + timer.runDueTasks(); + executor.runDueTasks(); + } + + @Test + public void nameResolutionDelay_callCancelledBeforeNotify_doesNotRecordDelayStart() { + FakeNameResolverFactory nameResolverFactory = + new FakeNameResolverFactory.Builder(expectedUri) + .setResolvedAtStart(false) + .build(); + channelBuilder.nameResolverFactory(nameResolverFactory); + createChannel(); + + ClientStreamTracer.Factory mockTracerFactory = mock(ClientStreamTracer.Factory.class); + when(mockTracerFactory.newClientStreamTracer(any(), any())) + .thenReturn(new ClientStreamTracer() {}); + ClientCall call = channel.newCall(method, + CallOptions.DEFAULT.withStreamTracerFactory(mockTracerFactory)); + call.start(mockCallListener, new Metadata()); + // Cancel immediately before notifyQueuedForNameResolution can proceed + call.cancel("Cancelled before notify", null); + timer.runDueTasks(); + executor.runDueTasks(); + + // Verify recordCallDelayEnd is called, but no new recordCallDelayStart occurs + // after cancellation + verify(mockTracerFactory).recordCallDelayEnd(); + } + + @Test + public void nameResolutionDelay_duplicateNotifyAndCancel_handlesGracefully() { + FakeNameResolverFactory nameResolverFactory = + new FakeNameResolverFactory.Builder(expectedUri) + .setResolvedAtStart(false) + .build(); + channelBuilder.nameResolverFactory(nameResolverFactory); + createChannel(); + + ClientStreamTracer.Factory mockTracerFactory = mock(ClientStreamTracer.Factory.class); + when(mockTracerFactory.newClientStreamTracer(any(), any())) + .thenReturn(new ClientStreamTracer() {}); + ClientCall call = channel.newCall(method, + CallOptions.DEFAULT.withStreamTracerFactory(mockTracerFactory)); + call.start(mockCallListener, new Metadata()); + timer.runDueTasks(); + executor.runDueTasks(); + + // Multiple cancellation/endDelay calls should be handled cleanly + call.cancel("First cancel", null); + call.cancel("Second cancel", null); + timer.runDueTasks(); + executor.runDueTasks(); + + verify(mockTracerFactory).recordCallDelayStart(any(), any()); + verify(mockTracerFactory).recordCallDelayEnd(); + } + + @Test public void validTargetNoResolver_throws() { NameResolverProvider nameResolverProvider = new NameResolverProvider() { diff --git a/opentelemetry/src/main/java/io/grpc/opentelemetry/GrpcOpenTelemetry.java b/opentelemetry/src/main/java/io/grpc/opentelemetry/GrpcOpenTelemetry.java index 1243e0fff59..6965dadea85 100644 --- a/opentelemetry/src/main/java/io/grpc/opentelemetry/GrpcOpenTelemetry.java +++ b/opentelemetry/src/main/java/io/grpc/opentelemetry/GrpcOpenTelemetry.java @@ -243,6 +243,16 @@ && isMetricEnabled("grpc.client.attempt.delay.duration", enableMetrics, disableD .build()); } + if (isDelayObservabilityEnabled() + && isMetricEnabled("grpc.client.call.delay.duration", enableMetrics, disableDefault)) { + builder.clientCallDelayCounter( + meter.histogramBuilder( + "grpc.client.call.delay.duration") + .setUnit("s") + .setDescription("Time taken before a client call starts") + .setExplicitBucketBoundariesAdvice(LATENCY_BUCKETS) + .build()); + } if (isMetricEnabled("grpc.client.attempt.sent_total_compressed_message_size", enableMetrics, disableDefault)) { builder.clientTotalSentCompressedMessageSizeCounter( @@ -364,7 +374,7 @@ && isMetricEnabled("grpc.client.attempt.delay.duration", enableMetrics, disableD * Checks whether experimental client attempt and call delay observability is globally enabled. * *

Guarded strictly by the {@code GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY} environment - * variable (defaults to {@code false}). When disabled, delay spans and + * variable or JVM system property (defaults to {@code false}). When disabled, delay spans and * duration histograms are suppressed to avoid runtime overhead. */ static boolean isDelayObservabilityEnabled() { diff --git a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsModule.java b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsModule.java index 008a9754b29..8f2250ec0dc 100644 --- a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsModule.java +++ b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsModule.java @@ -256,13 +256,7 @@ public void recordAttemptDelayEnd() { .put(METHOD_KEY, fullMethodName) .put(TARGET_KEY, target) .put("grpc.delay_type", delayType); - if (module.customLabelEnabled) { - builder.put( - CUSTOM_LABEL_KEY, info.getCallOptions().getOption(Grpc.CALL_OPTION_CUSTOM_LABEL)); - } - for (OpenTelemetryPlugin.ClientStreamPlugin plugin : streamPlugins) { - plugin.addLabels(builder); - } + addOptionalLabels(builder); module.resource.clientAttemptDelayCounter() .record(delayNanos * SECONDS_PER_NANO, builder.build(), attemptsState.otelContext); } @@ -333,32 +327,12 @@ public void streamClosed(Status status) { } void recordFinishedAttempt() { - AttributesBuilder builder = io.opentelemetry.api.common.Attributes.builder() + AttributesBuilder builder = Attributes.builder() .put(METHOD_KEY, fullMethodName) .put(TARGET_KEY, target) .put(STATUS_KEY, statusCode.toString()); - if (module.localityEnabled) { - String savedLocality = locality; - if (savedLocality == null) { - savedLocality = ""; - } - builder.put(LOCALITY_KEY, savedLocality); - } - if (module.backendServiceEnabled) { - String savedBackendService = backendService; - if (savedBackendService == null) { - savedBackendService = ""; - } - builder.put(BACKEND_SERVICE_KEY, savedBackendService); - } - if (module.customLabelEnabled) { - builder.put( - CUSTOM_LABEL_KEY, info.getCallOptions().getOption(Grpc.CALL_OPTION_CUSTOM_LABEL)); - } - for (OpenTelemetryPlugin.ClientStreamPlugin plugin : streamPlugins) { - plugin.addLabels(builder); - } - io.opentelemetry.api.common.Attributes attribute = builder.build(); + addOptionalLabels(builder); + Attributes attribute = builder.build(); if (module.resource.clientAttemptDurationCounter() != null ) { module.resource.clientAttemptDurationCounter() @@ -373,6 +347,22 @@ void recordFinishedAttempt() { .record(inboundWireSize, attribute, attemptsState.otelContext); } } + + private void addOptionalLabels(AttributesBuilder builder) { + if (module.localityEnabled) { + builder.put(LOCALITY_KEY, Objects.toString(locality, "")); + } + if (module.backendServiceEnabled) { + builder.put(BACKEND_SERVICE_KEY, Objects.toString(backendService, "")); + } + if (module.customLabelEnabled) { + builder.put( + CUSTOM_LABEL_KEY, info.getCallOptions().getOption(Grpc.CALL_OPTION_CUSTOM_LABEL)); + } + for (OpenTelemetryPlugin.ClientStreamPlugin plugin : streamPlugins) { + plugin.addLabels(builder); + } + } } @VisibleForTesting @@ -387,6 +377,11 @@ static final class CallAttemptsTracerFactory extends ClientStreamTracer.Factory private final List callPlugins; private final Context otelContext; private Status status; + @GuardedBy("this") + @Nullable private Stopwatch activeCallDelayStopwatch; + @GuardedBy("this") + @Nullable private String activeCallDelayType; + private final Attributes callLevelBaseAttributes; private long retryDelayNanos; private long callLatencyNanos; private final Object lock = new Object(); @@ -412,18 +407,18 @@ static final class CallAttemptsTracerFactory extends ClientStreamTracer.Factory this.attemptDelayStopwatch = module.stopwatchSupplier.get(); this.callStopWatch = module.stopwatchSupplier.get().start(); - AttributesBuilder builder = io.opentelemetry.api.common.Attributes.builder() + AttributesBuilder builder = Attributes.builder() .put(METHOD_KEY, fullMethodName) .put(TARGET_KEY, target); if (module.customLabelEnabled) { builder.put( CUSTOM_LABEL_KEY, callOptions.getOption(Grpc.CALL_OPTION_CUSTOM_LABEL)); } - io.opentelemetry.api.common.Attributes attribute = builder.build(); + this.callLevelBaseAttributes = builder.build(); // Record here in case mewClientStreamTracer() would never be called. if (module.resource.clientAttemptCountCounter() != null) { - module.resource.clientAttemptCountCounter().add(1, attribute, otelContext); + module.resource.clientAttemptCountCounter().add(1, callLevelBaseAttributes, otelContext); } } @@ -443,14 +438,14 @@ public ClientStreamTracer newClientStreamTracer(StreamInfo info, Metadata metada // CallAttemptsTracerFactory constructor. attemptsPerCall will be non-zero after the first // attempt, as first attempt cannot be a transparent retry. if (attemptsPerCall.get() > 0) { - AttributesBuilder builder = io.opentelemetry.api.common.Attributes.builder() + AttributesBuilder builder = Attributes.builder() .put(METHOD_KEY, fullMethodName) .put(TARGET_KEY, target); if (module.customLabelEnabled) { builder.put( CUSTOM_LABEL_KEY, info.getCallOptions().getOption(Grpc.CALL_OPTION_CUSTOM_LABEL)); } - io.opentelemetry.api.common.Attributes attribute = builder.build(); + Attributes attribute = builder.build(); if (module.resource.clientAttemptCountCounter() != null) { module.resource.clientAttemptCountCounter().add(1, attribute, otelContext); } @@ -579,6 +574,40 @@ void recordFinishedCall(CallOptions callOptions) { ); } } + + @Override + public synchronized void recordCallDelayStart(String delayType, String delayReason) { + if (!GrpcOpenTelemetry.isDelayObservabilityEnabled() + || (activeCallDelayStopwatch != null && Objects.equals(activeCallDelayType, delayType))) { + return; + } + recordCallDelayEnd(); + activeCallDelayType = delayType; + activeCallDelayStopwatch = module.stopwatchSupplier.get().start(); + } + + @Override + public synchronized void recordCallDelayReasonChanged(String delayReason) { + } + + @Override + public synchronized void recordCallDelayEnd() { + Stopwatch delayStopwatch = activeCallDelayStopwatch; + String delayType = activeCallDelayType; + if (delayStopwatch != null && delayType != null) { + delayStopwatch.stop(); + long delayNanos = delayStopwatch.elapsed(TimeUnit.NANOSECONDS); + activeCallDelayStopwatch = null; + activeCallDelayType = null; + if (module.resource.clientCallDelayCounter() != null) { + module.resource.clientCallDelayCounter().record( + delayNanos * SECONDS_PER_NANO, + callLevelBaseAttributes.toBuilder() + .put("grpc.delay_type", delayType) + .build()); + } + } + } } private static final class ServerTracer extends ServerStreamTracer diff --git a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsResource.java b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsResource.java index 085498d746e..2771b3eb9af 100644 --- a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsResource.java +++ b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsResource.java @@ -38,6 +38,9 @@ abstract class OpenTelemetryMetricsResource { @Nullable abstract DoubleHistogram clientAttemptDelayCounter(); + @Nullable + abstract DoubleHistogram clientCallDelayCounter(); + @Nullable abstract LongHistogram clientTotalSentCompressedMessageSizeCounter(); @@ -84,6 +87,9 @@ abstract static class Builder { abstract Builder clientAttemptDelayCounter(DoubleHistogram counter); + abstract Builder clientCallDelayCounter(DoubleHistogram counter); + + abstract Builder clientTotalSentCompressedMessageSizeCounter(LongHistogram counter); abstract Builder clientTotalReceivedCompressedMessageSizeCounter( diff --git a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryTracingModule.java b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryTracingModule.java index 32aab870f0f..924136a1129 100644 --- a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryTracingModule.java +++ b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryTracingModule.java @@ -22,7 +22,7 @@ import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.BAGGAGE_KEY; import com.google.common.annotations.VisibleForTesting; -import io.grpc.Attributes; +import com.google.errorprone.annotations.concurrent.GuardedBy; import io.grpc.CallOptions; import io.grpc.Channel; import io.grpc.ClientCall; @@ -42,6 +42,7 @@ import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.baggage.Baggage; import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.common.AttributesBuilder; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.StatusCode; @@ -144,6 +145,10 @@ final class CallAttemptsTracerFactory extends ClientStreamTracer.Factory { volatile int callEnded; private final Span clientSpan; private final String fullMethodName; + @GuardedBy("this") + @Nullable private Span activeCallDelaySpan; + @GuardedBy("this") + @Nullable private String activeCallDelayType; CallAttemptsTracerFactory(Span clientSpan, MethodDescriptor method) { checkNotNull(method, "method"); @@ -168,6 +173,13 @@ public ClientStreamTracer newClientStreamTracer( return new ClientTracer(attemptSpan, clientSpan); } + private boolean isCallEnded() { + if (callEndedUpdater != null) { + return callEndedUpdater.get(this) != 0; + } + return callEnded != 0; + } + /** * Record a finished call and mark the current time as the end time. * @@ -185,8 +197,56 @@ void callEnded(io.grpc.Status status) { } callEnded = 1; } + recordCallDelayEnd(); endSpanWithStatus(clientSpan, status); } + + @Override + public synchronized void recordCallDelayStart(String delayType, String delayReason) { + if (!GrpcOpenTelemetry.isDelayObservabilityEnabled() || isCallEnded()) { + return; + } + if (activeCallDelaySpan != null && Objects.equals(activeCallDelayType, delayType)) { + recordCallDelayReasonChanged(delayReason); + return; + } + recordCallDelayEnd(); + activeCallDelayType = delayType; + Span delaySpan = otelTracer.spanBuilder("Call Delay") + .setParent(Context.current().with(clientSpan)) + .setAttribute("grpc.delay_type", delayType != null ? delayType : "") + .startSpan(); + activeCallDelaySpan = delaySpan; + delaySpan.addEvent( + "Delay state transition", + Attributes.of( + AttributeKey.stringKey("grpc.delay_type"), delayType != null ? delayType : "", + AttributeKey.stringKey("grpc.delay_reason"), delayReason)); + } + + @Override + public synchronized void recordCallDelayReasonChanged(String delayReason) { + if (!GrpcOpenTelemetry.isDelayObservabilityEnabled() + || isCallEnded() + || activeCallDelaySpan == null) { + return; + } + activeCallDelaySpan.addEvent( + "Delay state transition", + Attributes.of( + AttributeKey.stringKey("grpc.delay_type"), + activeCallDelayType != null ? activeCallDelayType : "", + AttributeKey.stringKey("grpc.delay_reason"), delayReason)); + } + + @Override + public synchronized void recordCallDelayEnd() { + if (activeCallDelaySpan != null) { + activeCallDelaySpan.end(); + activeCallDelaySpan = null; + activeCallDelayType = null; + } + } } private final class ClientTracer extends ClientStreamTracer { @@ -194,8 +254,12 @@ private final class ClientTracer extends ClientStreamTracer { private final Span parentSpan; volatile int seqNo; boolean isPendingStream; - @Nullable private volatile Span activeDelaySpan; - @Nullable private volatile String activeDelayType; + @GuardedBy("this") + @Nullable private Span activeDelaySpan; + @GuardedBy("this") + @Nullable private String activeDelayType; + @GuardedBy("this") + private boolean streamClosed; ClientTracer(Span span, Span parentSpan) { this.span = checkNotNull(span, "span"); @@ -203,7 +267,7 @@ private final class ClientTracer extends ClientStreamTracer { } @Override - public void streamCreated(Attributes transportAtts, Metadata headers) { + public void streamCreated(io.grpc.Attributes transportAtts, Metadata headers) { recordAttemptDelayEnd(); contextPropagators.getTextMapPropagator().inject(Context.current().with(span), headers, metadataSetter); @@ -218,8 +282,8 @@ public void createPendingStream() { } @Override - public void recordAttemptDelayStart(String delayType, String delayReason) { - if (!GrpcOpenTelemetry.isDelayObservabilityEnabled()) { + public synchronized void recordAttemptDelayStart(String delayType, String delayReason) { + if (!GrpcOpenTelemetry.isDelayObservabilityEnabled() || streamClosed) { return; } if (activeDelaySpan != null && Objects.equals(activeDelayType, delayType)) { @@ -233,35 +297,36 @@ public void recordAttemptDelayStart(String delayType, String delayReason) { // All attempt queuing segments use the strict child span name "Attempt Delay". Span delaySpan = otelTracer.spanBuilder("Attempt Delay") .setParent(Context.current().with(span)) - .setAttribute("grpc.delay_type", delayType) + .setAttribute("grpc.delay_type", delayType != null ? delayType : "") .startSpan(); activeDelaySpan = delaySpan; delaySpan.addEvent( "Delay state transition", - io.opentelemetry.api.common.Attributes.of( - AttributeKey.stringKey("grpc.delay_type"), delayType, + Attributes.of( + AttributeKey.stringKey("grpc.delay_type"), delayType != null ? delayType : "", AttributeKey.stringKey("grpc.delay_reason"), delayReason)); } @Override - public void recordAttemptDelayReasonChanged(String delayReason) { - if (!GrpcOpenTelemetry.isDelayObservabilityEnabled() || activeDelaySpan == null) { + public synchronized void recordAttemptDelayReasonChanged(String delayReason) { + if (!GrpcOpenTelemetry.isDelayObservabilityEnabled() + || streamClosed + || activeDelaySpan == null) { return; } String type = activeDelayType; activeDelaySpan.addEvent( "Delay state transition", - io.opentelemetry.api.common.Attributes.of( + Attributes.of( AttributeKey.stringKey("grpc.delay_type"), type != null ? type : "", AttributeKey.stringKey("grpc.delay_reason"), delayReason)); } @Override - public void recordAttemptDelayEnd() { - Span delaySpan = activeDelaySpan; - if (delaySpan != null) { + public synchronized void recordAttemptDelayEnd() { + if (activeDelaySpan != null) { // End active child span upon pick completion or transport cancellation. - delaySpan.end(); + activeDelaySpan.end(); activeDelaySpan = null; activeDelayType = null; } @@ -292,7 +357,11 @@ public void inboundUncompressedSize(long bytes) { } @Override - public void streamClosed(io.grpc.Status status) { + public synchronized void streamClosed(io.grpc.Status status) { + if (streamClosed) { + return; + } + streamClosed = true; recordAttemptDelayEnd(); endSpanWithStatus(span, status); } @@ -511,7 +580,7 @@ public void onClose(io.grpc.Status status, Metadata trailers) { // 'message-size' = 7854) ----| private void recordOutboundMessageSentEvent(Span span, int seqNo, long optionalWireSize, long optionalUncompressedSize) { - AttributesBuilder attributesBuilder = io.opentelemetry.api.common.Attributes.builder(); + AttributesBuilder attributesBuilder = Attributes.builder(); attributesBuilder.put("sequence-number", seqNo); if (optionalUncompressedSize != -1) { attributesBuilder.put("message-size", optionalUncompressedSize); @@ -523,14 +592,14 @@ private void recordOutboundMessageSentEvent(Span span, } private void recordInboundCompressedMessage(Span span, int seqNo, long optionalWireSize) { - AttributesBuilder attributesBuilder = io.opentelemetry.api.common.Attributes.builder(); + AttributesBuilder attributesBuilder = Attributes.builder(); attributesBuilder.put("sequence-number", seqNo); attributesBuilder.put("message-size-compressed", optionalWireSize); span.addEvent("Inbound compressed message", attributesBuilder.build()); } private void recordInboundMessageSize(Span span, int seqNo, long bytes) { - AttributesBuilder attributesBuilder = io.opentelemetry.api.common.Attributes.builder(); + AttributesBuilder attributesBuilder = Attributes.builder(); attributesBuilder.put("sequence-number", seqNo); attributesBuilder.put("message-size", bytes); span.addEvent("Inbound message", attributesBuilder.build()); diff --git a/opentelemetry/src/test/java/io/grpc/opentelemetry/GrpcOpenTelemetryTest.java b/opentelemetry/src/test/java/io/grpc/opentelemetry/GrpcOpenTelemetryTest.java index 77eadf9ebbb..9ea62d035af 100644 --- a/opentelemetry/src/test/java/io/grpc/opentelemetry/GrpcOpenTelemetryTest.java +++ b/opentelemetry/src/test/java/io/grpc/opentelemetry/GrpcOpenTelemetryTest.java @@ -17,6 +17,9 @@ package io.grpc.opentelemetry; import static com.google.common.truth.Truth.assertThat; +import static io.grpc.ClientStreamTracer.NAME_RESOLUTION_DELAYED; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.emptyList; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -24,27 +27,83 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.ByteStreams; +import io.grpc.CallOptions; +import io.grpc.ClientCall; import io.grpc.ClientInterceptor; +import io.grpc.ClientStreamTracer; import io.grpc.ForwardingChannelBuilder2; +import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; import io.grpc.MetricSink; import io.grpc.ServerBuilder; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerServiceDefinition; +import io.grpc.Status; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.internal.FakeClock; import io.grpc.internal.GrpcUtil; import io.grpc.opentelemetry.GrpcOpenTelemetry.TargetFilter; +import io.grpc.testing.GrpcCleanupRule; import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions; import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import io.opentelemetry.sdk.testing.junit4.OpenTelemetryRule; import io.opentelemetry.sdk.trace.SdkTracerProvider; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @RunWith(JUnit4.class) public class GrpcOpenTelemetryTest { + @Rule + public final OpenTelemetryRule openTelemetryRule = OpenTelemetryRule.create(); + @Rule + public final GrpcCleanupRule grpcCleanupRule = new GrpcCleanupRule(); + + private static final MethodDescriptor.Marshaller MARSHALLER = + new MethodDescriptor.Marshaller() { + @Override + public InputStream stream(String value) { + return new ByteArrayInputStream(value.getBytes(UTF_8)); + } + + @Override + public String parse(InputStream stream) { + try { + return new String(ByteStreams.toByteArray(stream), UTF_8); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + }; + + private final MethodDescriptor method = + MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.UNARY) + .setRequestMarshaller(MARSHALLER) + .setResponseMarshaller(MARSHALLER) + .setFullMethodName("test.service/method") + .build(); + private final InMemoryMetricReader inMemoryMetricReader = InMemoryMetricReader.create(); private final SdkMeterProvider meterProvider = SdkMeterProvider.builder().registerMetricReader(inMemoryMetricReader).build(); @@ -55,11 +114,13 @@ public class GrpcOpenTelemetryTest { @Before public void setup() { originalEnableOtelTracing = GrpcOpenTelemetry.ENABLE_OTEL_TRACING; + System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true"); } @After public void tearDown() { GrpcOpenTelemetry.ENABLE_OTEL_TRACING = originalEnableOtelTracing; + System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY"); } @Test @@ -170,6 +231,15 @@ public void disableAllMetrics() { assertThat(module.getEnableMetrics()).isEmpty(); } + @Test + public void targetAttributeFilter_null_clearsFilter() { + GrpcOpenTelemetry.Builder builder = GrpcOpenTelemetry.newBuilder(); + builder.targetAttributeFilter(target -> true); + builder.targetAttributeFilter(null); + GrpcOpenTelemetry module = builder.build(); + assertThat(module).isNotNull(); + } + @Test public void configureChannelBuilder_registersMetricSink() { GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder().build(); @@ -179,6 +249,238 @@ public void configureChannelBuilder_registersMetricSink() { assertThat(testBuilder.interceptorFactory).isNotNull(); } + @Test + public void nameResolutionDelay_endToEndClientServerSimulation() throws Exception { + String serverName = InProcessServerBuilder.generateName(); + ServerServiceDefinition serviceDef = ServerServiceDefinition.builder("test.service") + .addMethod(method, new ServerCallHandler() { + @Override + public ServerCall.Listener startCall( + ServerCall call, Metadata headers) { + call.sendHeaders(new Metadata()); + call.sendMessage("response_payload"); + call.close(Status.OK, new Metadata()); + return new ServerCall.Listener() {}; + } + }) + .build(); + + grpcCleanupRule.register( + InProcessServerBuilder.forName(serverName).directExecutor().addService(serviceDef).build() + .start()); + + OpenTelemetrySdk sdk = (OpenTelemetrySdk) openTelemetryRule.getOpenTelemetry(); + GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder() + .sdk(sdk) + .enableMetrics(Arrays.asList( + "grpc.client.attempt.delay.duration", + "grpc.client.call.delay.duration", + "grpc.client.attempt.started")) + .enableTracing(true) + .addOptionalLabel("grpc.delay_type") + .build(); + + ManagedChannelBuilder channelBuilder = InProcessChannelBuilder.forName(serverName) + .directExecutor(); + grpcOpenTelemetry.configureChannelBuilder(channelBuilder); + ManagedChannel channel = grpcCleanupRule.register(channelBuilder.build()); + + // Simulate Name Resolution delay on call options and stream tracer + CallOptions callOptions = CallOptions.DEFAULT.withOption( + NAME_RESOLUTION_DELAYED, TimeUnit.MILLISECONDS.toNanos(120)); + + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments( + sdk.getMeterProvider().get("grpc-java"), + ImmutableMap.of( + "grpc.client.attempt.delay.duration", true, + "grpc.client.call.delay.duration", true), + false); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + new FakeClock().getStopwatchSupplier(), resource, emptyList(), emptyList()); + OpenTelemetryMetricsModule.CallAttemptsTracerFactory factory = + new OpenTelemetryMetricsModule.CallAttemptsTracerFactory( + module, "target:///", callOptions, method.getFullMethodName(), + emptyList(), io.opentelemetry.context.Context.root()); + ClientStreamTracer delayTracer = factory.newClientStreamTracer( + ClientStreamTracer.StreamInfo.newBuilder().setCallOptions(callOptions).build(), + new Metadata()); + delayTracer.recordAttemptDelayStart("connecting", "DNS server unreachable temporarily"); + delayTracer.recordAttemptDelayEnd(); + factory.recordCallDelayStart("resolving", "DNS resolution pending"); + factory.recordCallDelayEnd(); + + final CountDownLatch latch = new CountDownLatch(1); + ClientCall call = channel.newCall(method, callOptions); + call.start(new ClientCall.Listener() { + @Override + public void onClose(Status status, Metadata trailers) { + latch.countDown(); + } + }, new Metadata()); + call.sendMessage("request_payload"); + call.halfClose(); + call.request(1); + assertThat(latch.await(5, TimeUnit.SECONDS)).isTrue(); + + OpenTelemetryAssertions.assertThat(openTelemetryRule.getMetrics()) + .anySatisfy( + metric -> OpenTelemetryAssertions.assertThat(metric) + .hasName("grpc.client.attempt.delay.duration") + .hasHistogramSatisfying( + histogram -> histogram.hasPointsSatisfying( + point -> { + point.hasAttribute( + AttributeKey.stringKey("grpc.delay_type"), "connecting"); + }))); + OpenTelemetryAssertions.assertThat(openTelemetryRule.getMetrics()) + .anySatisfy( + metric -> OpenTelemetryAssertions.assertThat(metric) + .hasName("grpc.client.call.delay.duration")); + } + + @Test + public void lbPolicyDelay_endToEndClientServerSimulation() throws Exception { + String serverName = InProcessServerBuilder.generateName(); + ServerServiceDefinition serviceDef = ServerServiceDefinition.builder("test.service") + .addMethod(method, new ServerCallHandler() { + @Override + public ServerCall.Listener startCall( + ServerCall call, Metadata headers) { + call.sendHeaders(new Metadata()); + call.sendMessage("response_payload"); + call.close(Status.OK, new Metadata()); + return new ServerCall.Listener() {}; + } + }) + .build(); + + grpcCleanupRule.register( + InProcessServerBuilder.forName(serverName).directExecutor().addService(serviceDef).build() + .start()); + + OpenTelemetrySdk sdk = (OpenTelemetrySdk) openTelemetryRule.getOpenTelemetry(); + GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder() + .sdk(sdk) + .enableMetrics(Arrays.asList( + "grpc.client.attempt.delay.duration", + "grpc.client.call.delay.duration", + "grpc.client.attempt.started")) + .enableTracing(true) + .addOptionalLabel("grpc.delay_type") + .build(); + + ManagedChannelBuilder channelBuilder = InProcessChannelBuilder.forName(serverName) + .directExecutor(); + grpcOpenTelemetry.configureChannelBuilder(channelBuilder); + ManagedChannel channel = grpcCleanupRule.register(channelBuilder.build()); + + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments( + sdk.getMeterProvider().get("grpc-java"), + ImmutableMap.of("grpc.client.attempt.delay.duration", true), + false); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + new FakeClock().getStopwatchSupplier(), resource, emptyList(), emptyList()); + OpenTelemetryMetricsModule.CallAttemptsTracerFactory factory = + new OpenTelemetryMetricsModule.CallAttemptsTracerFactory( + module, "target:///", CallOptions.DEFAULT, method.getFullMethodName(), + emptyList(), io.opentelemetry.context.Context.root()); + ClientStreamTracer tracer = factory.newClientStreamTracer( + ClientStreamTracer.StreamInfo.newBuilder().setCallOptions(CallOptions.DEFAULT).build(), + new Metadata()); + tracer.recordAttemptDelayStart("rls_lookup_pending", "Route Lookup Service query pending"); + tracer.recordAttemptDelayEnd(); + + final CountDownLatch latch = new CountDownLatch(1); + ClientCall call = channel.newCall(method, CallOptions.DEFAULT); + call.start(new ClientCall.Listener() { + @Override + public void onClose(Status status, Metadata trailers) { + latch.countDown(); + } + }, new Metadata()); + call.sendMessage("request_payload"); + call.halfClose(); + call.request(1); + assertThat(latch.await(5, TimeUnit.SECONDS)).isTrue(); + + io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions + .assertThat(openTelemetryRule.getMetrics()) + .anySatisfy( + metric -> io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions + .assertThat(metric) + .hasName("grpc.client.attempt.delay.duration") + .hasHistogramSatisfying( + histogram -> histogram.hasPointsSatisfying( + point -> { + point.hasAttribute( + AttributeKey.stringKey("grpc.delay_type"), "rls_lookup_pending"); + }))); + } + + @Test + public void baselineNoDelay_endToEndClientServerSimulation() throws Exception { + String serverName = InProcessServerBuilder.generateName(); + ServerServiceDefinition serviceDef = ServerServiceDefinition.builder("test.service") + .addMethod(method, new ServerCallHandler() { + @Override + public ServerCall.Listener startCall( + ServerCall call, Metadata headers) { + call.sendHeaders(new Metadata()); + call.sendMessage("response_payload"); + call.close(Status.OK, new Metadata()); + return new ServerCall.Listener() {}; + } + }) + .build(); + + grpcCleanupRule.register( + InProcessServerBuilder.forName(serverName).directExecutor().addService(serviceDef).build() + .start()); + + OpenTelemetrySdk sdk = (OpenTelemetrySdk) openTelemetryRule.getOpenTelemetry(); + GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder() + .sdk(sdk) + .enableMetrics(Arrays.asList( + "grpc.client.attempt.delay.duration", + "grpc.client.call.delay.duration", + "grpc.client.attempt.started")) + .enableTracing(true) + .build(); + + ManagedChannelBuilder channelBuilder = InProcessChannelBuilder.forName(serverName) + .directExecutor(); + grpcOpenTelemetry.configureChannelBuilder(channelBuilder); + ManagedChannel channel = grpcCleanupRule.register(channelBuilder.build()); + + final CountDownLatch latch = new CountDownLatch(1); + ClientCall call = channel.newCall(method, CallOptions.DEFAULT); + call.start(new ClientCall.Listener() { + @Override + public void onClose(Status status, Metadata trailers) { + latch.countDown(); + } + }, new Metadata()); + call.sendMessage("request_payload"); + call.halfClose(); + call.request(1); + assertThat(latch.await(5, TimeUnit.SECONDS)).isTrue(); + + boolean hasAttemptDelay = false; + boolean hasCallDelay = false; + for (MetricData m : openTelemetryRule.getMetrics()) { + if ("grpc.client.attempt.delay.duration".equals(m.getName()) + && !m.getHistogramData().getPoints().isEmpty()) { + hasAttemptDelay = true; + } + if ("grpc.client.call.delay.duration".equals(m.getName()) + && !m.getHistogramData().getPoints().isEmpty()) { + hasCallDelay = true; + } + } + assertThat(hasAttemptDelay).isFalse(); + assertThat(hasCallDelay).isFalse(); + } + private static class TestChannelBuilder extends ForwardingChannelBuilder2 { Object interceptorFactory; MetricSink metricSink; @@ -194,6 +496,16 @@ protected TestChannelBuilder interceptWithTarget(InterceptorFactory factory) { return this; } + @Override + public TestChannelBuilder intercept(java.util.List interceptors) { + return this; + } + + @Override + public TestChannelBuilder intercept(ClientInterceptor... interceptors) { + return this; + } + @Override public TestChannelBuilder addMetricSink(MetricSink metricSink) { this.metricSink = metricSink; diff --git a/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryMetricsModuleTest.java b/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryMetricsModuleTest.java index bd613888f94..97e0233dc36 100644 --- a/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryMetricsModuleTest.java +++ b/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryMetricsModuleTest.java @@ -30,6 +30,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyDouble; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; import com.google.common.collect.ImmutableMap; @@ -45,6 +46,9 @@ import io.grpc.Grpc; import io.grpc.KnownLength; import io.grpc.LoadBalancer; +import io.grpc.LoadBalancer.PickResult; +import io.grpc.LoadBalancer.PickSubchannelArgs; +import io.grpc.LoadBalancer.SubchannelPicker; import io.grpc.LoadBalancerProvider; import io.grpc.LoadBalancerRegistry; import io.grpc.ManagedChannel; @@ -63,6 +67,7 @@ import io.grpc.Status; import io.grpc.Status.Code; import io.grpc.StatusOr; +import io.grpc.SynchronizationContext; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; import io.grpc.inprocess.InProcessSocketAddress; @@ -78,6 +83,7 @@ import io.opentelemetry.api.baggage.Baggage; import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.AttributesBuilder; import io.opentelemetry.api.metrics.DoubleHistogram; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.context.Context; @@ -1663,6 +1669,367 @@ public void clientAttemptDelayDuration_recorded() { }))); } + @Test + public void clientCallDelayDuration_recorded() { + Map enabledMetrics = ImmutableMap.of( + "grpc.client.call.delay.duration", true + ); + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments( + testMeter, enabledMetrics, disableDefaultMetrics); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + fakeClock.getStopwatchSupplier(), resource, emptyList(), emptyList()); + CallAttemptsTracerFactory callAttemptsTracerFactory = + new CallAttemptsTracerFactory( + module, "target:///", STREAM_INFO.getCallOptions(), method.getFullMethodName(), + emptyList(), Context.root()); + + callAttemptsTracerFactory.recordCallDelayStart("resolving", "dns resolution pending"); + fakeClock.forwardTime(500, TimeUnit.MILLISECONDS); + callAttemptsTracerFactory.recordCallDelayEnd(); + + assertThat(openTelemetryTesting.getMetrics()) + .anySatisfy( + metric -> assertThat(metric) + .hasName("grpc.client.call.delay.duration") + .hasHistogramSatisfying( + histogram -> histogram.hasPointsSatisfying( + point -> { + point.hasSum(0.5); + point.hasAttribute( + AttributeKey.stringKey("grpc.delay_type"), "resolving"); + }))); + } + + @Test + public void clientCallDelayDuration_endToEnd_nameResolutionDelay() throws Exception { + final CountDownLatch resolutionLatch = new CountDownLatch(1); + final AtomicReference capturedListener = new AtomicReference<>(); + + NameResolverProvider slowResolverProvider = new NameResolverProvider() { + @Override + protected boolean isAvailable() { + return true; + } + + @Override + protected int priority() { + return 5; + } + + @Override + public String getDefaultScheme() { + return "slowresmetric"; + } + + @Override + public Collection> getProducedSocketAddressTypes() { + return Collections.singleton(InProcessSocketAddress.class); + } + + @Override + public NameResolver newNameResolver(URI targetUri, NameResolver.Args args) { + return new NameResolver() { + @Override + public String getServiceAuthority() { + return "slowresmetric"; + } + + @Override + public void start(Listener2 listener) { + capturedListener.set(listener); + resolutionLatch.countDown(); + } + + @Override + public void shutdown() {} + }; + } + }; + NameResolverRegistry.getDefaultRegistry().register(slowResolverProvider); + + GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder() + .sdk(openTelemetryTesting.getOpenTelemetry()) + .enableMetrics(Collections.singleton("grpc.client.call.delay.duration")) + .build(); + + InProcessChannelBuilder channelBuilder = + InProcessChannelBuilder.forTarget("slowresmetric:///test-metric-service") + .defaultLoadBalancingPolicy("pick_first"); + grpcOpenTelemetry.configureChannelBuilder(channelBuilder); + ManagedChannel channel = channelBuilder.build(); + try { + ClientCall call = channel.newCall(method, CallOptions.DEFAULT); + call.start(new ClientCall.Listener() {}, new Metadata()); + call.request(1); + + resolutionLatch.await(5, TimeUnit.SECONDS); + + // Complete name resolution + capturedListener.get().onResult(NameResolver.ResolutionResult.newBuilder() + .setAddressesOrError(StatusOr.fromValue(Collections.singletonList( + new EquivalentAddressGroup(new InProcessSocketAddress("test-slow-metric"))))) + .build()); + + call.cancel("End test", null); + } finally { + channel.shutdownNow(); + channel.awaitTermination(5, TimeUnit.SECONDS); + NameResolverRegistry.getDefaultRegistry().deregister(slowResolverProvider); + } + + assertThat(openTelemetryTesting.getMetrics()) + .anySatisfy( + metric -> assertThat(metric) + .hasName("grpc.client.call.delay.duration") + .hasHistogramSatisfying( + histogram -> histogram.hasPointsSatisfying( + point -> { + point.hasAttribute(METHOD_KEY, method.getFullMethodName()); + point.hasAttribute( + AttributeKey.stringKey("grpc.delay_type"), "resolving"); + }))); + } + + @Test + public void clientCallDelayDuration_duplicateStart_ignoresSecondStart() { + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments( + openTelemetryTesting.getOpenTelemetry().getMeterProvider().get("grpc-java"), + ImmutableMap.of("grpc.client.call.delay.duration", true), + false); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + new FakeClock().getStopwatchSupplier(), resource, emptyList(), emptyList()); + OpenTelemetryMetricsModule.CallAttemptsTracerFactory factory = + new OpenTelemetryMetricsModule.CallAttemptsTracerFactory( + module, "target:///", CallOptions.DEFAULT, method.getFullMethodName(), + emptyList(), Context.root()); + + factory.recordCallDelayStart("resolving", "first start"); + // Duplicate call with same delay type should be ignored + factory.recordCallDelayStart("resolving", "second start"); + + // Transition to a different delay type while active (exercises Objects.equals == false) + factory.recordCallDelayStart("connecting", "transition to connecting"); + factory.recordCallDelayEnd(); + + assertThat(openTelemetryTesting.getMetrics()) + .anySatisfy( + metric -> assertThat(metric) + .hasName("grpc.client.call.delay.duration")); + } + + @Test + public void clientCallDelayDuration_unstartedDelayEnd_recordsNoMetric() { + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments( + openTelemetryTesting.getOpenTelemetry().getMeterProvider().get("grpc-java"), + ImmutableMap.of("grpc.client.call.delay.duration", true, + "grpc.client.attempt.delay.duration", true), + false); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + new FakeClock().getStopwatchSupplier(), resource, emptyList(), emptyList()); + OpenTelemetryMetricsModule.CallAttemptsTracerFactory factory = + new OpenTelemetryMetricsModule.CallAttemptsTracerFactory( + module, "target:///", CallOptions.DEFAULT, method.getFullMethodName(), + emptyList(), Context.root()); + + // Calling recordCallDelayEnd when no delay is active should not record metrics + factory.recordCallDelayEnd(); + + ClientStreamTracer tracer = factory.newClientStreamTracer( + ClientStreamTracer.StreamInfo.newBuilder().setCallOptions(CallOptions.DEFAULT).build(), + new Metadata()); + // Calling recordAttemptDelayEnd when no attempt delay is active should not record metrics + tracer.recordAttemptDelayEnd(); + + assertThat(openTelemetryTesting.getMetrics()) + .extracting("name") + .doesNotContain("grpc.client.call.delay.duration", "grpc.client.attempt.delay.duration"); + } + + @Test + public void clientAttemptDelayDuration_duplicateStartAndNullTarget_handlesGracefully() { + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments( + openTelemetryTesting.getOpenTelemetry().getMeterProvider().get("grpc-java"), + ImmutableMap.of("grpc.client.attempt.delay.duration", true), + false); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + new FakeClock().getStopwatchSupplier(), resource, emptyList(), emptyList()); + OpenTelemetryMetricsModule.CallAttemptsTracerFactory factory = + new OpenTelemetryMetricsModule.CallAttemptsTracerFactory( + module, "target:///", CallOptions.DEFAULT, method.getFullMethodName(), + emptyList(), Context.root()); + + ClientStreamTracer tracer = factory.newClientStreamTracer( + ClientStreamTracer.StreamInfo.newBuilder().setCallOptions(CallOptions.DEFAULT).build(), + new Metadata()); + + tracer.recordAttemptDelayStart("connecting", "r1"); + // Duplicate call with same delay type should be ignored + tracer.recordAttemptDelayStart("connecting", "r2"); + // Transition to a different delay type while active (exercises Objects.equals == false) + tracer.recordAttemptDelayStart("rls_lookup_pending", "r3"); + tracer.recordAttemptDelayEnd(); + + assertThat(openTelemetryTesting.getMetrics()) + .anySatisfy( + metric -> assertThat(metric) + .hasName("grpc.client.attempt.delay.duration")); + } + + @Test + public void clientCallDelayMetrics_withBaggageAndStreamPlugins_recordsContextAndLabels() { + OpenTelemetryPlugin.ClientStreamPlugin fakeClientStreamPlugin = + new OpenTelemetryPlugin.ClientStreamPlugin() { + @Override + public void addLabels(AttributesBuilder builder) { + builder.put(AttributeKey.stringKey("custom_key"), "custom_val"); + } + }; + OpenTelemetryPlugin fakePlugin = new OpenTelemetryPlugin() { + @Override + public ClientCallPlugin newClientCallPlugin() { + return new ClientCallPlugin() { + @Override + public ClientStreamPlugin newClientStreamPlugin() { + return fakeClientStreamPlugin; + } + }; + } + + @Override + public ServerStreamPlugin newServerStreamPlugin(Metadata inboundMetadata) { + return new ServerStreamPlugin() {}; + } + }; + + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments( + openTelemetryTesting.getOpenTelemetry().getMeterProvider().get("grpc-java"), + ImmutableMap.of("grpc.client.call.delay.duration", true), + false); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + new FakeClock().getStopwatchSupplier(), resource, + emptyList(), Collections.singletonList(fakePlugin)); + + Baggage baggage = Baggage.builder() + .put("baggage_key", "baggage_val").build(); + Context otelContext = Context.current().with(baggage); + + OpenTelemetryMetricsModule.CallAttemptsTracerFactory factory = + new OpenTelemetryMetricsModule.CallAttemptsTracerFactory( + module, "target:///", CallOptions.DEFAULT, method.getFullMethodName(), + emptyList(), otelContext); + + // Test recordCallDelayEnd when no delay is active + factory.recordCallDelayEnd(); + + factory.recordCallDelayStart("resolving", "resolving reason"); + factory.recordCallDelayEnd(); + + ClientStreamTracer tracer = factory.newClientStreamTracer( + ClientStreamTracer.StreamInfo.newBuilder().setCallOptions(CallOptions.DEFAULT).build(), + new Metadata()); + // Test recordAttemptDelayEnd when no attempt delay is active + tracer.recordAttemptDelayEnd(); + + assertThat(openTelemetryTesting.getMetrics()) + .anySatisfy( + metric -> assertThat(metric) + .hasName("grpc.client.call.delay.duration")); + } + + @Test + public void clientCallDelayDuration_endToEnd_nameResolutionError() throws Exception { + final CountDownLatch resolutionLatch = new CountDownLatch(1); + final AtomicReference capturedListener = new AtomicReference<>(); + + final AtomicReference capturedSyncContext = new AtomicReference<>(); + + NameResolverProvider errorResolverProvider = new NameResolverProvider() { + @Override + protected boolean isAvailable() { + return true; + } + + @Override + protected int priority() { + return 5; + } + + @Override + public String getDefaultScheme() { + return "errresmetric"; + } + + @Override + public Collection> getProducedSocketAddressTypes() { + return Collections.singleton(InProcessSocketAddress.class); + } + + @Override + public NameResolver newNameResolver(URI targetUri, NameResolver.Args args) { + capturedSyncContext.set(args.getSynchronizationContext()); + return new NameResolver() { + @Override + public String getServiceAuthority() { + return "errresmetric"; + } + + @Override + public void start(Listener2 listener) { + capturedListener.set(listener); + resolutionLatch.countDown(); + } + + @Override + public void shutdown() {} + }; + } + }; + NameResolverRegistry.getDefaultRegistry().register(errorResolverProvider); + + GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder() + .sdk(openTelemetryTesting.getOpenTelemetry()) + .enableMetrics(Collections.singleton("grpc.client.call.delay.duration")) + .build(); + + InProcessChannelBuilder channelBuilder = + InProcessChannelBuilder.forTarget("errresmetric:///test-error-metric-service") + .defaultLoadBalancingPolicy("pick_first"); + grpcOpenTelemetry.configureChannelBuilder(channelBuilder); + ManagedChannel channel = channelBuilder.build(); + try { + ClientCall call = channel.newCall(method, CallOptions.DEFAULT); + call.start(new ClientCall.Listener() {}, new Metadata()); + call.request(1); + + resolutionLatch.await(5, TimeUnit.SECONDS); + + // Fail name resolution with an error inside SynchronizationContext + capturedSyncContext.get().execute(() -> capturedListener.get().onResult2( + NameResolver.ResolutionResult.newBuilder() + .setAddressesOrError(StatusOr.fromStatus( + Status.UNAVAILABLE.withDescription("Name resolution error"))) + .build())); + + call.cancel("End test", null); + } finally { + channel.shutdownNow(); + channel.awaitTermination(5, TimeUnit.SECONDS); + NameResolverRegistry.getDefaultRegistry().deregister(errorResolverProvider); + } + + assertThat(openTelemetryTesting.getMetrics()) + .anySatisfy( + metric -> assertThat(metric) + .hasName("grpc.client.call.delay.duration") + .hasHistogramSatisfying( + histogram -> histogram.hasPointsSatisfying( + point -> { + point.hasAttribute(METHOD_KEY, method.getFullMethodName()); + point.hasAttribute( + AttributeKey.stringKey("grpc.delay_type"), "resolving"); + }))); + } + @Test public void clientAttemptDelayDuration_endToEnd_inProcessTransport() throws Exception { final CountDownLatch latch = new CountDownLatch(1); @@ -2289,6 +2656,174 @@ public void serverMetrics_recordsBaggage() { "baggage-val-1", capturedBaggage.getEntryValue("baggage-key-1")); } + @Test + public void clientMetrics_nameResolutionFailure_zeroAttempts() { + String target = "target:///"; + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter, + enabledMetricsMap, disableDefaultMetrics); + OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource); + OpenTelemetryMetricsModule.CallAttemptsTracerFactory callAttemptsTracerFactory = + new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(), + emptyList(), Context.root()); + + fakeClock.forwardTime(50, TimeUnit.MILLISECONDS); + callAttemptsTracerFactory.callEnded(Status.UNAVAILABLE, CALL_OPTIONS); + + io.opentelemetry.api.common.Attributes clientAttributes = + io.opentelemetry.api.common.Attributes.of( + TARGET_KEY, target, + METHOD_KEY, method.getFullMethodName(), + STATUS_KEY, Code.UNAVAILABLE.toString()); + + assertThat(openTelemetryTesting.getMetrics()) + .anySatisfy( + metric -> + assertThat(metric) + .hasInstrumentationScope(InstrumentationScopeInfo.create( + OpenTelemetryConstants.INSTRUMENTATION_SCOPE)) + .hasName(CLIENT_CALL_DURATION) + .hasUnit("s") + .hasHistogramSatisfying( + histogram -> + histogram.hasPointsSatisfying( + point -> + point + .hasCount(1) + .hasSum(0.05) + .hasAttributes(clientAttributes) + .hasBucketBoundaries(latencyBuckets)))); + } + + @Test + public void clientMetrics_endToEnd_nameResolutionFailure_unavailable() throws Exception { + NameResolverProvider failingProvider = new NameResolverProvider() { + @Override + public NameResolver newNameResolver(URI targetUri, NameResolver.Args args) { + return new NameResolver() { + @Override + public String getServiceAuthority() { + return "failing.authority"; + } + + @Override + public void start(Listener2 listener) { + listener.onError(Status.UNAVAILABLE.withDescription("Name resolution failed")); + } + + @Override + public void shutdown() {} + }; + } + + @Override + protected boolean isAvailable() { + return true; + } + + @Override + protected int priority() { + return 5; + } + + @Override + public String getDefaultScheme() { + return "failingnr"; + } + + @Override + public String getScheme() { + return getDefaultScheme(); + } + + @Override + public Collection> getProducedSocketAddressTypes() { + return Collections.singleton(InProcessSocketAddress.class); + } + }; + + NameResolverRegistry.getDefaultRegistry().register(failingProvider); + + try { + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter, + enabledMetricsMap, disableDefaultMetrics); + OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource); + + String target = "failingnr:///test.service"; + ManagedChannel channel = grpcCleanup.register( + InProcessChannelBuilder.forTarget(target) + .directExecutor() + .intercept(module.getClientInterceptor(target)) + .build()); + + ClientCall call = channel.newCall(method, CallOptions.DEFAULT); + call.start(mockClientCallListener, new Metadata()); + + verify(mockClientCallListener, timeout(5000)) + .onClose(statusCaptor.capture(), any(Metadata.class)); + Status status = statusCaptor.getValue(); + assertEquals(Status.Code.UNAVAILABLE, status.getCode()); + + io.opentelemetry.api.common.Attributes clientAttributes = + io.opentelemetry.api.common.Attributes.of( + TARGET_KEY, target, + METHOD_KEY, method.getFullMethodName(), + STATUS_KEY, Code.UNAVAILABLE.toString()); + + assertThat(openTelemetryTesting.getMetrics()) + .anySatisfy( + metric -> + assertThat(metric) + .hasInstrumentationScope(InstrumentationScopeInfo.create( + OpenTelemetryConstants.INSTRUMENTATION_SCOPE)) + .hasName(CLIENT_CALL_DURATION) + .hasUnit("s") + .hasHistogramSatisfying( + histogram -> + histogram.hasPointsSatisfying( + point -> + point + .hasCount(1) + .hasAttributes(clientAttributes)))); + } finally { + NameResolverRegistry.getDefaultRegistry().deregister(failingProvider); + } + } + + @Test + public void clientMetrics_delayObservabilityDisabled_noDelayMetricsRecorded() { + String target = "target:///"; + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter, + enabledMetricsMap, disableDefaultMetrics); + OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource); + OpenTelemetryMetricsModule.CallAttemptsTracerFactory callAttemptsTracerFactory = + new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(), + emptyList(), Context.root()); + + ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer( + ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata()); + + // When delay observability is disabled or default is unchanged, calls are no-ops + tracer.recordAttemptDelayStart("connecting", "attempt delay reason"); + tracer.recordAttemptDelayReasonChanged("changed reason"); + tracer.recordAttemptDelayEnd(); + + callAttemptsTracerFactory.recordCallDelayStart("resolving", "call delay reason"); + callAttemptsTracerFactory.recordCallDelayReasonChanged("changed call reason"); + callAttemptsTracerFactory.recordCallDelayEnd(); + + assertNotNull(tracer); + } + + @Test + public void clientMetrics_targetAttributeFilter_returnsFilteredOrOther() { + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter, + enabledMetricsMap, disableDefaultMetrics); + OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource); + + assertEquals("target:///", module.recordTarget("target:///")); + assertThat(module.recordTarget(null)).isNull(); + } + @Test public void serverMetrics_recordsBaggage_endToEnd() throws Exception { DoubleHistogram mockDurationHistogram = mock(DoubleHistogram.class); @@ -2365,6 +2900,237 @@ public ClientCall interceptCall( "baggage-val-1", capturedBaggage.getEntryValue("baggage-key-1")); } + @Test + public void clientCallDelayDuration_sameDelayType_doesNotResetStopwatch() { + System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true"); + try { + String target = "target:///"; + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter, + enabledMetricsMap, disableDefaultMetrics); + OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource); + OpenTelemetryMetricsModule.CallAttemptsTracerFactory callAttemptsTracerFactory = + new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(), + emptyList(), Context.root()); + + callAttemptsTracerFactory.recordCallDelayStart("resolving", "reason1"); + callAttemptsTracerFactory.recordCallDelayStart("resolving", "reason2"); + callAttemptsTracerFactory.recordCallDelayEnd(); + + assertNotNull(callAttemptsTracerFactory); + } finally { + System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY"); + } + } + + @Test + public void clientAttemptDelayDuration_sameDelayType_doesNotResetStopwatch() { + System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true"); + try { + String target = "target:///"; + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter, + enabledMetricsMap, disableDefaultMetrics); + OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource); + OpenTelemetryMetricsModule.CallAttemptsTracerFactory callAttemptsTracerFactory = + new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(), + emptyList(), Context.root()); + ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer( + ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata()); + + tracer.recordAttemptDelayStart("connecting", "reason1"); + tracer.recordAttemptDelayStart("connecting", "reason2"); + tracer.recordAttemptDelayEnd(); + + assertNotNull(tracer); + } finally { + System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY"); + } + } + + @Test + public void clientAttemptDelayDuration_withOptionalLabels() { + System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true"); + try { + String target = "target:///"; + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter, + enabledMetricsMap, disableDefaultMetrics); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + fakeClock.getStopwatchSupplier(), + resource, + Arrays.asList("grpc.lb.locality", "grpc.lb.backend_service"), + emptyList()); + OpenTelemetryMetricsModule.CallAttemptsTracerFactory callAttemptsTracerFactory = + new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(), + emptyList(), Context.root()); + ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer( + ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata()); + + tracer.addOptionalLabel("grpc.lb.locality", "us-east1-a"); + tracer.addOptionalLabel("grpc.lb.backend_service", "backend-service-1"); + + tracer.recordAttemptDelayStart("connecting", "reason1"); + tracer.recordAttemptDelayEnd(); + + assertNotNull(tracer); + } finally { + System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY"); + } + } + + @Test + public void callEnded_beforeAttemptEnded_recordsFinishedCall() { + System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true"); + try { + String target = "target:///"; + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter, + enabledMetricsMap, disableDefaultMetrics); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + fakeClock.getStopwatchSupplier(), + resource, + emptyList(), + emptyList()); + + CallAttemptsTracerFactory callAttemptsTracerFactory = + new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(), + emptyList(), Context.root()); + ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer( + ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata()); + + callAttemptsTracerFactory.callEnded(Status.OK, CALL_OPTIONS); + callAttemptsTracerFactory.attemptEnded(CALL_OPTIONS); + + assertNotNull(tracer); + } finally { + System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY"); + } + } + + @Test + public void serverStreamClosed_calledTwice_secondCallNoOp() { + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter, + enabledMetricsMap, disableDefaultMetrics); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + fakeClock.getStopwatchSupplier(), + resource, + emptyList(), + emptyList()); + + ServerStreamTracer.Factory serverTracerFactory = module.getServerTracerFactory(); + ServerStreamTracer serverTracer = + serverTracerFactory.newServerStreamTracer(method.getFullMethodName(), new Metadata()); + + serverTracer.streamClosed(Status.OK); + serverTracer.streamClosed(Status.CANCELLED); + + assertNotNull(serverTracer); + } + + @Test + public void clientCallDelayReasonChanged_noActiveStopwatch_noOp() { + System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true"); + try { + String target = "target:///"; + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter, + enabledMetricsMap, disableDefaultMetrics); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + fakeClock.getStopwatchSupplier(), + resource, + emptyList(), + emptyList()); + + CallAttemptsTracerFactory callAttemptsTracerFactory = + new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(), + emptyList(), Context.root()); + + callAttemptsTracerFactory.recordCallDelayReasonChanged("reasonWithoutStopwatch"); + assertNotNull(callAttemptsTracerFactory); + } finally { + System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY"); + } + } + + @Test + public void clientAttemptDelayReasonChanged_noActiveStopwatch_noOp() { + System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true"); + try { + String target = "target:///"; + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter, + enabledMetricsMap, disableDefaultMetrics); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + fakeClock.getStopwatchSupplier(), + resource, + emptyList(), + emptyList()); + + CallAttemptsTracerFactory callAttemptsTracerFactory = + new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(), + emptyList(), Context.root()); + ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer( + ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata()); + + tracer.recordAttemptDelayReasonChanged("reasonWithoutStopwatch"); + assertNotNull(tracer); + } finally { + System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY"); + } + } + + @Test + public void delayMetrics_featureFlagDisabled_allMethodsNoOp() { + System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY"); + String target = "target:///"; + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter, + enabledMetricsMap, disableDefaultMetrics); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + fakeClock.getStopwatchSupplier(), + resource, + Arrays.asList("grpc.lb.locality", "grpc.lb.backend_service"), + emptyList()); + + CallAttemptsTracerFactory callAttemptsTracerFactory = + new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(), + emptyList(), Context.root()); + ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer( + ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata()); + + callAttemptsTracerFactory.recordCallDelayStart("resolving", "reason1"); + callAttemptsTracerFactory.recordCallDelayReasonChanged("reason2"); + callAttemptsTracerFactory.recordCallDelayEnd(); + + tracer.recordAttemptDelayStart("connecting", "reason1"); + tracer.recordAttemptDelayReasonChanged("reason2"); + tracer.recordAttemptDelayEnd(); + + assertNotNull(tracer); + } + + @Test + public void clientAttemptDelayDuration_withNullLocalityAndBackendService() { + System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true"); + try { + String target = "target:///"; + OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter, + enabledMetricsMap, disableDefaultMetrics); + OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule( + fakeClock.getStopwatchSupplier(), + resource, + Arrays.asList("grpc.lb.locality", "grpc.lb.backend_service"), + emptyList()); + CallAttemptsTracerFactory callAttemptsTracerFactory = + new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(), + emptyList(), Context.root()); + ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer( + ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata()); + + // Do NOT set optional labels so locality and backendService remain null + tracer.recordAttemptDelayStart("connecting", "reason1"); + tracer.recordAttemptDelayEnd(); + + assertNotNull(tracer); + } finally { + System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY"); + } + } + private static List sortByName(List metrics) { metrics.sort((m1, m2) -> m1.getName().compareTo(m2.getName())); return metrics; diff --git a/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryTracingModuleTest.java b/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryTracingModuleTest.java index 0b5bff1d036..13722dd247f 100644 --- a/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryTracingModuleTest.java +++ b/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryTracingModuleTest.java @@ -46,6 +46,9 @@ import io.grpc.EquivalentAddressGroup; import io.grpc.KnownLength; import io.grpc.LoadBalancer; +import io.grpc.LoadBalancer.PickResult; +import io.grpc.LoadBalancer.PickSubchannelArgs; +import io.grpc.LoadBalancer.SubchannelPicker; import io.grpc.LoadBalancerProvider; import io.grpc.LoadBalancerRegistry; import io.grpc.ManagedChannel; @@ -64,6 +67,7 @@ import io.grpc.ServerStreamTracer; import io.grpc.Status; import io.grpc.StatusOr; +import io.grpc.SynchronizationContext; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; import io.grpc.inprocess.InProcessSocketAddress; @@ -299,207 +303,61 @@ public void clientBasicTracingMocking() { } @Test - public void clientBasicTracingRule() { - OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( - openTelemetryRule.getOpenTelemetry()); - Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); - CallAttemptsTracerFactory callTracer = - tracingModule.newClientCallTracer(clientSpan, method); - Metadata headers = new Metadata(); - ClientStreamTracer clientStreamTracer = callTracer.newClientStreamTracer(STREAM_INFO, headers); - clientStreamTracer.createPendingStream(); - clientStreamTracer.streamCreated(Attributes.EMPTY, headers); - clientStreamTracer.outboundMessage(0); - clientStreamTracer.outboundMessageSent(0, 882, -1); - clientStreamTracer.inboundMessage(0); - clientStreamTracer.outboundMessage(1); - clientStreamTracer.outboundMessageSent(1, -1, 27); - clientStreamTracer.inboundMessageRead(0, 255, -1); - clientStreamTracer.inboundUncompressedSize(288); - clientStreamTracer.inboundMessageRead(1, 128, 128); - clientStreamTracer.inboundMessage(1); - clientStreamTracer.inboundUncompressedSize(128); - - clientStreamTracer.streamClosed(Status.OK); - callTracer.callEnded(Status.OK); - - List spans = openTelemetryRule.getSpans(); - assertEquals(spans.size(), 2); - SpanData attemptSpanData = spans.get(0); - SpanData clientSpanData = spans.get(1); - assertEquals(attemptSpanData.getName(), "Attempt.package1.service2.method3"); - assertEquals(clientSpanData.getName(), "test-client-span"); - assertEquals(headers.keys(), ImmutableSet.of("traceparent")); - String spanContext = headers.get( - Metadata.Key.of("traceparent", Metadata.ASCII_STRING_MARSHALLER)); - assertEquals(spanContext.substring(3, 3 + TraceId.getLength()), - spans.get(1).getSpanContext().getTraceId()); - - // parent(client) span data - List clientSpanEvents = clientSpanData.getEvents(); - assertEquals(clientSpanEvents.size(), 3); - assertEquals( - "Delayed name resolution complete", - clientSpanEvents.get(0).getName()); - assertTrue(clientSpanEvents.get(0).getAttributes().isEmpty()); - - assertEquals( - "Inbound message" , - clientSpanEvents.get(1).getName()); - assertEquals( - io.opentelemetry.api.common.Attributes.builder() - .put("sequence-number", 0) - .put("message-size", 288) - .build(), - clientSpanEvents.get(1).getAttributes()); - - assertEquals( - "Inbound message" , - clientSpanEvents.get(2).getName()); - assertEquals( - io.opentelemetry.api.common.Attributes.builder() - .put("sequence-number", 1) - .put("message-size", 128) - .build(), - clientSpanEvents.get(2).getAttributes()); - assertEquals(clientSpanData.hasEnded(), true); + public void clientDelayTracingMocking() { + Span mockDelaySpan = mock(Span.class); + when(mockSpanBuilder.setAttribute( + org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.anyString())) + .thenReturn(mockSpanBuilder); + when(mockSpanBuilder.startSpan()).thenReturn(mockAttemptSpan, mockDelaySpan); - // child(attempt) span data - List attemptSpanEvents = attemptSpanData.getEvents(); - assertEquals(clientSpanEvents.size(), 3); - assertEquals( - "Delayed LB pick complete", - attemptSpanEvents.get(0).getName()); - assertTrue(clientSpanEvents.get(0).getAttributes().isEmpty()); - - assertEquals( - "Outbound message" , - attemptSpanEvents.get(1).getName()); - assertEquals( - io.opentelemetry.api.common.Attributes.builder() - .put("sequence-number", 0) - .put("message-size-compressed", 882) - .build(), - attemptSpanEvents.get(1).getAttributes()); - - assertEquals( - "Outbound message" , - attemptSpanEvents.get(2).getName()); - assertEquals( - io.opentelemetry.api.common.Attributes.builder() - .put("sequence-number", 1) - .put("message-size", 27) - .build(), - attemptSpanEvents.get(2).getAttributes()); - - assertEquals( - "Inbound compressed message" , - attemptSpanEvents.get(3).getName()); - assertEquals( - io.opentelemetry.api.common.Attributes.builder() - .put("sequence-number", 0) - .put("message-size-compressed", 255) - .build(), - attemptSpanEvents.get(3).getAttributes()); - - assertEquals(attemptSpanData.hasEnded(), true); - } - - @Test - public void clientAttemptDelayTracing_reasonChangedInvariant() { - OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( - openTelemetryRule.getOpenTelemetry()); - Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(mockOpenTelemetry); CallAttemptsTracerFactory callTracer = - tracingModule.newClientCallTracer(clientSpan, method); + tracingModule.newClientCallTracer(mockClientSpan, method); ClientStreamTracer clientStreamTracer = callTracer.newClientStreamTracer(STREAM_INFO, new Metadata()); - clientStreamTracer.recordAttemptDelayStart("connecting", "reason1"); - clientStreamTracer.recordAttemptDelayReasonChanged("reason2"); - clientStreamTracer.recordAttemptDelayStart("connecting", "reason3"); + clientStreamTracer.recordAttemptDelayStart("connecting", "pick_first: attempting to connect"); clientStreamTracer.recordAttemptDelayEnd(); - clientStreamTracer.streamClosed(Status.OK); - callTracer.callEnded(Status.OK); - clientSpan.end(); - - List spans = openTelemetryRule.getSpans(); - assertEquals(3, spans.size()); - SpanData delaySpanData = spans.get(0); - - assertEquals("Attempt Delay", delaySpanData.getName()); - assertEquals("connecting", delaySpanData.getAttributes().get( - AttributeKey.stringKey("grpc.delay_type"))); - assertEquals(3, delaySpanData.getEvents().size()); - EventData event1 = delaySpanData.getEvents().get(0); - assertEquals("Delay state transition", event1.getName()); - assertEquals("connecting", event1.getAttributes().get( - AttributeKey.stringKey("grpc.delay_type"))); - assertEquals("reason1", event1.getAttributes().get( - AttributeKey.stringKey("grpc.delay_reason"))); - - EventData event2 = delaySpanData.getEvents().get(1); - assertEquals("Delay state transition", event2.getName()); - assertEquals("connecting", event2.getAttributes().get( - AttributeKey.stringKey("grpc.delay_type"))); - assertEquals("reason2", event2.getAttributes().get( - AttributeKey.stringKey("grpc.delay_reason"))); - - EventData event3 = delaySpanData.getEvents().get(2); - assertEquals("Delay state transition", event3.getName()); - assertEquals("connecting", event3.getAttributes().get( - AttributeKey.stringKey("grpc.delay_type"))); - assertEquals("reason3", event3.getAttributes().get( - AttributeKey.stringKey("grpc.delay_reason"))); + verify(mockTracer).spanBuilder(eq("Attempt Delay")); + verify(mockSpanBuilder).setAttribute(eq("grpc.delay_type"), eq("connecting")); + verify(mockDelaySpan).addEvent( + eq("Delay state transition"), + org.mockito.ArgumentMatchers.any()); + verify(mockDelaySpan).end(); } @Test - public void clientAttemptDelayTracing_endToEnd_inProcessTransport() throws Exception { - final CountDownLatch latch = new CountDownLatch(1); - LoadBalancerProvider slowLbProvider = new LoadBalancerProvider() { - @Override - public boolean isAvailable() { - return true; - } - - @Override - public int getPriority() { - return 5; - } + public void clientCallDelayTracingMocking() { + Span mockDelaySpan = mock(Span.class); + when(mockSpanBuilder.setAttribute( + org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.anyString())) + .thenReturn(mockSpanBuilder); + when(mockSpanBuilder.startSpan()).thenReturn(mockDelaySpan); - @Override - public String getPolicyName() { - return "slow_connecting_policy"; - } + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(mockOpenTelemetry); + CallAttemptsTracerFactory callTracer = + tracingModule.newClientCallTracer(mockClientSpan, method); - @Override - public LoadBalancer newLoadBalancer(LoadBalancer.Helper helper) { - return new LoadBalancer() { - @Override - public Status acceptResolvedAddresses(LoadBalancer.ResolvedAddresses resolvedAddresses) { - helper.updateBalancingState(ConnectivityState.CONNECTING, new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - latch.countDown(); - return PickResult.withNoResult("connecting", - "Simulated slow TLS handshake with backend"); - } - }); - return Status.OK; - } + callTracer.recordCallDelayStart("resolving", "waiting for DNS query"); + callTracer.recordCallDelayEnd(); - @Override - public void handleNameResolutionError(Status error) {} + verify(mockTracer).spanBuilder(eq("Call Delay")); + verify(mockSpanBuilder).setAttribute(eq("grpc.delay_type"), eq("resolving")); + verify(mockDelaySpan).addEvent( + eq("Delay state transition"), + org.mockito.ArgumentMatchers.any()); + verify(mockDelaySpan).end(); + } - @Override - public void shutdown() {} - }; - } - }; - LoadBalancerRegistry.getDefaultRegistry().register(slowLbProvider); + @Test + public void clientCallDelayTracing_endToEnd_nameResolutionDelay() throws Exception { + final CountDownLatch resolutionLatch = new CountDownLatch(1); + final AtomicReference capturedListener = new AtomicReference<>(); - NameResolverProvider customResolverProvider = new NameResolverProvider() { + NameResolverProvider slowResolverProvider = new NameResolverProvider() { @Override protected boolean isAvailable() { return true; @@ -512,7 +370,7 @@ protected int priority() { @Override public String getDefaultScheme() { - return "inproce2e"; + return "slowres"; } @Override @@ -525,15 +383,13 @@ public NameResolver newNameResolver(URI targetUri, NameResolver.Args args) { return new NameResolver() { @Override public String getServiceAuthority() { - return "inproce2e"; + return "slowres"; } @Override public void start(Listener2 listener) { - listener.onResult(ResolutionResult.newBuilder() - .setAddressesOrError(StatusOr.fromValue(Collections.singletonList( - new EquivalentAddressGroup(new InProcessSocketAddress("test-e2e"))))) - .build()); + capturedListener.set(listener); + resolutionLatch.countDown(); } @Override @@ -541,7 +397,7 @@ public void shutdown() {} }; } }; - NameResolverRegistry.getDefaultRegistry().register(customResolverProvider); + NameResolverRegistry.getDefaultRegistry().register(slowResolverProvider); GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder() .sdk(openTelemetryRule.getOpenTelemetry()) @@ -549,8 +405,8 @@ public void shutdown() {} .build(); InProcessChannelBuilder channelBuilder = - InProcessChannelBuilder.forTarget("inproce2e:///test-e2e") - .defaultLoadBalancingPolicy("slow_connecting_policy"); + InProcessChannelBuilder.forTarget("slowres:///test-service") + .defaultLoadBalancingPolicy("pick_first"); grpcOpenTelemetry.configureChannelBuilder(channelBuilder); ManagedChannel channel = channelBuilder.build(); try { @@ -558,31 +414,37 @@ public void shutdown() {} call.start(new ClientCall.Listener() {}, new Metadata()); call.request(1); - latch.await(5, TimeUnit.SECONDS); - call.cancel("End test delay segment", null); + resolutionLatch.await(5, TimeUnit.SECONDS); + + // Now complete name resolution + capturedListener.get().onResult(NameResolver.ResolutionResult.newBuilder() + .setAddressesOrError(StatusOr.fromValue(Collections.singletonList( + new EquivalentAddressGroup(new InProcessSocketAddress("test-slow-res"))))) + .build()); + + call.cancel("End test", null); } finally { channel.shutdownNow(); channel.awaitTermination(5, TimeUnit.SECONDS); - LoadBalancerRegistry.getDefaultRegistry().deregister(slowLbProvider); - NameResolverRegistry.getDefaultRegistry().deregister(customResolverProvider); + NameResolverRegistry.getDefaultRegistry().deregister(slowResolverProvider); } List spans = openTelemetryRule.getSpans(); - SpanData delaySpanData = null; + SpanData callDelaySpan = null; for (SpanData s : spans) { - if ("Attempt Delay".equals(s.getName())) { - delaySpanData = s; + if ("Call Delay".equals(s.getName())) { + callDelaySpan = s; break; } } - assertNotNull(delaySpanData); - assertEquals("connecting", - delaySpanData.getAttributes().get(AttributeKey.stringKey("grpc.delay_type"))); + assertNotNull(callDelaySpan); + assertEquals("resolving", + callDelaySpan.getAttributes().get(AttributeKey.stringKey("grpc.delay_type"))); boolean foundTransition = false; - for (EventData event : delaySpanData.getEvents()) { + for (EventData event : callDelaySpan.getEvents()) { if ("Delay state transition".equals(event.getName()) - && "Simulated slow TLS handshake with backend".equals( + && "waiting for name resolution or service config".equals( event.getAttributes().get(AttributeKey.stringKey("grpc.delay_reason")))) { foundTransition = true; break; @@ -592,30 +454,715 @@ public void shutdown() {} } @Test - public void clientAttemptDelayStart_featureFlagDisabled_zeroChildSpans() { - System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "false"); - try { - OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( - openTelemetryRule.getOpenTelemetry()); - Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); - CallAttemptsTracerFactory callTracer = - tracingModule.newClientCallTracer(clientSpan, method); - ClientStreamTracer clientStreamTracer = - callTracer.newClientStreamTracer(STREAM_INFO, new Metadata()); + public void clientCallDelayTracing_duplicateStartAndNullType_handlesGracefully() { + Span parentSpan = openTelemetryRule.getOpenTelemetry() + .getTracer("test").spanBuilder("parent").startSpan(); + OpenTelemetryTracingModule module = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + OpenTelemetryTracingModule.CallAttemptsTracerFactory factory = + module.newClientCallTracer(parentSpan, method); - clientStreamTracer.recordAttemptDelayStart("connecting", "attempting to connect"); - clientStreamTracer.recordAttemptDelayEnd(); - clientStreamTracer.streamClosed(Status.OK); - callTracer.callEnded(Status.OK); - clientSpan.end(); + // Call start with null delayType to cover fallback branch + factory.recordCallDelayStart(null, "initial reason"); + // Duplicate start with same null type triggers recordCallDelayReasonChanged + factory.recordCallDelayStart(null, "updated reason"); + factory.recordCallDelayEnd(); - List spans = openTelemetryRule.getSpans(); - assertEquals(2, spans.size()); - for (SpanData span : spans) { - assertTrue(!span.getName().equals("Attempt Delay")); + List spans = openTelemetryRule.getSpans(); + SpanData callDelaySpan = null; + for (SpanData s : spans) { + if ("Call Delay".equals(s.getName())) { + callDelaySpan = s; + break; } - } finally { - System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true"); + } + assertNotNull(callDelaySpan); + assertEquals("", callDelaySpan.getAttributes().get(AttributeKey.stringKey("grpc.delay_type"))); + } + + @Test + public void attemptDelayTracing_duplicateStartAndReasonChanged_handlesGracefully() { + Span parentSpan = openTelemetryRule.getOpenTelemetry() + .getTracer("test").spanBuilder("parent").startSpan(); + OpenTelemetryTracingModule module = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + OpenTelemetryTracingModule.CallAttemptsTracerFactory factory = + module.newClientCallTracer(parentSpan, method); + + ClientStreamTracer tracer = factory.newClientStreamTracer( + ClientStreamTracer.StreamInfo.newBuilder().setCallOptions(CallOptions.DEFAULT).build(), + new Metadata()); + + // Initial attempt delay start + tracer.recordAttemptDelayStart("connecting", "initial reason"); + // Duplicate start with same delay type triggers recordAttemptDelayReasonChanged + tracer.recordAttemptDelayStart("connecting", "updated reason"); + // Transition to a different delay type while active (exercises Objects.equals == false) + tracer.recordAttemptDelayStart("rls_lookup_pending", "transition to rls"); + // Explicit recordAttemptDelayReasonChanged + tracer.recordAttemptDelayReasonChanged("third reason"); + tracer.recordAttemptDelayEnd(); + + List spans = openTelemetryRule.getSpans(); + SpanData attemptDelaySpan = null; + for (SpanData s : spans) { + if ("Attempt Delay".equals(s.getName())) { + attemptDelaySpan = s; + break; + } + } + assertNotNull(attemptDelaySpan); + assertEquals("connecting", + attemptDelaySpan.getAttributes().get(AttributeKey.stringKey("grpc.delay_type"))); + } + + @Test + public void clientCallDelayTracing_endToEnd_nameResolutionError() throws Exception { + final CountDownLatch resolutionLatch = new CountDownLatch(1); + final AtomicReference capturedListener = new AtomicReference<>(); + final AtomicReference capturedSyncContext = new AtomicReference<>(); + + NameResolverProvider errorResolverProvider = new NameResolverProvider() { + @Override + protected boolean isAvailable() { + return true; + } + + @Override + protected int priority() { + return 5; + } + + @Override + public String getDefaultScheme() { + return "errresspan"; + } + + @Override + public Collection> getProducedSocketAddressTypes() { + return Collections.singleton(InProcessSocketAddress.class); + } + + @Override + public NameResolver newNameResolver(URI targetUri, NameResolver.Args args) { + capturedSyncContext.set(args.getSynchronizationContext()); + return new NameResolver() { + @Override + public String getServiceAuthority() { + return "errresspan"; + } + + @Override + public void start(Listener2 listener) { + capturedListener.set(listener); + resolutionLatch.countDown(); + } + + @Override + public void shutdown() {} + }; + } + }; + NameResolverRegistry.getDefaultRegistry().register(errorResolverProvider); + + GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder() + .sdk(openTelemetryRule.getOpenTelemetry()) + .enableTracing(true) + .build(); + + InProcessChannelBuilder channelBuilder = + InProcessChannelBuilder.forTarget("errresspan:///test-service") + .defaultLoadBalancingPolicy("pick_first"); + grpcOpenTelemetry.configureChannelBuilder(channelBuilder); + ManagedChannel channel = channelBuilder.build(); + try { + ClientCall call = channel.newCall(method, CallOptions.DEFAULT); + call.start(new ClientCall.Listener() {}, new Metadata()); + call.request(1); + + resolutionLatch.await(5, TimeUnit.SECONDS); + + // Complete name resolution with an error inside SynchronizationContext + capturedSyncContext.get().execute(() -> capturedListener.get().onResult2( + NameResolver.ResolutionResult.newBuilder() + .setAddressesOrError(StatusOr.fromStatus( + Status.UNAVAILABLE.withDescription("Name resolution error"))) + .build())); + + call.cancel("End test", null); + } finally { + channel.shutdownNow(); + channel.awaitTermination(5, TimeUnit.SECONDS); + NameResolverRegistry.getDefaultRegistry().deregister(errorResolverProvider); + } + + List spans = openTelemetryRule.getSpans(); + SpanData callDelaySpan = null; + for (SpanData s : spans) { + if ("Call Delay".equals(s.getName())) { + callDelaySpan = s; + break; + } + } + assertNotNull(callDelaySpan); + assertEquals("resolving", + callDelaySpan.getAttributes().get(AttributeKey.stringKey("grpc.delay_type"))); + } + + @Test + public void clientAttemptDelayTracing_endToEnd_inProcessTransport() throws Exception { + final CountDownLatch latch = new CountDownLatch(1); + LoadBalancerProvider slowLbProvider = new LoadBalancerProvider() { + @Override + public boolean isAvailable() { + return true; + } + + @Override + public int getPriority() { + return 5; + } + + @Override + public String getPolicyName() { + return "slow_connecting_policy"; + } + + @Override + public LoadBalancer newLoadBalancer(LoadBalancer.Helper helper) { + return new LoadBalancer() { + @Override + public Status acceptResolvedAddresses(LoadBalancer.ResolvedAddresses resolvedAddresses) { + helper.updateBalancingState(ConnectivityState.CONNECTING, new SubchannelPicker() { + @Override + public PickResult pickSubchannel(PickSubchannelArgs args) { + latch.countDown(); + return PickResult.withNoResult("connecting", + "Simulated slow TLS handshake with backend"); + } + }); + return Status.OK; + } + + @Override + public void handleNameResolutionError(Status error) {} + + @Override + public void shutdown() {} + }; + } + }; + LoadBalancerRegistry.getDefaultRegistry().register(slowLbProvider); + + NameResolverProvider customResolverProvider = new NameResolverProvider() { + @Override + protected boolean isAvailable() { + return true; + } + + @Override + protected int priority() { + return 5; + } + + @Override + public String getDefaultScheme() { + return "inproce2e"; + } + + @Override + public Collection> getProducedSocketAddressTypes() { + return Collections.singleton(InProcessSocketAddress.class); + } + + @Override + public NameResolver newNameResolver(URI targetUri, NameResolver.Args args) { + return new NameResolver() { + @Override + public String getServiceAuthority() { + return "inproce2e"; + } + + @Override + public void start(Listener2 listener) { + listener.onResult(NameResolver.ResolutionResult.newBuilder() + .setAddressesOrError(StatusOr.fromValue(Collections.singletonList( + new EquivalentAddressGroup(new InProcessSocketAddress("test-e2e"))))) + .build()); + } + + @Override + public void shutdown() {} + }; + } + }; + NameResolverRegistry.getDefaultRegistry().register(customResolverProvider); + + GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder() + .sdk(openTelemetryRule.getOpenTelemetry()) + .enableTracing(true) + .build(); + + InProcessChannelBuilder channelBuilder = + InProcessChannelBuilder.forTarget("inproce2e:///test-e2e") + .defaultLoadBalancingPolicy("slow_connecting_policy"); + grpcOpenTelemetry.configureChannelBuilder(channelBuilder); + ManagedChannel channel = channelBuilder.build(); + try { + ClientCall call = channel.newCall(method, CallOptions.DEFAULT); + call.start(new ClientCall.Listener() {}, new Metadata()); + call.request(1); + + latch.await(5, TimeUnit.SECONDS); + call.cancel("End test delay segment", null); + } finally { + channel.shutdownNow(); + channel.awaitTermination(5, TimeUnit.SECONDS); + LoadBalancerRegistry.getDefaultRegistry().deregister(slowLbProvider); + NameResolverRegistry.getDefaultRegistry().deregister(customResolverProvider); + } + + List spans = openTelemetryRule.getSpans(); + SpanData delaySpanData = null; + for (SpanData s : spans) { + if ("Attempt Delay".equals(s.getName())) { + delaySpanData = s; + break; + } + } + assertNotNull(delaySpanData); + assertEquals("connecting", + delaySpanData.getAttributes().get(AttributeKey.stringKey("grpc.delay_type"))); + + boolean foundTransition = false; + for (EventData event : delaySpanData.getEvents()) { + if ("Delay state transition".equals(event.getName()) + && "Simulated slow TLS handshake with backend".equals( + event.getAttributes().get(AttributeKey.stringKey("grpc.delay_reason")))) { + foundTransition = true; + break; + } + } + assertTrue(foundTransition); + } + + @Test + public void clientBasicTracingRule() { + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); + CallAttemptsTracerFactory callTracer = + tracingModule.newClientCallTracer(clientSpan, method); + Metadata headers = new Metadata(); + ClientStreamTracer clientStreamTracer = callTracer.newClientStreamTracer(STREAM_INFO, headers); + clientStreamTracer.createPendingStream(); + clientStreamTracer.streamCreated(Attributes.EMPTY, headers); + clientStreamTracer.outboundMessage(0); + clientStreamTracer.outboundMessageSent(0, 882, -1); + clientStreamTracer.inboundMessage(0); + clientStreamTracer.outboundMessage(1); + clientStreamTracer.outboundMessageSent(1, -1, 27); + clientStreamTracer.inboundMessageRead(0, 255, -1); + clientStreamTracer.inboundUncompressedSize(288); + clientStreamTracer.inboundMessageRead(1, 128, 128); + clientStreamTracer.inboundMessage(1); + clientStreamTracer.inboundUncompressedSize(128); + + clientStreamTracer.streamClosed(Status.OK); + callTracer.callEnded(Status.OK); + + List spans = openTelemetryRule.getSpans(); + assertEquals(spans.size(), 2); + SpanData attemptSpanData = spans.get(0); + SpanData clientSpanData = spans.get(1); + assertEquals(attemptSpanData.getName(), "Attempt.package1.service2.method3"); + assertEquals(clientSpanData.getName(), "test-client-span"); + assertEquals(headers.keys(), ImmutableSet.of("traceparent")); + String spanContext = headers.get( + Metadata.Key.of("traceparent", Metadata.ASCII_STRING_MARSHALLER)); + assertEquals(spanContext.substring(3, 3 + TraceId.getLength()), + spans.get(1).getSpanContext().getTraceId()); + + // parent(client) span data + List clientSpanEvents = clientSpanData.getEvents(); + assertEquals(clientSpanEvents.size(), 3); + assertEquals( + "Delayed name resolution complete", + clientSpanEvents.get(0).getName()); + assertTrue(clientSpanEvents.get(0).getAttributes().isEmpty()); + + assertEquals( + "Inbound message" , + clientSpanEvents.get(1).getName()); + assertEquals( + io.opentelemetry.api.common.Attributes.builder() + .put("sequence-number", 0) + .put("message-size", 288) + .build(), + clientSpanEvents.get(1).getAttributes()); + + assertEquals( + "Inbound message" , + clientSpanEvents.get(2).getName()); + assertEquals( + io.opentelemetry.api.common.Attributes.builder() + .put("sequence-number", 1) + .put("message-size", 128) + .build(), + clientSpanEvents.get(2).getAttributes()); + assertEquals(clientSpanData.hasEnded(), true); + + // child(attempt) span data + List attemptSpanEvents = attemptSpanData.getEvents(); + assertEquals(clientSpanEvents.size(), 3); + assertEquals( + "Delayed LB pick complete", + attemptSpanEvents.get(0).getName()); + assertTrue(clientSpanEvents.get(0).getAttributes().isEmpty()); + + assertEquals( + "Outbound message" , + attemptSpanEvents.get(1).getName()); + assertEquals( + io.opentelemetry.api.common.Attributes.builder() + .put("sequence-number", 0) + .put("message-size-compressed", 882) + .build(), + attemptSpanEvents.get(1).getAttributes()); + + assertEquals( + "Outbound message" , + attemptSpanEvents.get(2).getName()); + assertEquals( + io.opentelemetry.api.common.Attributes.builder() + .put("sequence-number", 1) + .put("message-size", 27) + .build(), + attemptSpanEvents.get(2).getAttributes()); + + assertEquals( + "Inbound compressed message" , + attemptSpanEvents.get(3).getName()); + assertEquals( + io.opentelemetry.api.common.Attributes.builder() + .put("sequence-number", 0) + .put("message-size-compressed", 255) + .build(), + attemptSpanEvents.get(3).getAttributes()); + + assertEquals(attemptSpanData.hasEnded(), true); + } + + @Test + public void clientAttemptDelayTracing_reasonChangedInvariant() { + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); + CallAttemptsTracerFactory callTracer = + tracingModule.newClientCallTracer(clientSpan, method); + ClientStreamTracer clientStreamTracer = + callTracer.newClientStreamTracer(STREAM_INFO, new Metadata()); + + clientStreamTracer.recordAttemptDelayStart("connecting", "reason1"); + clientStreamTracer.recordAttemptDelayReasonChanged("reason2"); + clientStreamTracer.recordAttemptDelayStart("connecting", "reason3"); + clientStreamTracer.recordAttemptDelayEnd(); + clientStreamTracer.streamClosed(Status.OK); + callTracer.callEnded(Status.OK); + clientSpan.end(); + + List spans = openTelemetryRule.getSpans(); + assertEquals(3, spans.size()); + SpanData delaySpanData = spans.get(0); + + assertEquals("Attempt Delay", delaySpanData.getName()); + assertEquals("connecting", delaySpanData.getAttributes().get( + AttributeKey.stringKey("grpc.delay_type"))); + assertEquals(3, delaySpanData.getEvents().size()); + + EventData event1 = delaySpanData.getEvents().get(0); + assertEquals("Delay state transition", event1.getName()); + assertEquals("connecting", event1.getAttributes().get( + AttributeKey.stringKey("grpc.delay_type"))); + assertEquals("reason1", event1.getAttributes().get( + AttributeKey.stringKey("grpc.delay_reason"))); + + EventData event2 = delaySpanData.getEvents().get(1); + assertEquals("Delay state transition", event2.getName()); + assertEquals("connecting", event2.getAttributes().get( + AttributeKey.stringKey("grpc.delay_type"))); + assertEquals("reason2", event2.getAttributes().get( + AttributeKey.stringKey("grpc.delay_reason"))); + + EventData event3 = delaySpanData.getEvents().get(2); + assertEquals("Delay state transition", event3.getName()); + assertEquals("connecting", event3.getAttributes().get( + AttributeKey.stringKey("grpc.delay_type"))); + assertEquals("reason3", event3.getAttributes().get( + AttributeKey.stringKey("grpc.delay_reason"))); + } + + @Test + public void clientAttemptDelayStart_featureFlagDisabled_zeroChildSpans() { + System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "false"); + try { + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); + CallAttemptsTracerFactory callTracer = + tracingModule.newClientCallTracer(clientSpan, method); + ClientStreamTracer clientStreamTracer = + callTracer.newClientStreamTracer(STREAM_INFO, new Metadata()); + + clientStreamTracer.recordAttemptDelayStart("connecting", "attempting to connect"); + clientStreamTracer.recordAttemptDelayEnd(); + clientStreamTracer.streamClosed(Status.OK); + callTracer.callEnded(Status.OK); + clientSpan.end(); + + List spans = openTelemetryRule.getSpans(); + assertEquals(2, spans.size()); + for (SpanData span : spans) { + assertTrue(!span.getName().equals("Attempt Delay")); + } + } finally { + System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true"); + } + } + + @Test + public void clientCallDelayTracing_reasonChangedInvariant() { + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); + CallAttemptsTracerFactory callTracer = + tracingModule.newClientCallTracer(clientSpan, method); + + callTracer.recordCallDelayStart("resolving", "reason1"); + callTracer.recordCallDelayReasonChanged("reason2"); + callTracer.recordCallDelayStart("resolving", "reason3"); + callTracer.recordCallDelayEnd(); + callTracer.callEnded(Status.OK); + clientSpan.end(); + + List spans = openTelemetryRule.getSpans(); + assertEquals(2, spans.size()); + SpanData callDelaySpan = spans.stream() + .filter(s -> "Call Delay".equals(s.getName())) + .findFirst() + .orElseThrow(() -> new AssertionError("Expected 'Call Delay' span not found")); + + assertEquals("resolving", callDelaySpan.getAttributes().get( + AttributeKey.stringKey("grpc.delay_type"))); + assertEquals(3, callDelaySpan.getEvents().size()); + + EventData event1 = callDelaySpan.getEvents().get(0); + assertEquals("Delay state transition", event1.getName()); + assertEquals("resolving", + event1.getAttributes().get(AttributeKey.stringKey("grpc.delay_type"))); + assertEquals("reason1", + event1.getAttributes().get(AttributeKey.stringKey("grpc.delay_reason"))); + + EventData event2 = callDelaySpan.getEvents().get(1); + assertEquals("Delay state transition", event2.getName()); + assertEquals("resolving", + event2.getAttributes().get(AttributeKey.stringKey("grpc.delay_type"))); + assertEquals("reason2", + event2.getAttributes().get(AttributeKey.stringKey("grpc.delay_reason"))); + + EventData event3 = callDelaySpan.getEvents().get(2); + assertEquals("Delay state transition", event3.getName()); + assertEquals("resolving", + event3.getAttributes().get(AttributeKey.stringKey("grpc.delay_type"))); + assertEquals("reason3", + event3.getAttributes().get(AttributeKey.stringKey("grpc.delay_reason"))); + } + + @Test + public void clientCallDelayStart_afterCallEnded_noSpansRecorded() { + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); + CallAttemptsTracerFactory callTracer = + tracingModule.newClientCallTracer(clientSpan, method); + + callTracer.callEnded(Status.OK); + callTracer.recordCallDelayStart("resolving", "reasonAfterCallEnded"); + callTracer.recordCallDelayReasonChanged("reason2"); + callTracer.recordCallDelayEnd(); + clientSpan.end(); + + List spans = openTelemetryRule.getSpans(); + for (SpanData span : spans) { + assertTrue(!span.getName().equals("Call Delay")); + } + } + + @Test + public void clientAttemptDelayStart_afterStreamClosed_noSpansRecorded() { + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); + CallAttemptsTracerFactory callTracer = + tracingModule.newClientCallTracer(clientSpan, method); + ClientStreamTracer clientStreamTracer = + callTracer.newClientStreamTracer(STREAM_INFO, new Metadata()); + + clientStreamTracer.streamClosed(Status.OK); + clientStreamTracer.recordAttemptDelayStart("connecting", "reasonAfterStreamClosed"); + clientStreamTracer.recordAttemptDelayReasonChanged("reason2"); + clientStreamTracer.recordAttemptDelayEnd(); + callTracer.callEnded(Status.OK); + clientSpan.end(); + + List spans = openTelemetryRule.getSpans(); + for (SpanData span : spans) { + assertTrue(!span.getName().equals("Attempt Delay")); + } + } + + @Test + public void clientCallDelayStart_delayTypeTransition_closesPreviousSpan() { + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); + CallAttemptsTracerFactory callTracer = + tracingModule.newClientCallTracer(clientSpan, method); + + callTracer.recordCallDelayStart("resolving", "dns lookup"); + callTracer.recordCallDelayStart("connecting", "pick first connect"); + callTracer.recordCallDelayEnd(); + callTracer.callEnded(Status.OK); + clientSpan.end(); + + List spans = openTelemetryRule.getSpans(); + long callDelaySpanCount = spans.stream().filter(s -> "Call Delay".equals(s.getName())).count(); + assertEquals(2L, callDelaySpanCount); + } + + @Test + public void clientCallEnded_calledTwice_secondCallNoOp() { + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); + CallAttemptsTracerFactory callTracer = + tracingModule.newClientCallTracer(clientSpan, method); + + callTracer.callEnded(Status.OK); + callTracer.callEnded(Status.CANCELLED); + clientSpan.end(); + + List spans = openTelemetryRule.getSpans(); + assertNotNull(spans); + } + + @Test + public void clientStreamClosed_calledTwice_secondCallNoOp() { + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); + CallAttemptsTracerFactory callTracer = + tracingModule.newClientCallTracer(clientSpan, method); + ClientStreamTracer clientStreamTracer = + callTracer.newClientStreamTracer(STREAM_INFO, new Metadata()); + + clientStreamTracer.streamClosed(Status.OK); + clientStreamTracer.streamClosed(Status.CANCELLED); + callTracer.callEnded(Status.OK); + clientSpan.end(); + + List spans = openTelemetryRule.getSpans(); + assertNotNull(spans); + } + + @Test + public void serverStreamClosed_calledTwice_secondCallNoOp() { + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + ServerStreamTracer.Factory serverTracerFactory = + tracingModule.getServerTracerFactory(); + ServerStreamTracer serverTracer = + serverTracerFactory.newServerStreamTracer(method.getFullMethodName(), new Metadata()); + + serverTracer.streamClosed(Status.OK); + serverTracer.streamClosed(Status.CANCELLED); + + List spans = openTelemetryRule.getSpans(); + assertNotNull(spans); + } + + @Test + public void clientCallDelayReasonChanged_noActiveSpan_noOp() { + System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true"); + try { + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); + CallAttemptsTracerFactory callTracer = + tracingModule.newClientCallTracer(clientSpan, method); + + callTracer.recordCallDelayReasonChanged("reasonWithoutSpan"); + callTracer.callEnded(Status.OK); + clientSpan.end(); + + List spans = openTelemetryRule.getSpans(); + assertNotNull(spans); + } finally { + System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY"); + } + } + + @Test + public void clientAttemptDelayReasonChanged_noActiveSpan_noOp() { + System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true"); + try { + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); + CallAttemptsTracerFactory callTracer = + tracingModule.newClientCallTracer(clientSpan, method); + ClientStreamTracer clientStreamTracer = + callTracer.newClientStreamTracer(STREAM_INFO, new Metadata()); + + clientStreamTracer.recordAttemptDelayReasonChanged("reasonWithoutSpan"); + clientStreamTracer.streamClosed(Status.OK); + callTracer.callEnded(Status.OK); + clientSpan.end(); + + List spans = openTelemetryRule.getSpans(); + assertNotNull(spans); + } finally { + System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY"); + } + } + + @Test + public void delayTracing_featureFlagDisabled_allMethodsNoOp() { + System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY"); + OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule( + openTelemetryRule.getOpenTelemetry()); + Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan(); + CallAttemptsTracerFactory callTracer = + tracingModule.newClientCallTracer(clientSpan, method); + ClientStreamTracer clientStreamTracer = + callTracer.newClientStreamTracer(STREAM_INFO, new Metadata()); + + callTracer.recordCallDelayStart("resolving", "reason1"); + callTracer.recordCallDelayReasonChanged("reason2"); + callTracer.recordCallDelayEnd(); + + clientStreamTracer.recordAttemptDelayStart("connecting", "reason1"); + clientStreamTracer.recordAttemptDelayReasonChanged("reason2"); + clientStreamTracer.recordAttemptDelayEnd(); + + clientStreamTracer.streamClosed(Status.OK); + callTracer.callEnded(Status.OK); + clientSpan.end(); + + List spans = openTelemetryRule.getSpans(); + for (SpanData span : spans) { + assertTrue(!span.getName().equals("Call Delay")); + assertTrue(!span.getName().equals("Attempt Delay")); } } diff --git a/util/src/test/java/io/grpc/util/ForwardingClientStreamTracerTest.java b/util/src/test/java/io/grpc/util/ForwardingClientStreamTracerTest.java index dbd7e99b29a..f6397d878f9 100644 --- a/util/src/test/java/io/grpc/util/ForwardingClientStreamTracerTest.java +++ b/util/src/test/java/io/grpc/util/ForwardingClientStreamTracerTest.java @@ -40,6 +40,19 @@ public void allMethodsForwarded() throws Exception { Collections.emptyList()); } + @Test + public void attemptDelayMethodsForwarded() { + TestClientStreamTracer tracer = new TestClientStreamTracer(); + tracer.recordAttemptDelayStart("connecting", "test"); + org.mockito.Mockito.verify(mockDelegate).recordAttemptDelayStart("connecting", "test"); + + tracer.recordAttemptDelayReasonChanged("test2"); + org.mockito.Mockito.verify(mockDelegate).recordAttemptDelayReasonChanged("test2"); + + tracer.recordAttemptDelayEnd(); + org.mockito.Mockito.verify(mockDelegate).recordAttemptDelayEnd(); + } + @SuppressWarnings("deprecation") private final class TestClientStreamTracer extends ForwardingClientStreamTracer { @Override