Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
b50ca84
core,api,xds: Implement load balancing policy delay plumbing
AgraVator May 13, 2026
c38ce1d
fix: tests
AgraVator May 14, 2026
a992bdf
fix: minor changes
AgraVator May 19, 2026
6a55ff2
add missing endDelay()
AgraVator Jun 8, 2026
389b96f
core,api,rls,util,xds: Implement dual Load Balancer delay APIs and ca…
AgraVator Jun 19, 2026
5e56f38
core: Add 100% test coverage for dual LB delay APIs and cadence rules
AgraVator Jun 19, 2026
6a3572b
opentelemetry: Implement dual Load Balancer delay spans and metrics
AgraVator Jun 22, 2026
9a4f21f
Merge remote-tracking branch 'upstream/master' into lb-policy-delay
AgraVator Jun 22, 2026
d25d064
Implement Name Resolution and unified RPC Delay Observability specifi…
AgraVator Jun 23, 2026
ffb485e
Ensure thread-safety and unit test coverage for Call-Level Delay APIs…
AgraVator Jul 6, 2026
42352f2
Add comprehensive End-to-End tests for Call-Level Name Resolution Del…
AgraVator Jul 6, 2026
a76996c
Ensure Call-Level delay recording only triggers when RPCs are queued …
AgraVator Jul 6, 2026
7c94743
Fix CdsLoadBalancer2Test atLeastOnce static import and assertions on …
AgraVator Jul 6, 2026
a7830ff
Fix PriorityLoadBalancerTest handleNameResolutionError assertion on n…
AgraVator Jul 6, 2026
b482c43
Fix checkstyle import ordering in OpenTelemetryTracingModuleTest
AgraVator Jul 6, 2026
fbbc1b9
Merge remote-tracking branch 'upstream/master' into name-resolution-d…
AgraVator Jul 27, 2026
df48bee
core, opentelemetry: Harden Name Resolution & LB delay state machines…
AgraVator Jul 27, 2026
fc9721d
api: update @since 1.82.0 to 1.84.0
AgraVator Jul 27, 2026
162caef
test: remove temporary stress test files prior to PR submission
AgraVator Jul 27, 2026
f3bdcd9
core: align PendingStream synchronized (this) blocks with ManagedChan…
AgraVator Jul 27, 2026
9cc310b
opentelemetry: add targeted unit tests to expand branch coverage for …
AgraVator Jul 27, 2026
0a7eb26
core: harden PendingStream synchronization and add multithreaded race…
AgraVator Jul 28, 2026
0bc1634
opentelemetry: add end-to-end Client/Server simulation tests for dela…
AgraVator Jul 29, 2026
20fc8d0
Fix PR #12893 CI failures: revert PickResult.withError default delay,…
AgraVator Jul 29, 2026
a7398d7
cleanup: remove extra stress and unit tests, keeping only essential C…
AgraVator Jul 29, 2026
a9ea9e7
test: restore unit tests from master and add test coverage for delay …
AgraVator Jul 29, 2026
35547a8
test: restore pickResult_withSubchannelReplacement and pickResult_wit…
AgraVator Jul 29, 2026
6571c2d
test: remove multi-threaded stress tests from DelayedClientTransportT…
AgraVator Jul 29, 2026
b7227a7
test: remove unused imports from ManagedChannelImplTest and DelayedCl…
AgraVator Jul 29, 2026
129b8e5
test(opentelemetry): add unit tests for delay metrics and tracing bra…
AgraVator Aug 1, 2026
ff1ed50
test(opentelemetry): remove A121DelayObservabilityWrapperTest
AgraVator Aug 1, 2026
d48258e
test(opentelemetry): add tests for callEnded/streamClosed guards and …
AgraVator Aug 1, 2026
24cd5c3
test(opentelemetry): add tests for delay reason-changed guards withou…
AgraVator Aug 1, 2026
e63b9ee
test: expand branch coverage for OobChannel, delay metrics, and traci…
AgraVator Aug 1, 2026
42e8686
refactor: simplify delay synchronization and telemetry style across P…
AgraVator Aug 3, 2026
f7216e3
core: restore state transition comments in DelayedClientTransport.upd…
AgraVator Aug 5, 2026
ffc15db
test, opentelemetry: address PR review feedback and add gRFC A121 nam…
AgraVator Aug 12, 2026
2f71b12
test, opentelemetry: add unit tests for missing code coverage branche…
AgraVator Aug 12, 2026
2416612
test, opentelemetry: fix unit test setups for checkstyle and null checks
AgraVator Aug 12, 2026
6f6edf2
test: add unit tests covering missing branch combinations in DelayedC…
AgraVator Aug 12, 2026
9c5406c
test: expand unit test coverage for targetAttributeFilter, clientAtte…
AgraVator Aug 12, 2026
e23ccfa
test: add coverage for initialType null, Baggage context, and stream …
AgraVator Aug 12, 2026
6d5feab
test: add unstarted delay end tests for OpenTelemetryMetricsModule
AgraVator Aug 12, 2026
3c67ba3
test: add emptyTracersAndNullInitialReason test for DelayedClientTran…
AgraVator Aug 12, 2026
0309b66
style: remove unnecessary fully qualified class names across PR files
AgraVator Aug 12, 2026
acf4c1e
test: resolve all review discussions on delay observability branch co…
AgraVator Aug 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 41 additions & 3 deletions api/src/main/java/io/grpc/ClientStreamTracer.java
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public void createPendingStream() {
*
* @param delayType canonical low-cardinality label categorizing the delay (e.g., "connecting")
* @param delayReason high-cardinality diagnostic string describing granular runtime conditions
* @since 1.82.0
* @since 1.84.0
*/
public void recordAttemptDelayStart(String delayType, String delayReason) {
}
Expand All @@ -80,7 +80,7 @@ public void recordAttemptDelayStart(String delayType, String delayReason) {
* on the active delay span without recreating the span or resetting cumulative timers.
*
* @param delayReason updated high-cardinality diagnostic string describing new conditions
* @since 1.82.0
* @since 1.84.0
*/
public void recordAttemptDelayReasonChanged(String delayReason) {
}
Expand All @@ -91,7 +91,7 @@ public void recordAttemptDelayReasonChanged(String delayReason) {
* <p>Implementations should simultaneously close active child tracing spans and record elapsed
* duration to the {@code grpc.client.attempt.delay.duration} histogram.
*
* @since 1.82.0
* @since 1.84.0
*/
public void recordAttemptDelayEnd() {
}
Expand Down Expand Up @@ -156,6 +156,44 @@ public abstract static class Factory {
public ClientStreamTracer newClientStreamTracer(StreamInfo info, Metadata headers) {
throw new UnsupportedOperationException("Not implemented");
}

/**
* Called when a call-level delay segment (such as waiting for name resolution or service
* configuration parsing) starts before any individual RPC attempt is created.
*
* <p>Implementations should start logical timers and create child tracing spans (named strictly
* {@code "Call Delay"}) carrying the canonical {@code grpc.delay_type} attribute.
*
* @param delayType canonical low-cardinality label categorizing the delay (e.g., "resolving")
* @param delayReason high-cardinality diagnostic string describing granular runtime conditions
* @since 1.84.0
*/
public void recordCallDelayStart(String delayType, String delayReason) {
}

/**
* Called when a call-level delay reason changes while the active delay segment continues.
*
* <p>Implementations should emit structured events (such as {@code "Delay state transition"})
* on the active call delay span without recreating the span or resetting timers.
*
* @param delayReason updated high-cardinality diagnostic string describing new conditions
* @since 1.84.0
*/
public void recordCallDelayReasonChanged(String delayReason) {
}

/**
* Called when a call-level delay segment ends upon successful name resolution or when an RPC
* is cancelled before resolution completes.
*
* <p>Implementations should close active call delay spans and record elapsed duration to the
* {@code grpc.client.call.delay.duration} histogram.
*
* @since 1.84.0
*/
public void recordCallDelayEnd() {
}
}

/**
Expand Down
2 changes: 1 addition & 1 deletion api/src/main/java/io/grpc/LoadBalancer.java
Original file line number Diff line number Diff line change
Expand Up @@ -739,7 +739,7 @@ public static PickResult withNoResult() {
*
* @param delayType low-cardinality root cause label (e.g., "connecting")
* @param delayReason high-cardinality diagnostic string for trace events
* @since 1.82.0
* @since 1.84.0
*/
public static PickResult withNoResult(String delayType, String delayReason) {
Preconditions.checkNotNull(delayType, "delayType");
Expand Down
13 changes: 13 additions & 0 deletions api/src/test/java/io/grpc/ClientStreamTracerTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -57,4 +57,17 @@ public void streamInfo_toBuilder() {
StreamInfo info2 = info1.toBuilder().build();
assertThat(info2.getCallOptions()).isSameInstanceAs(callOptions);
}

@Test
public void defaultDelayMethodsNoOp() {
ClientStreamTracer tracer = new ClientStreamTracer() {};
tracer.recordAttemptDelayStart("connecting", "test");
tracer.recordAttemptDelayReasonChanged("test2");
tracer.recordAttemptDelayEnd();

ClientStreamTracer.Factory factory = new ClientStreamTracer.Factory() {};
factory.recordCallDelayStart("resolving", "test");
factory.recordCallDelayReasonChanged("test2");
factory.recordCallDelayEnd();
}
}
20 changes: 14 additions & 6 deletions core/src/main/java/io/grpc/internal/DelayedClientTransport.java
Original file line number Diff line number Diff line change
Expand Up @@ -396,7 +396,7 @@ private static String determineQueuingDelayReason(@Nullable PickResult pickResul
return "client channel: waiting for picker";
}

private class PendingStream extends DelayedStream {
class PendingStream extends DelayedStream {
private final PickSubchannelArgs args;
private final Context context = Context.current();
private final ClientStreamTracer[] tracers;
Expand All @@ -405,6 +405,8 @@ private class PendingStream extends DelayedStream {
@Nullable private String activeDelayType;
@GuardedBy("this")
@Nullable private String activeDelayReason;
@GuardedBy("this")
private boolean delayEnded;

private PendingStream(PickSubchannelArgs args, ClientStreamTracer[] tracers,
@Nullable String initialType, @Nullable String initialReason) {
Expand All @@ -428,11 +430,11 @@ private PendingStream(PickSubchannelArgs args, ClientStreamTracer[] tracers,
* structured transition event is appended to the active span without span re-creation.
*/
synchronized void updateDelay(@Nullable String newType, @Nullable String newReason) {
if (getRealStream() != null) {
if (getRealStream() != null || delayEnded) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Branch getRealStream() == null && delayEnded == true is missing coverage. To cover it, a unit test only needs to invoke endDelay() directly on a pending stream while getRealStream() == null and then call updateDelay(...):

// Example in DelayedClientTransportTest.java
PendingStream pendingStream = ...;
pendingStream.endDelay(); // sets delayEnded = true while getRealStream() is still null
pendingStream.updateDelay("connecting", "new reason"); // triggers missing branch > returns

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lets not do this, I didn't realize this change requires changing the class access modifier for PendingStream.

return;
}
// Delay type changed (e.g., from RLS lookup to connecting). End the previous delay.
if (!Objects.equals(activeDelayType, newType)) {
// Delay type changed (e.g., from RLS lookup to connecting). End the previous delay.
if (activeDelayType != null) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The is reported missing coverage for the branch when activeDelayType is null but it cannot be meaningfully tested because determineQueuingDelayType always returns non null value and there are other complications if we make this unconditional.

for (ClientStreamTracer tracer : tracers) {
tracer.recordAttemptDelayEnd();
Expand All @@ -446,8 +448,9 @@ synchronized void updateDelay(@Nullable String newType, @Nullable String newReas
}
}
}
if (newType != null && newReason != null && !Objects.equals(activeDelayReason, newReason)) {
// Delay type is unchanged, but the reason changed (e.g., priority failover).
// Delay type is unchanged, but the reason changed (e.g., priority failover).
if (newType != null && newReason != null
&& !Objects.equals(activeDelayReason, newReason)) {
activeDelayReason = newReason;
for (ClientStreamTracer tracer : tracers) {
tracer.recordAttemptDelayReasonChanged(newReason);
Expand All @@ -459,6 +462,10 @@ synchronized void updateDelay(@Nullable String newType, @Nullable String newReas
* Ends active attempt delay segment telemetry upon stream creation or stream cancellation.
*/
synchronized void endDelay() {
if (delayEnded) {
return;
}
delayEnded = true;
if (activeDelayType != null) {
for (ClientStreamTracer tracer : tracers) {
tracer.recordAttemptDelayEnd();
Expand All @@ -469,8 +476,9 @@ synchronized void endDelay() {
}

Runnable setStreamAndEndDelay(ClientStream stream) {
Runnable runnable = setStream(stream);
endDelay();
return setStream(stream);
return runnable;
}

/** Runnable may be null. */
Expand Down
42 changes: 39 additions & 3 deletions core/src/main/java/io/grpc/internal/ManagedChannelImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -918,6 +918,7 @@ public void run() {
inUseStateAggregator.updateObjectInUse(pendingCallsInUseObject, true);
}
pendingCalls.add(pendingCall);
pendingCall.notifyQueuedForNameResolution();
} else {
pendingCall.reprocess();
}
Expand Down Expand Up @@ -997,6 +998,9 @@ private final class PendingCall<ReqT, RespT> extends DelayedClientCall<ReqT, Res
final MethodDescriptor<ReqT, RespT> method;
final CallOptions callOptions;
private final long callCreationTime;
private volatile boolean queuedForResolution;
private volatile boolean callCancelled;
private final AtomicBoolean delayEnded = new AtomicBoolean();

PendingCall(Context context, MethodDescriptor<ReqT, RespT> method, CallOptions callOptions) {
super(
Expand All @@ -1010,14 +1014,42 @@ private final class PendingCall<ReqT, RespT> extends DelayedClientCall<ReqT, Res
this.callCreationTime = ticker.nanoTime();
}

void notifyQueuedForNameResolution() {
boolean shouldStart = false;
synchronized (this) {
if (!callCancelled && !queuedForResolution) {
queuedForResolution = true;
shouldStart = true;
}
}
if (shouldStart) {
for (ClientStreamTracer.Factory factory : callOptions.getStreamTracerFactories()) {
factory.recordCallDelayStart(
"resolving", "waiting for name resolution or service config");
}
}
}

private void endDelayIfNeeded() {
if (queuedForResolution && delayEnded.compareAndSet(false, true)) {
for (ClientStreamTracer.Factory factory : callOptions.getStreamTracerFactories()) {
factory.recordCallDelayEnd();
}
}
}

/** Called when it's ready to create a real call and reprocess the pending call. */
void reprocess() {
endDelayIfNeeded();
ClientCall<ReqT, RespT> realCall;
Context previous = context.attach();
try {
CallOptions delayResolutionOption = callOptions.withOption(NAME_RESOLUTION_DELAYED,
ticker.nanoTime() - callCreationTime);
realCall = newClientCall(method, delayResolutionOption);
CallOptions effectiveOptions = callOptions;
if (queuedForResolution) {
effectiveOptions = callOptions.withOption(NAME_RESOLUTION_DELAYED,
ticker.nanoTime() - callCreationTime);
}
realCall = newClientCall(method, effectiveOptions);
} finally {
context.detach(previous);
}
Expand All @@ -1037,6 +1069,10 @@ public void run() {

@Override
protected void callCancelled() {
synchronized (this) {
callCancelled = true;
}
endDelayIfNeeded();
super.callCancelled();
syncContext.execute(new PendingCallRemoval());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -972,6 +972,86 @@ public PickResult pickSubchannel(PickSubchannelArgs args) {
assertEquals(1, fakeTracer.delayEndedCount);
}

@Test
public void streamDelayMetrics_allBranchCombinations() {
FakeStreamTracer fakeTracer = new FakeStreamTracer();
ClientStreamTracer[] customTracers = new ClientStreamTracer[] { fakeTracer };

// 1. Initial stream with no picker set (initialType defaults to "connecting")
ClientStream stream = delayedTransport.newStream(method, headers, callOptions, customTracers);
stream.start(streamListener);

assertEquals(Collections.singletonList("connecting"), fakeTracer.startedDelayTypes);
assertEquals(Collections.singletonList("client channel: waiting for picker"),
fakeTracer.startedDelayReasons);

// 2. Reprocess with same delay type ("connecting") but new reason ("attempting to connect")
delayedTransport.reprocess(fakePicker(
PickResult.withNoResult("connecting", "attempting to connect")));
assertEquals(Collections.singletonList("attempting to connect"),
fakeTracer.changedDelayReasons);

// 3. Reprocess with a different delay type ("rls_lookup_pending")
delayedTransport.reprocess(fakePicker(
PickResult.withNoResult("rls_lookup_pending", "RLS pending")));
assertEquals(1, fakeTracer.delayEndedCount);
assertEquals(Arrays.asList("connecting", "rls_lookup_pending"), fakeTracer.startedDelayTypes);

// 4. Reprocess with default PickResult.withNoResult() (defaults to "connecting")
delayedTransport.reprocess(fakePicker(PickResult.withNoResult()));
assertEquals(2, fakeTracer.delayEndedCount);

// 5. Cancel stream ends active delay
stream.cancel(Status.CANCELLED);
assertEquals(3, fakeTracer.delayEndedCount);

// 6. Direct updateDelay with null type and null reason
fakeTracer = new FakeStreamTracer();
customTracers = new ClientStreamTracer[] { fakeTracer };
ClientStream stream2 = delayedTransport.newStream(method, headers, callOptions, customTracers);
fakeTracer.startedDelayTypes.clear();
stream2.start(streamListener);

// Explicitly update delay to null
delayedTransport.reprocess(fakePicker(PickResult.withNoResult()));

// 7. Cancel stream2 (endDelay when activeDelayType is null)
stream2.cancel(Status.CANCELLED);
}

@Test
public void streamDelayMetrics_emptyTracersAndNullInitialReason() {
ClientStreamTracer[] emptyTracers = new ClientStreamTracer[0];
delayedTransport.reprocess(fakePicker(PickResult.withNoResult("resolving", "")));
ClientStream stream = delayedTransport.newStream(method, headers, callOptions, emptyTracers);
stream.start(streamListener);

// Update delay with empty tracers and empty reason
delayedTransport.reprocess(fakePicker(PickResult.withNoResult("connecting", "")));
stream.cancel(Status.CANCELLED);
}

@Test
public void streamDelayMetrics_delayEndedWhileRealStreamNull_updateDelayReturnsEarly() {
FakeStreamTracer fakeTracer = new FakeStreamTracer();
ClientStreamTracer[] customTracers = new ClientStreamTracer[] { fakeTracer };

DelayedClientTransport.PendingStream pendingStream =
(DelayedClientTransport.PendingStream) delayedTransport.newStream(
method, headers, callOptions, customTracers);
pendingStream.start(streamListener);

// End delay directly while realStream is still null (sets delayEnded = true)
pendingStream.endDelay();
assertEquals(1, fakeTracer.delayEndedCount);

// updateDelay after delayEnded when getRealStream() == null should return early
// without starting a new delay segment
pendingStream.updateDelay("connecting", "new reason");
assertEquals(Collections.singletonList("connecting"), fakeTracer.startedDelayTypes);
assertEquals(1, fakeTracer.delayEndedCount);
}

private static TransportProvider newTransportProvider(final ClientTransport transport) {
return new TransportProvider() {
@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package io.grpc.internal;

import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;

import io.grpc.ClientStreamTracer;
import io.grpc.ForwardingTestUtil;
Expand All @@ -40,6 +41,19 @@ public void allMethodsForwarded() throws Exception {
Collections.<Method>emptyList());
}

@Test
public void attemptDelayMethodsForwarded() {
TestClientStreamTracer tracer = new TestClientStreamTracer();
tracer.recordAttemptDelayStart("connecting", "test");
verify(mockDelegate).recordAttemptDelayStart("connecting", "test");

tracer.recordAttemptDelayReasonChanged("test2");
verify(mockDelegate).recordAttemptDelayReasonChanged("test2");

tracer.recordAttemptDelayEnd();
verify(mockDelegate).recordAttemptDelayEnd();
}

private final class TestClientStreamTracer extends ForwardingClientStreamTracer {
@Override
protected ClientStreamTracer delegate() {
Expand Down
Loading
Loading