Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,23 +34,31 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.jspecify.annotations.NullMarked;

/**
* A composite implementation of {@link ApiTracer} that delegates all tracing events to a list of
* underlying tracers.
*
* <p>Completion and failure callbacks execute asynchronously on background worker threads where no
* trace span is active on thread-local storage. Wrapping those callbacks in {@code try (Scope
* ignored = inScope())} briefly activates the current span on the callback thread before any child
* tracer ends it, allowing sibling tracers (such as logging and metrics) to correlate error logs
* and metric exemplars with the active span before the scope closes on the same thread.
* <p>Attempt and operation completion callbacks often run on a different thread than the one that
* started the attempt, where no span is current. They are dispatched inside {@link #inScope()} so
* that sibling tracers (such as logging and metrics) observe the span activated by a tracing child
* before that child ends it. Entering and exiting that scope is best-effort: these callbacks are
* invoked from GAX's retry and future-completion machinery, where an exception would skip the
* remaining callbacks or prevent the RPC future from completing.
*
* <p>For internal use only.
*/
@NullMarked
@InternalApi
class CompositeTracer extends BaseApiTracer {
private static final Logger LOGGER = Logger.getLogger(CompositeTracer.class.getName());
private static final AtomicBoolean SCOPE_FAILURE_WARNED = new AtomicBoolean();
private static final Scope NOOP_CALLBACK_SCOPE = () -> {};

private final List<ApiTracer> children;

public CompositeTracer(List<ApiTracer> children) {
Expand Down Expand Up @@ -104,9 +112,43 @@ public Scope inScope() {
};
}

/**
* Best-effort {@link #inScope()} for completion callbacks.
*
* <p>Completion callbacks are invoked from {@code BasicRetryingFuture.handleAttempt} and from
* future listeners. If entering or closing the scope threw there, the remaining callbacks would
* be skipped and, on the attempt path, the RPC future would never complete. Failures are logged
* and the callbacks are dispatched without the scope instead.
*/
private Scope enterCallbackScope() {
final Scope scope;
try {
scope = inScope();
} catch (RuntimeException e) {
logScopeFailure("Failed to enter tracer scope; dispatching callbacks without it", e);
return NOOP_CALLBACK_SCOPE;
}
return () -> {
try {
scope.close();
} catch (RuntimeException e) {
logScopeFailure("Failed to close tracer scope", e);
}
};
}

/**
* Logs a best-effort scope failure at {@link Level#WARNING} once, and at {@link Level#FINE}
* thereafter to avoid log flooding.
*/
private static void logScopeFailure(String message, RuntimeException e) {
Level level = SCOPE_FAILURE_WARNED.compareAndSet(false, true) ? Level.WARNING : Level.FINE;
LOGGER.log(level, message, e);
}
Comment thread
jinseopkim0 marked this conversation as resolved.
Resolved

@Override
public void operationSucceeded() {
try (Scope ignored = inScope()) {
try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).operationSucceeded();
}
Expand All @@ -115,7 +157,7 @@ public void operationSucceeded() {

@Override
public void operationCancelled() {
try (Scope ignored = inScope()) {
try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).operationCancelled();
}
Expand All @@ -124,7 +166,7 @@ public void operationCancelled() {

@Override
public void operationFailed(Throwable error) {
try (Scope ignored = inScope()) {
try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).operationFailed(error);
}
Expand Down Expand Up @@ -155,7 +197,7 @@ public void attemptStarted(Object request, int attemptNumber) {

@Override
public void attemptSucceeded() {
try (Scope ignored = inScope()) {
try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptSucceeded();
}
Expand All @@ -164,7 +206,7 @@ public void attemptSucceeded() {

@Override
public void attemptCancelled() {
try (Scope ignored = inScope()) {
try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptCancelled();
}
Expand All @@ -173,7 +215,7 @@ public void attemptCancelled() {

@Override
public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) {
try (Scope ignored = inScope()) {
try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptFailed(error, delay);
}
Expand All @@ -182,7 +224,7 @@ public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) {

@Override
public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
try (Scope ignored = inScope()) {
try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptFailedDuration(error, delay);
}
Expand All @@ -191,7 +233,7 @@ public void attemptFailedDuration(Throwable error, java.time.Duration delay) {

@Override
public void attemptFailedRetriesExhausted(Throwable error) {
try (Scope ignored = inScope()) {
try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptFailedRetriesExhausted(error);
}
Expand All @@ -200,7 +242,7 @@ public void attemptFailedRetriesExhausted(Throwable error) {

@Override
public void attemptPermanentFailure(Throwable error) {
try (Scope ignored = inScope()) {
try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptPermanentFailure(error);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
/*
* Copyright 2026 Google LLC
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google LLC nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.api.gax.rpc;

import static com.google.common.truth.Truth.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;

import com.google.api.core.ApiFuture;
import com.google.api.core.ApiFutures;
import com.google.api.core.NanoClock;
import com.google.api.gax.retrying.RetrySettings;
import com.google.api.gax.rpc.testing.FakeCallContext;
import com.google.api.gax.rpc.testing.FakeStatusCode;
import com.google.api.gax.tracing.ApiTracer;
import com.google.api.gax.tracing.ApiTracerFactory;
import com.google.api.gax.tracing.BaseApiTracer;
import com.google.api.gax.tracing.BaseApiTracerFactory;
import com.google.api.gax.tracing.CompositeTracerFactory;
import com.google.api.gax.tracing.SpanName;
import com.google.api.gax.tracing.TracedUnaryCallable;
import com.google.common.collect.ImmutableList;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

/**
* End-to-end regression test verifying that a retrying traced RPC future completes with the
* original RPC error instead of hanging when a child tracer's {@link ApiTracer#inScope()} throws.
*/
class RetryingTracerScopeFailureTest {
private ScheduledExecutorService executor;
private ClientContext clientContext;

/** Initializes a scheduled executor and {@link ClientContext} for retrying callable tests. */
@BeforeEach
void setUp() {
executor = Executors.newSingleThreadScheduledExecutor();
clientContext =
ClientContext.newBuilder()
.setExecutor(executor)
.setClock(NanoClock.getDefaultClock())
.setDefaultCallContext(FakeCallContext.createDefault())
.build();
}

/** Shuts down the test executor. */
@AfterEach
void tearDown() {
executor.shutdownNow();
}

/**
* Verifies that the retrying future completes immediately with the original RPC error even when a
* child tracer's {@link ApiTracer#inScope()} throws during completion callbacks.
*/
@Test
void retryingFutureCompletes_whenTracerScopeThrows() {
ApiException rpcError =
new InvalidArgumentException(
null, FakeStatusCode.of(StatusCode.Code.INVALID_ARGUMENT), false);

UnaryCallable<Integer, Integer> transport =
new UnaryCallable<Integer, Integer>() {
@Override
public ApiFuture<Integer> futureCall(Integer request, ApiCallContext context) {
return ApiFutures.immediateFailedFuture(rpcError);
}
};

ApiTracer brokenScopeTracer =
new BaseApiTracer() {
@Override
public Scope inScope() {
throw new IllegalStateException("broken inScope()");
}
};

ApiTracerFactory tracerFactory =
new CompositeTracerFactory(
ImmutableList.of(
BaseApiTracerFactory.getInstance(),
(parent, spanName, operationType) -> brokenScopeTracer));

UnaryCallSettings<Integer, Integer> callSettings =
UnaryCallSettings.<Integer, Integer>newUnaryCallSettingsBuilder()
.setRetryableCodes(StatusCode.Code.UNAVAILABLE)
.setRetrySettings(RetrySettings.newBuilder().setMaxAttempts(3).build())
.build();

UnaryCallable<Integer, Integer> callable =
new TracedUnaryCallable<>(
Callables.retrying(transport, callSettings, clientContext),
tracerFactory,
SpanName.of("FakeClient", "FakeMethod"));

ApiFuture<Integer> future = callable.futureCall(1, FakeCallContext.createDefault());

// If the exception from inScope() escapes BasicRetryingFuture.handleAttempt, Guava's listener
// loop swallows it and the retrying future is never completed.
assertThat(future.isDone()).isTrue();
ExecutionException e = assertThrows(ExecutionException.class, future::get);
Comment thread
jinseopkim0 marked this conversation as resolved.
Resolved
assertThat(e).hasCauseThat().isSameInstanceAs(rpcError);
}
}
Loading
Loading