From df161ba812b74e57f460e84e6c493145728a16b0 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian <179120858+sakthivelmanii@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:18:19 +0000 Subject: [PATCH 1/2] test(spanner): unflake EndpointLifecycleManagerTest Wait for the initial zero-delay background probe to read CONNECTING state before mutating channel health and driving manual probe calls in EndpointLifecycleManagerTest. --- .../spi/v1/EndpointLifecycleManagerTest.java | 27 ++++++++++++++----- .../spanner/spi/v1/KeyRangeCacheTest.java | 15 ++++++++++- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/EndpointLifecycleManagerTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/EndpointLifecycleManagerTest.java index 5eb73362ee1f..67685792acd1 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/EndpointLifecycleManagerTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/EndpointLifecycleManagerTest.java @@ -79,6 +79,25 @@ private static void awaitCondition(String message, BooleanSupplier condition) { assertTrue(message, condition.getAsBoolean()); } + /** + * Registers {@code address} with its channel already CONNECTING and waits until the background + * prober's initial run (scheduled with zero delay) has read that state. A CONNECTING probe does + * not touch the transient failure counter, so once this returns, manual {@code probe()} calls are + * the only writers of that counter until the next scheduled probe one interval later. + */ + private static void registerAndAwaitInitialConnectingProbe( + EndpointLifecycleManager mgr, KeyRangeCacheTest.FakeEndpointCache cache, String address) { + KeyRangeCacheTest.FakeEndpoint endpoint = (KeyRangeCacheTest.FakeEndpoint) cache.get(address); + endpoint.setState(KeyRangeCacheTest.EndpointHealthState.CONNECTING); + registerAddresses(mgr, address); + awaitCondition( + "background prober should observe the initial CONNECTING state", + () -> endpoint.getStateCallCount() > 0); + EndpointLifecycleManager.EndpointState state = mgr.getEndpointState(address); + assertNotNull(state); + assertEquals(0, state.consecutiveTransientFailures); + } + @Test public void endpointCreationStartsProbing() throws Exception { KeyRangeCacheTest.FakeEndpointCache cache = new KeyRangeCacheTest.FakeEndpointCache(); @@ -277,9 +296,7 @@ public void transientFailureOscillationWithConnectingStillEvictsEndpoint() throw new EndpointLifecycleManager( cache, /* probeIntervalSeconds= */ 60, Duration.ofMinutes(30), Clock.systemUTC()); - registerAddresses(manager, "server1"); - awaitCondition( - "endpoint should be created in background", () -> cache.getIfPresent("server1") != null); + registerAndAwaitInitialConnectingProbe(manager, cache, "server1"); cache.setState("server1", KeyRangeCacheTest.EndpointHealthState.TRANSIENT_FAILURE); manager.probe("server1"); @@ -312,9 +329,7 @@ public void readyResetsTransientFailureCounterAfterRecovery() throws Exception { new EndpointLifecycleManager( cache, /* probeIntervalSeconds= */ 60, Duration.ofMinutes(30), Clock.systemUTC()); - registerAddresses(manager, "server1"); - awaitCondition( - "endpoint should be created in background", () -> cache.getIfPresent("server1") != null); + registerAndAwaitInitialConnectingProbe(manager, cache, "server1"); cache.setState("server1", KeyRangeCacheTest.EndpointHealthState.TRANSIENT_FAILURE); manager.probe("server1"); diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/KeyRangeCacheTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/KeyRangeCacheTest.java index c8dec3cef6ee..ff721cb369b7 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/KeyRangeCacheTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/KeyRangeCacheTest.java @@ -2023,6 +2023,11 @@ void setState(EndpointHealthState state) { channel.setConnectivityState(toConnectivityState(state)); } + /** Returns how many times the channel connectivity state has been read. */ + int getStateCallCount() { + return channel.getStateCallCount(); + } + private static ConnectivityState toConnectivityState(EndpointHealthState healthState) { switch (healthState) { case READY: @@ -2044,14 +2049,22 @@ private static ConnectivityState toConnectivityState(EndpointHealthState healthS private static final class FakeManagedChannel extends ManagedChannel { private volatile boolean shutdown = false; private volatile ConnectivityState connectivityState = ConnectivityState.READY; + private final AtomicInteger getStateCalls = new AtomicInteger(); void setConnectivityState(ConnectivityState state) { this.connectivityState = state; } + int getStateCallCount() { + return getStateCalls.get(); + } + @Override public ConnectivityState getState(boolean requestConnection) { - return shutdown ? ConnectivityState.SHUTDOWN : connectivityState; + ConnectivityState result = shutdown ? ConnectivityState.SHUTDOWN : connectivityState; + // Count after the state has been read so observers know the read already happened. + getStateCalls.incrementAndGet(); + return result; } @Override From 37308a218f8a23e75aa6bbbfacc2ab9ba45c6b26 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian <179120858+sakthivelmanii@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:40:24 +0000 Subject: [PATCH 2/2] Addressed comments --- .../spi/v1/EndpointLifecycleManager.java | 20 ++++++++++ .../spi/v1/EndpointLifecycleManagerTest.java | 39 ++++++++----------- .../spanner/spi/v1/KeyRangeCacheTest.java | 15 +------ 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/EndpointLifecycleManager.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/EndpointLifecycleManager.java index b0790a5e05b5..4a551a7b577a 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/EndpointLifecycleManager.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/EndpointLifecycleManager.java @@ -133,6 +133,7 @@ static final class EndpointState { private final long probeIntervalSeconds; private final Duration idleEvictionDuration; private final Clock clock; + private final boolean enableBackgroundProbing; private final String defaultEndpointAddress; private ScheduledFuture evictionFuture; @@ -161,10 +162,26 @@ private PendingActiveAddressUpdate(Set activeAddresses, long generation) long probeIntervalSeconds, Duration idleEvictionDuration, Clock clock) { + this( + endpointCache, + probeIntervalSeconds, + idleEvictionDuration, + clock, + /* enableBackgroundProbing= */ true); + } + + @VisibleForTesting + EndpointLifecycleManager( + ChannelEndpointCache endpointCache, + long probeIntervalSeconds, + Duration idleEvictionDuration, + Clock clock, + boolean enableBackgroundProbing) { this.endpointCache = endpointCache; this.probeIntervalSeconds = probeIntervalSeconds; this.idleEvictionDuration = idleEvictionDuration; this.clock = clock; + this.enableBackgroundProbing = enableBackgroundProbing; this.defaultEndpointAddress = endpointCache.defaultChannel().getAddress(); this.activeAddressUpdateExecutor = Executors.newSingleThreadExecutor( @@ -439,6 +456,9 @@ private void createAndStartProbing(String address) { /** Starts periodic probing for an endpoint. */ private void startProbing(String address) { + if (!enableBackgroundProbing) { + return; + } EndpointState state = endpoints.get(address); if (state == null || isShutdown.get()) { return; diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/EndpointLifecycleManagerTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/EndpointLifecycleManagerTest.java index 67685792acd1..95ba1cbaca76 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/EndpointLifecycleManagerTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/EndpointLifecycleManagerTest.java @@ -79,25 +79,6 @@ private static void awaitCondition(String message, BooleanSupplier condition) { assertTrue(message, condition.getAsBoolean()); } - /** - * Registers {@code address} with its channel already CONNECTING and waits until the background - * prober's initial run (scheduled with zero delay) has read that state. A CONNECTING probe does - * not touch the transient failure counter, so once this returns, manual {@code probe()} calls are - * the only writers of that counter until the next scheduled probe one interval later. - */ - private static void registerAndAwaitInitialConnectingProbe( - EndpointLifecycleManager mgr, KeyRangeCacheTest.FakeEndpointCache cache, String address) { - KeyRangeCacheTest.FakeEndpoint endpoint = (KeyRangeCacheTest.FakeEndpoint) cache.get(address); - endpoint.setState(KeyRangeCacheTest.EndpointHealthState.CONNECTING); - registerAddresses(mgr, address); - awaitCondition( - "background prober should observe the initial CONNECTING state", - () -> endpoint.getStateCallCount() > 0); - EndpointLifecycleManager.EndpointState state = mgr.getEndpointState(address); - assertNotNull(state); - assertEquals(0, state.consecutiveTransientFailures); - } - @Test public void endpointCreationStartsProbing() throws Exception { KeyRangeCacheTest.FakeEndpointCache cache = new KeyRangeCacheTest.FakeEndpointCache(); @@ -294,9 +275,15 @@ public void transientFailureOscillationWithConnectingStillEvictsEndpoint() throw KeyRangeCacheTest.FakeEndpointCache cache = new KeyRangeCacheTest.FakeEndpointCache(); manager = new EndpointLifecycleManager( - cache, /* probeIntervalSeconds= */ 60, Duration.ofMinutes(30), Clock.systemUTC()); + cache, + /* probeIntervalSeconds= */ 60, + Duration.ofMinutes(30), + Clock.systemUTC(), + /* enableBackgroundProbing= */ false); - registerAndAwaitInitialConnectingProbe(manager, cache, "server1"); + registerAddresses(manager, "server1"); + awaitCondition( + "endpoint should be created in background", () -> cache.getIfPresent("server1") != null); cache.setState("server1", KeyRangeCacheTest.EndpointHealthState.TRANSIENT_FAILURE); manager.probe("server1"); @@ -327,9 +314,15 @@ public void readyResetsTransientFailureCounterAfterRecovery() throws Exception { KeyRangeCacheTest.FakeEndpointCache cache = new KeyRangeCacheTest.FakeEndpointCache(); manager = new EndpointLifecycleManager( - cache, /* probeIntervalSeconds= */ 60, Duration.ofMinutes(30), Clock.systemUTC()); + cache, + /* probeIntervalSeconds= */ 60, + Duration.ofMinutes(30), + Clock.systemUTC(), + /* enableBackgroundProbing= */ false); - registerAndAwaitInitialConnectingProbe(manager, cache, "server1"); + registerAddresses(manager, "server1"); + awaitCondition( + "endpoint should be created in background", () -> cache.getIfPresent("server1") != null); cache.setState("server1", KeyRangeCacheTest.EndpointHealthState.TRANSIENT_FAILURE); manager.probe("server1"); diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/KeyRangeCacheTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/KeyRangeCacheTest.java index ff721cb369b7..c8dec3cef6ee 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/KeyRangeCacheTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/KeyRangeCacheTest.java @@ -2023,11 +2023,6 @@ void setState(EndpointHealthState state) { channel.setConnectivityState(toConnectivityState(state)); } - /** Returns how many times the channel connectivity state has been read. */ - int getStateCallCount() { - return channel.getStateCallCount(); - } - private static ConnectivityState toConnectivityState(EndpointHealthState healthState) { switch (healthState) { case READY: @@ -2049,22 +2044,14 @@ private static ConnectivityState toConnectivityState(EndpointHealthState healthS private static final class FakeManagedChannel extends ManagedChannel { private volatile boolean shutdown = false; private volatile ConnectivityState connectivityState = ConnectivityState.READY; - private final AtomicInteger getStateCalls = new AtomicInteger(); void setConnectivityState(ConnectivityState state) { this.connectivityState = state; } - int getStateCallCount() { - return getStateCalls.get(); - } - @Override public ConnectivityState getState(boolean requestConnection) { - ConnectivityState result = shutdown ? ConnectivityState.SHUTDOWN : connectivityState; - // Count after the state has been read so observers know the read already happened. - getStateCalls.incrementAndGet(); - return result; + return shutdown ? ConnectivityState.SHUTDOWN : connectivityState; } @Override