From 952a2898fb85d3698083bc9db528271b39267a2e Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 15:25:44 +0000 Subject: [PATCH 1/6] fix(bigtable): count pre-OpenSessionResponse stream failures in ChannelPoolDpImpl --- .../internal/channels/ChannelPoolDpImpl.java | 24 ++++++++++++-- .../channels/ChannelPoolDpImplTest.java | 31 +++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java index 1ef72a91937a..c5157de5ab5b 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java @@ -308,6 +308,7 @@ public synchronized SessionStream newStream( return new SessionStreamImpl(innerCall) { // null until onBeforeSessionStart fires @Nullable AfeId afeId = null; + boolean sessionEstablished = false; @Override public void start(Listener responseListener, Metadata headers) { @@ -317,8 +318,6 @@ public void start(Listener responseListener, Metadata headers) { public void onBeforeSessionStart(PeerInfo peerInfo) { afeId = AfeId.extract(peerInfo); synchronized (ChannelPoolDpImpl.this) { - fw.consecutiveFailures = 0; - recycleBackoff = INITIAL_RECYCLE_BACKOFF; sessionsPerAfeId.add(afeId); if (!fw.channel.isShutdown()) { routeObservations.put(new RouteKey(fw.id, tenant), afeId); @@ -328,13 +327,32 @@ public void onBeforeSessionStart(PeerInfo peerInfo) { super.onBeforeSessionStart(peerInfo); } + @Override + public void onMessage(SessionResponse msg) { + if (!sessionEstablished && msg.hasOpenSession()) { + synchronized (ChannelPoolDpImpl.this) { + sessionEstablished = true; + fw.consecutiveFailures = 0; + recycleBackoff = INITIAL_RECYCLE_BACKOFF; + } + } + super.onMessage(msg); + } + @Override public void onClose(Status status, Metadata trailers) { synchronized (ChannelPoolDpImpl.this) { if (afeId != null) { sessionsPerAfeId.remove(afeId); - } else if (!status.isOk() && status.getCode() != Code.CANCELLED) { + } + if (status.isOk()) { + fw.consecutiveFailures = 0; + recycleBackoff = INITIAL_RECYCLE_BACKOFF; + } else if (!sessionEstablished && status.getCode() != Code.CANCELLED) { fw.consecutiveFailures++; + if (afeId != null) { + routeObservations.remove(new RouteKey(fw.id, tenant), afeId); + } } releaseChannel(fw, status); } diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java index 4efca0494718..d1e8afb38d2b 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java @@ -632,4 +632,35 @@ void testRecycleChannelBackoff() { pool.close(); } + + @Test + void testPreOpenSessionFailureWithPeerInfoIncrementsFailures() { + when(channelSupplier.get()).thenReturn(channel); + when(channel.newCall(any(), any())).thenReturn(clientCall); + doNothing().when(clientCall).start(listener.capture(), any()); + doReturn(Attributes.EMPTY).when(clientCall).getAttributes(); + + ChannelPoolDpImpl pool = + new ChannelPoolDpImpl(channelSupplier, defaultConfig, debugTagTracer, bgExecutor); + + PeerInfo peerInfo = PeerInfo.newBuilder().setApplicationFrontendId(555).build(); + Metadata headers = new Metadata(); + headers.put( + SessionStreamImpl.PEER_INFO_KEY, + Base64.getEncoder().encodeToString(peerInfo.toByteArray())); + + // 5 pre-OpenSessionResponse rejections (onHeaders with PeerInfo followed by onClose(!OK) + // without OpenSessionResponse) must count as consecutive failures and recycle the channel. + for (int i = 0; i < 5; i++) { + pool.newStream(FakeSessionGrpc.getOpenSessionMethod(), CallOptions.DEFAULT) + .start(mock(Listener.class), new Metadata()); + listener.getValue().onHeaders(headers); + listener.getValue().onClose(Status.RESOURCE_EXHAUSTED, new Metadata()); + } + + verify(channel, times(1)).shutdown(); + verify(channelSupplier, times(2)).get(); + + pool.close(); + } } From e99f404f016cffdedd4c54702eae75cdd8fefc0d Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 17 Nov 2026 18:38:21 +0000 Subject: [PATCH 2/6] fix(bigtable): address PR 15039 review comments on onMessage synchronization and comments --- .../data/v2/internal/channels/ChannelPoolDpImpl.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java index c5157de5ab5b..1c98a5ac574a 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java @@ -329,9 +329,16 @@ public void onBeforeSessionStart(PeerInfo peerInfo) { @Override public void onMessage(SessionResponse msg) { + // Wait for OpenSessionResponse before marking the session established and resetting + // consecutiveFailures. onBeforeSessionStart is not enough because the AFE sends + // initial PeerInfo response headers before completing OpenSession, so streams + // rejected during OpenSession (e.g. RESOURCE_EXHAUSTED) still fire + // onBeforeSessionStart before closing with an error. if (!sessionEstablished && msg.hasOpenSession()) { + sessionEstablished = true; + // Synchronize on the pool to update shared channel/pool failure state + // (recycleBackoff is @GuardedBy("this")). synchronized (ChannelPoolDpImpl.this) { - sessionEstablished = true; fw.consecutiveFailures = 0; recycleBackoff = INITIAL_RECYCLE_BACKOFF; } From 62cc4ddfa15baf2fbe32d48c18a8070fa6dcd218 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 17 Nov 2026 18:57:47 +0000 Subject: [PATCH 3/6] refactor(bigtable): extract SessionStream.afterSessionStart instead of inspecting onMessage --- .../internal/channels/ChannelPoolDpImpl.java | 38 ++++++++----------- .../channels/FailingSessionStream.java | 5 +++ .../channels/ForwardingSessionStream.java | 5 +++ .../v2/internal/channels/SessionStream.java | 2 + .../internal/channels/SessionStreamImpl.java | 3 ++ .../data/v2/internal/session/SessionImpl.java | 1 + .../channels/ChannelPoolDpImplTest.java | 8 ++-- 7 files changed, 36 insertions(+), 26 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java index 1c98a5ac574a..00bdbb9d9df7 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java @@ -310,6 +310,20 @@ public synchronized SessionStream newStream( @Nullable AfeId afeId = null; boolean sessionEstablished = false; + @Override + public void afterSessionStart() { + // Only mark the session established and reset consecutiveFailures after + // OpenSessionResponse is processed. onBeforeSessionStart is not enough because the AFE + // sends initial PeerInfo response headers before completing OpenSession, so streams + // rejected during OpenSession (e.g. RESOURCE_EXHAUSTED) still fire + // onBeforeSessionStart before closing with an error. + synchronized (ChannelPoolDpImpl.this) { + sessionEstablished = true; + fw.consecutiveFailures = 0; + recycleBackoff = INITIAL_RECYCLE_BACKOFF; + } + } + @Override public void start(Listener responseListener, Metadata headers) { super.start( @@ -327,35 +341,13 @@ public void onBeforeSessionStart(PeerInfo peerInfo) { super.onBeforeSessionStart(peerInfo); } - @Override - public void onMessage(SessionResponse msg) { - // Wait for OpenSessionResponse before marking the session established and resetting - // consecutiveFailures. onBeforeSessionStart is not enough because the AFE sends - // initial PeerInfo response headers before completing OpenSession, so streams - // rejected during OpenSession (e.g. RESOURCE_EXHAUSTED) still fire - // onBeforeSessionStart before closing with an error. - if (!sessionEstablished && msg.hasOpenSession()) { - sessionEstablished = true; - // Synchronize on the pool to update shared channel/pool failure state - // (recycleBackoff is @GuardedBy("this")). - synchronized (ChannelPoolDpImpl.this) { - fw.consecutiveFailures = 0; - recycleBackoff = INITIAL_RECYCLE_BACKOFF; - } - } - super.onMessage(msg); - } - @Override public void onClose(Status status, Metadata trailers) { synchronized (ChannelPoolDpImpl.this) { if (afeId != null) { sessionsPerAfeId.remove(afeId); } - if (status.isOk()) { - fw.consecutiveFailures = 0; - recycleBackoff = INITIAL_RECYCLE_BACKOFF; - } else if (!sessionEstablished && status.getCode() != Code.CANCELLED) { + if (!sessionEstablished && !status.isOk() && status.getCode() != Code.CANCELLED) { fw.consecutiveFailures++; if (afeId != null) { routeObservations.remove(new RouteKey(fw.id, tenant), afeId); diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/FailingSessionStream.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/FailingSessionStream.java index df2ae9ad35c1..ae3f23630642 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/FailingSessionStream.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/FailingSessionStream.java @@ -39,6 +39,11 @@ public void start(Listener responseListener, Metadata headers) { responseListener.onClose(status, new Metadata()); } + @Override + public void afterSessionStart() { + // noop + } + @Override public void sendMessage(SessionRequest request) { // noop diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ForwardingSessionStream.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ForwardingSessionStream.java index 54652c044ee5..6fb8cae917f3 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ForwardingSessionStream.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ForwardingSessionStream.java @@ -37,6 +37,11 @@ public void start(Listener responseListener, Metadata headers) { delegate.start(responseListener, headers); } + @Override + public void afterSessionStart() { + delegate.afterSessionStart(); + } + @Override public void sendMessage(SessionRequest request) { delegate.sendMessage(request); diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStream.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStream.java index eb6dbfa339eb..5f99778e1c8c 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStream.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStream.java @@ -31,6 +31,8 @@ public interface SessionStream { public void start(Listener responseListener, Metadata headers); + public void afterSessionStart(); + public void sendMessage(SessionRequest request); public void halfClose(); diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStreamImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStreamImpl.java index fa46914df03b..4d271b5cbc10 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStreamImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/SessionStreamImpl.java @@ -103,6 +103,9 @@ public void onClose(Status status, Metadata trailers) { call.request(Integer.MAX_VALUE); } + @Override + public void afterSessionStart() {} + @Override public void sendMessage(SessionRequest request) { call.sendMessage(request); diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionImpl.java index d07bbc348d39..d80f301f63ac 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionImpl.java @@ -682,6 +682,7 @@ private void handleOpenSessionResponse(OpenSessionResponse openSession) { } PeerInfo localPeerInfo = stream.getPeerInfo(); updateState(SessionState.READY); + stream.afterSessionStart(); tracer.onOpen(localPeerInfo); sessionListener.onReady(openSession); } diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java index d1e8afb38d2b..bfdedf35f734 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImplTest.java @@ -517,9 +517,10 @@ void testResetConsecutiveFailuresOnSuccess() { } verify(channel, times(0)).shutdown(); - // A success: onHeaders (which calls onBeforeSessionStart) - pool.newStream(FakeSessionGrpc.getOpenSessionMethod(), CallOptions.DEFAULT) - .start(mock(Listener.class), new Metadata()); + // A success: onHeaders (which calls onBeforeSessionStart) followed by afterSessionStart() + SessionStream stream = + pool.newStream(FakeSessionGrpc.getOpenSessionMethod(), CallOptions.DEFAULT); + stream.start(mock(Listener.class), new Metadata()); PeerInfo peerInfo = PeerInfo.newBuilder().setApplicationFrontendId(555).build(); Metadata headers = new Metadata(); @@ -527,6 +528,7 @@ void testResetConsecutiveFailuresOnSuccess() { SessionStreamImpl.PEER_INFO_KEY, Base64.getEncoder().encodeToString(peerInfo.toByteArray())); listener.getValue().onHeaders(headers); + stream.afterSessionStart(); listener.getValue().onClose(Status.OK, new Metadata()); // Another 4 failures - should still not recycle because counter was reset From 3b6a79418a28d1deb89cc6aa8809100e9c277d4b Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 17 Nov 2026 21:06:15 +0000 Subject: [PATCH 4/6] fix(bigtable): mark sessionEstablished volatile in ChannelPoolDpImpl --- .../bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java index 00bdbb9d9df7..6907f40ee4a4 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java @@ -308,7 +308,7 @@ public synchronized SessionStream newStream( return new SessionStreamImpl(innerCall) { // null until onBeforeSessionStart fires @Nullable AfeId afeId = null; - boolean sessionEstablished = false; + volatile boolean sessionEstablished = false; @Override public void afterSessionStart() { @@ -317,8 +317,8 @@ public void afterSessionStart() { // sends initial PeerInfo response headers before completing OpenSession, so streams // rejected during OpenSession (e.g. RESOURCE_EXHAUSTED) still fire // onBeforeSessionStart before closing with an error. + sessionEstablished = true; synchronized (ChannelPoolDpImpl.this) { - sessionEstablished = true; fw.consecutiveFailures = 0; recycleBackoff = INITIAL_RECYCLE_BACKOFF; } From f60593cdf1ffedf1ba698d444937771023302d41 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 17 Nov 2026 21:26:38 +0000 Subject: [PATCH 5/6] docs(bigtable): document that ChannelWrapper mutable fields are guarded by ChannelPoolDpImpl --- .../bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java index 6907f40ee4a4..64ef972aa1bc 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java @@ -589,6 +589,7 @@ enum State { final ManagedChannel channel; final Instant createdAt; volatile State state = State.ACTIVE; + // Guarded by the enclosing ChannelPoolDpImpl instance. int numOutstanding = 0; int consecutiveFailures = 0; Instant lastUsedAt; From f8f8c83c789b776ba2d5b9642af1a01ddc36296e Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 17 Nov 2026 22:05:45 +0000 Subject: [PATCH 6/6] docs(bigtable): add guarded-by comment next to each mutable field in ChannelWrapper --- .../bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java index 64ef972aa1bc..f1db13451e75 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java @@ -591,7 +591,9 @@ enum State { volatile State state = State.ACTIVE; // Guarded by the enclosing ChannelPoolDpImpl instance. int numOutstanding = 0; + // Guarded by the enclosing ChannelPoolDpImpl instance. int consecutiveFailures = 0; + // Guarded by the enclosing ChannelPoolDpImpl instance. Instant lastUsedAt; ChannelWrapper(long id, ManagedChannel channel, Clock clock) {