From 650eaf2d32a04940b0b7f71c1d0740787783936b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:29:41 +0800 Subject: [PATCH] Fix pooled session cleanup and close races (#19288) * Fix pooled session and result-set cleanup * Serialize table session lifecycle operations * Make close concurrency test deterministic (cherry picked from commit 8b7cf270f7e47eec568afe412a7c366a46903502) --- .../iotdb/session/pool/SessionPool.java | 50 +++-- .../session/pool/SessionPoolCloseTest.java | 202 ++++++++++++++++++ 2 files changed, 239 insertions(+), 13 deletions(-) create mode 100644 iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/SessionPoolCloseTest.java diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java index dd522959d8555..495f79e75835a 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java @@ -534,7 +534,7 @@ public SessionPool(Builder builder) { } } - private Session constructNewSession() { + Session constructNewSession() { Session session; if (nodeUrls == null) { // Construct custom Session @@ -724,16 +724,20 @@ public int currentOccupiedSize() { @SuppressWarnings({"squid:S2446"}) private void putBack(ISession session) { - queue.push(session); synchronized (this) { - // we do not need to notifyAll as any waited thread can continue to work after waked up. - this.notify(); - // comment the following codes as putBack is too frequently called. - // if (logger.isTraceEnabled()) { - // logger.trace("put a session back and notify others..., queue.size = {}", - // queue.size()); - // } + if (!closed) { + queue.push(session); + // we do not need to notifyAll as any waited thread can continue to work after waked up. + this.notify(); + // comment the following codes as putBack is too frequently called. + // if (logger.isTraceEnabled()) { + // logger.trace("put a session back and notify others..., queue.size = {}", + // queue.size()); + // } + return; + } } + closeSession(session); } private void occupy(ISession session) { @@ -775,22 +779,42 @@ public synchronized void close() { @Override public void closeResultSet(SessionDataSetWrapper wrapper) { + ISession session; + synchronized (wrapper) { + session = wrapper.getSession(); + if (session == null) { + return; + } + // Detach before returning the session, so closing an old wrapper cannot release a new query. + wrapper.setSession(null); + session = occupied.remove(session); + } + if (session == null) { + return; + } boolean putback = true; try { wrapper.getSessionDataSet().closeOperationHandle(); } catch (IoTDBConnectionException | StatementExecutionException e) { - tryConstructNewSession(); putback = false; + closeSession(session); + tryConstructNewSession(); } finally { - ISession session = occupied.remove(wrapper.getSession()); - if (putback && session != null) { - putBack(wrapper.getSession()); + if (putback) { + putBack(session); } } } @SuppressWarnings({"squid:S2446"}) private void tryConstructNewSession() { + synchronized (this) { + if (closed) { + size--; + this.notify(); + return; + } + } Session session = constructNewSession(); try { session.open(enableCompression, connectionTimeoutInMs, deviceIdToEndpoint, availableNodes); diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/SessionPoolCloseTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/SessionPoolCloseTest.java new file mode 100644 index 0000000000000..8a8d05aed0662 --- /dev/null +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/SessionPoolCloseTest.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.pool; + +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.SessionDataSetWrapper; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; + +import org.apache.tsfile.write.record.Tablet; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SessionPoolCloseTest { + + private SessionPool pool; + private Session session; + private Session replacement; + + @Before + public void setUp() { + session = mock(Session.class); + replacement = mock(Session.class); + pool = spy(new SessionPool.Builder().maxSize(1).enableAutoFetch(false).build()); + doReturn(session, replacement).when(pool).constructNewSession(); + } + + @After + public void tearDown() { + pool.close(); + } + + @Test + public void testOldResultSetCannotReleaseReborrowedSession() throws Exception { + SessionDataSet firstDataSet = mock(SessionDataSet.class); + SessionDataSet secondDataSet = mock(SessionDataSet.class); + when(session.executeQueryStatement("select * from root.t")) + .thenReturn(firstDataSet, secondDataSet); + + try (SessionDataSetWrapper first = pool.executeQueryStatement("select * from root.t")) { + assertFalse(first.hasNext()); + try (SessionDataSetWrapper second = pool.executeQueryStatement("select * from root.t")) { + first.close(); + assertEquals(1, pool.currentOccupiedSize()); + assertEquals(0, pool.currentAvailableSize()); + } + } + + assertEquals(0, pool.currentOccupiedSize()); + assertEquals(1, pool.currentAvailableSize()); + verify(firstDataSet, times(1)).closeOperationHandle(); + verify(secondDataSet, times(1)).closeOperationHandle(); + } + + @Test + public void testResultSetConnectionFailureReplacesSessionOnce() throws Exception { + assertResultSetFailureReplacesSessionOnce(new IoTDBConnectionException("close failed")); + } + + @Test + public void testResultSetStatementFailureReplacesSessionOnce() throws Exception { + assertResultSetFailureReplacesSessionOnce(new StatementExecutionException("close failed")); + } + + @Test + public void testFailedPhysicalCloseStillReplacesSession() throws Exception { + doThrow(new IoTDBConnectionException("close failed")).when(session).close(); + assertResultSetFailureReplacesSessionOnce(new IoTDBConnectionException("close failed")); + } + + @Test + public void testConcurrentResultSetCloseReleasesSessionOnce() throws Exception { + SessionDataSet dataSet = mock(SessionDataSet.class); + when(session.executeQueryStatement("select * from root.t")).thenReturn(dataSet); + CountDownLatch closeStarted = new CountDownLatch(1); + CountDownLatch continueClose = new CountDownLatch(1); + doAnswer( + invocation -> { + closeStarted.countDown(); + assertTrue(continueClose.await(5, TimeUnit.SECONDS)); + return null; + }) + .when(dataSet) + .closeOperationHandle(); + + ExecutorService executor = Executors.newFixedThreadPool(2); + try (SessionDataSetWrapper result = pool.executeQueryStatement("select * from root.t")) { + Future firstClose = executor.submit(result::close); + assertTrue(closeStarted.await(5, TimeUnit.SECONDS)); + Future secondClose = executor.submit(result::close); + secondClose.get(5, TimeUnit.SECONDS); + assertEquals(0, pool.currentOccupiedSize()); + assertEquals(0, pool.currentAvailableSize()); + continueClose.countDown(); + firstClose.get(5, TimeUnit.SECONDS); + } finally { + continueClose.countDown(); + executor.shutdownNow(); + } + + verify(dataSet, times(1)).closeOperationHandle(); + assertEquals(1, pool.currentAvailableSize()); + } + + @Test + public void testReturningSessionAfterPoolCloseClosesConnection() throws Exception { + doAnswer( + invocation -> { + pool.close(); + return null; + }) + .when(session) + .executeNonQueryStatement("flush"); + + pool.executeNonQueryStatement("flush"); + + assertEquals(0, pool.currentAvailableSize()); + verify(session, times(1)).close(); + verify(pool, times(1)).constructNewSession(); + } + + @Test + public void testConnectionFailureAfterPoolCloseDoesNotCreateReplacement() throws Exception { + IoTDBConnectionException failure = new IoTDBConnectionException("insert failed"); + doAnswer( + invocation -> { + pool.close(); + throw failure; + }) + .when(session) + .insertTablet(any(Tablet.class), anyBoolean()); + + IoTDBConnectionException actual = + assertThrows(IoTDBConnectionException.class, () -> pool.insertTablet(mock(Tablet.class))); + + assertSame(failure, actual.getCause()); + assertEquals(0, pool.currentAvailableSize()); + verify(pool, times(1)).constructNewSession(); + verify(session, times(1)).close(); + } + + private void assertResultSetFailureReplacesSessionOnce(Exception failure) throws Exception { + SessionDataSet dataSet = mock(SessionDataSet.class); + when(session.executeQueryStatement("select * from root.t")).thenReturn(dataSet); + doThrow(failure).when(dataSet).closeOperationHandle(); + + try (SessionDataSetWrapper result = pool.executeQueryStatement("select * from root.t")) { + result.close(); + result.close(); + } + + verify(dataSet, times(1)).closeOperationHandle(); + assertEquals(0, pool.currentOccupiedSize()); + verify(session, times(1)).close(); + assertEquals(1, pool.currentAvailableSize()); + verify(pool, times(2)).constructNewSession(); + pool.executeNonQueryStatement("flush"); + verify(replacement).executeNonQueryStatement("flush"); + assertEquals(1, pool.currentAvailableSize()); + } +}