From a79c74663359dd6f04f0d5af47978ced2ecc62cb Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:43:56 +0800 Subject: [PATCH] [Test] Stabilize Pipe concurrency tests (#19143) * [Test] Stabilize Pipe concurrency tests * Stabilize DataNode shutdown status checks in integration tests (cherry picked from commit f12fe701c8ce4156b4929754ae8cbb6fe0d3d482) --- .../iotdb/it/env/cluster/env/AbstractEnv.java | 8 +- .../env/cluster/node/AbstractNodeWrapper.java | 11 +- .../it/partition/IoTDBPartitionDurableIT.java | 3 +- .../iotdb/session/it/pool/SessionPoolIT.java | 8 +- .../agent/task/PipeDataNodeTaskAgentTest.java | 8 +- ...ipeProcessorSubtaskExecutionGuardTest.java | 46 ++++-- .../PipeProcessorSubtaskWorkerTest.java | 84 ++++++---- .../subtask/sink/PipeSinkSubtaskTest.java | 151 ++++++++++++------ .../sink/PipeDataRegionSinkMetricsTest.java | 38 +++-- .../memory/PipeMemoryManagerTest.java | 11 +- .../PipeTransferTrackableHandlerTest.java | 16 +- .../disruptor/DisruptorShutdownTest.java | 81 ++++++---- 12 files changed, 296 insertions(+), 169 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java index ffbd9b78d70d4..560877e17fabd 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java @@ -81,6 +81,8 @@ public abstract class AbstractEnv implements BaseEnv { private static final Logger logger = IoTDBTestLogger.logger; private static final int DEFAULT_CLUSTER_READY_RETRY_COUNT = 30; + // Shutdown hooks may wait for Pipe events, progress persistence, and Ratis snapshots. + private static final int DATA_NODE_SHUTDOWN_TIMEOUT_SECONDS = 60; private static final String CLUSTER_READY_RETRY_COUNT_PROPERTY = "integrationTest.clusterReadyRetryCount"; @@ -1319,12 +1321,14 @@ public void startAllDataNodes() { @Override public void shutdownDataNode(final int index) { - dataNodeWrapperList.get(index).stop(); + dataNodeWrapperList.get(index).stop(DATA_NODE_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS); } @Override public void shutdownAllDataNodes() { - dataNodeWrapperList.forEach(AbstractNodeWrapper::stop); + dataNodeWrapperList.forEach( + dataNodeWrapper -> + dataNodeWrapper.stop(DATA_NODE_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)); } @Override diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index 701c19de81761..9b588291b026c 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -488,13 +488,20 @@ public void start() { @Override public void stop() { + stop(20, TimeUnit.SECONDS); + } + + public void stop(final long timeout, final TimeUnit unit) { if (this.instance == null) { return; } this.instance.destroy(); try { - if (!this.instance.waitFor(20, TimeUnit.SECONDS)) { - this.instance.destroyForcibly().waitFor(10, TimeUnit.SECONDS); + if (!this.instance.waitFor(timeout, unit)) { + logger.warn("Node {} does not exit within {} {}, killing it", getId(), timeout, unit); + if (!this.instance.destroyForcibly().waitFor(10, TimeUnit.SECONDS)) { + logger.error("Cannot forcibly stop node {}", getId()); + } } } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java index 93780ec68dc0b..f0c1f550a9ade 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java @@ -274,10 +274,11 @@ public void testReadOnlyDataNode() throws Exception { @Test public void testUnknownDataNode() throws Exception { // Shutdown a DataNode, the ConfigNode should still be able to create RegionGroup + DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(testDataNodeId); EnvFactory.getEnv().shutdownDataNode(testDataNodeId); EnvFactory.getEnv() .ensureNodeStatus( - Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(testDataNodeId)), + Collections.singletonList(dataNodeWrapper), Collections.singletonList(NodeStatus.Unknown)); try (SyncConfigNodeIServiceClient client = diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java index d8fb9bb4c5072..aba34ae2eb592 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java @@ -248,7 +248,7 @@ public void tryIfTheServerIsRestart() { BaseNodeWrapper node = EnvFactory.getEnv().getDataNodeWrapper(0); try { wrapper = pool.executeQueryStatement("select * from root.sg1.d1 where time > 1"); - node.stop(); + EnvFactory.getEnv().shutdownDataNode(0); EnvFactory.getEnv() .ensureNodeStatus( Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown)); @@ -259,7 +259,7 @@ public void tryIfTheServerIsRestart() { } catch (IoTDBConnectionException e) { pool.closeResultSet(wrapper); pool.close(); - node.stop(); + EnvFactory.getEnv().shutdownDataNode(0); EnvFactory.getEnv() .ensureNodeStatus( Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown)); @@ -285,7 +285,7 @@ public void tryIfTheServerIsRestart() { } catch (IoTDBConnectionException ec) { pool.closeResultSet(wrapper); pool.close(); - node.stop(); + EnvFactory.getEnv().shutdownDataNode(0); EnvFactory.getEnv() .ensureNodeStatus( Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown)); @@ -341,7 +341,7 @@ public void restart() { // stop the server. pool.close(); BaseNodeWrapper node = EnvFactory.getEnv().getDataNodeWrapper(0); - node.stop(); + EnvFactory.getEnv().shutdownDataNode(0); EnvFactory.getEnv() .ensureNodeStatus( Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgentTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgentTest.java index 072a6df702e3e..1231b9dacbde0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgentTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgentTest.java @@ -176,7 +176,7 @@ public void testSinkCriticalExceptionIsPropagatedOnlyWithinItsPipe() throws Exce taskAgent.stopAllPipesWithCriticalExceptionAndTrackException(failedTaskMeta, exception); Awaitility.await() - .atMost(5, TimeUnit.SECONDS) + .atMost(30, TimeUnit.SECONDS) .until( () -> PipeStatus.STOPPED.equals(failedPipeMeta.getRuntimeMeta().getStatus().get()) @@ -249,7 +249,7 @@ public void testDetachedTaskMetaMustIdentifyOnePipeUniquely() throws Exception { taskAgent.stopAllPipesWithCriticalExceptionAndTrackException( detachedTaskMeta, new PipeRuntimeCriticalException("ambiguous task meta", 1L)); - Awaitility.await().atMost(2, TimeUnit.SECONDS).until(detachedTaskMeta::hasExceptionMessages); + Awaitility.await().atMost(30, TimeUnit.SECONDS).until(detachedTaskMeta::hasExceptionMessages); pipeMetaKeeper.acquireWriteLock(); pipeMetaKeeper.releaseWriteLock(); @@ -286,7 +286,7 @@ public void testDetachedTaskMetaIsRecordedOnIdentifiedPipe() throws Exception { "detachedPipe", 2L, detachedTaskMeta, exception); Awaitility.await() - .atMost(5, TimeUnit.SECONDS) + .atMost(30, TimeUnit.SECONDS) .until( () -> PipeStatus.STOPPED.equals(pipeMeta.getRuntimeMeta().getStatus().get()) @@ -320,7 +320,7 @@ public void testCriticalExceptionWithNullTaskMetaStopsIdentifiedPipe() throws Ex "nullTaskMetaPipe", 3L, null, exception); Awaitility.await() - .atMost(5, TimeUnit.SECONDS) + .atMost(30, TimeUnit.SECONDS) .until( () -> PipeStatus.STOPPED.equals(failedPipeMeta.getRuntimeMeta().getStatus().get()) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskExecutionGuardTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskExecutionGuardTest.java index 02fb86c1538ff..78ea989de011e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskExecutionGuardTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskExecutionGuardTest.java @@ -177,7 +177,7 @@ public void testParserAdmissionYieldsWithoutBlockingAndResumes() throws Exceptio } } - @Test(timeout = 60000) + @Test(timeout = 120000) public void testCloseCancelsReservationEnqueuedByAWorkerInvocation() throws Exception { final CommonConfig config = CommonDescriptor.getInstance().getConfig(); final PipeMemoryManager manager = PipeDataNodeResourceManager.memory(); @@ -225,7 +225,7 @@ public void testCloseCancelsReservationEnqueuedByAWorkerInvocation() throws Exce eventToClose.close(); }); await() - .atMost(5, TimeUnit.SECONDS) + .atMost(30, TimeUnit.SECONDS) .until( () -> closingThread.get() != null @@ -240,27 +240,39 @@ public void testCloseCancelsReservationEnqueuedByAWorkerInvocation() throws Exce Assert.assertEquals(1, manager.getWaitingTsFileParserCount()); Assert.assertTrue(outputQueue.isEmpty()); } - closing.get(5, TimeUnit.SECONDS); + closing.get(30, TimeUnit.SECONDS); Assert.assertFalse(event.isWaitingForTsFileParserMemory()); Assert.assertEquals(0, manager.getWaitingTsFileParserCount()); Assert.assertEquals(1, manager.getReservedTsFileParserCount()); } finally { executor.shutdownNow(); - Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); - if (subtask != null) { - subtask.disallowSubmittingSelf(); - subtask.close(); - } - if (event != null) { - event.close(); - } - if (blockerReserved) { - manager.releaseTsFileParserMemory("blocker", 0, "0"); + try { + Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS)); + } finally { + try { + if (subtask != null) { + subtask.disallowSubmittingSelf(); + subtask.close(); + } + } finally { + try { + if (event != null) { + event.close(); + } + } finally { + try { + if (blockerReserved) { + manager.releaseTsFileParserMemory("blocker", 0, "0"); + } + } finally { + config.setPipeTsFileParserMemory(originalParserMemory); + config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit); + config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit); + FileUtils.deleteFileOrDirectory(tempDir); + } + } + } } - config.setPipeTsFileParserMemory(originalParserMemory); - config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit); - config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit); - FileUtils.deleteFileOrDirectory(tempDir); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskWorkerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskWorkerTest.java index d3d55552a0eee..5433789d7ba69 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskWorkerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskWorkerTest.java @@ -164,7 +164,7 @@ public void testRequestsOnOneWorkerDoNotStrandOtherWorkersParserSlots() throws E } } - @Test(timeout = 30000) + @Test(timeout = 120000) public void testOneHundredWaitingPipesDoNotWaitForWorkerPolling() throws Exception { final CommonConfig config = CommonDescriptor.getInstance().getConfig(); final PipeMemoryManager memoryManager = PipeDataNodeResourceManager.memory(); @@ -232,8 +232,8 @@ public void testOneHundredWaitingPipesDoNotWaitForWorkerPolling() throws Excepti final AtomicReference workerThread = workerThreads.get(i); // Consume the scheduling notification before all workers become idle on admission. worker.sleepIfNecessary(true); - // Make a missed wake-up observable without relying on a tight performance assertion. - worker.setSleepingTimeInMilliSecond(60000L); + // Keep polling beyond the whole test timeout so it cannot hide a missed wake-up. + worker.setSleepingTimeInMilliSecond(180000L); futures.add( executor.submit( () -> { @@ -244,7 +244,7 @@ public void testOneHundredWaitingPipesDoNotWaitForWorkerPolling() throws Excepti })); } await() - .atMost(5, TimeUnit.SECONDS) + .atMost(30, TimeUnit.SECONDS) .until( () -> workerThreads.stream() @@ -258,25 +258,34 @@ public void testOneHundredWaitingPipesDoNotWaitForWorkerPolling() throws Excepti } Assert.assertTrue( "Admission notifications must wake every queue head", - processed.await(5, TimeUnit.SECONDS)); + processed.await(30, TimeUnit.SECONDS)); Assert.assertEquals(0, memoryManager.getWaitingTsFileParserCount()); Assert.assertEquals(0, memoryManager.getReservedTsFileParserCount()); } finally { futures.forEach(future -> future.cancel(true)); executor.shutdownNow(); - Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); - for (int i = 0; i < requests.size(); i++) { - memoryManager.cancelTsFileParserMemoryReservation( - "waitingPipe" + i, 0, "0", requests.get(i)); - } - for (int i = 0; i < reservedBlockers.length; i++) { - if (reservedBlockers[i]) { - memoryManager.releaseTsFileParserMemory("blocker" + i, 0, "0"); + try { + Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS)); + } finally { + try { + for (int i = 0; i < requests.size(); i++) { + memoryManager.cancelTsFileParserMemoryReservation( + "waitingPipe" + i, 0, "0", requests.get(i)); + } + } finally { + try { + for (int i = 0; i < reservedBlockers.length; i++) { + if (reservedBlockers[i]) { + memoryManager.releaseTsFileParserMemory("blocker" + i, 0, "0"); + } + } + } finally { + config.setPipeTsFileParserMemory(originalParserMemory); + config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit); + config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit); + } } } - config.setPipeTsFileParserMemory(originalParserMemory); - config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit); - config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit); } } @@ -333,7 +342,7 @@ public void testWaitingParserIsSkippedUntilAdmissionNotification() throws Except } } - @Test(timeout = 10000) + @Test(timeout = 90000) public void testNotificationBeforeIdleWaitIsNotLost() throws Exception { final PipeProcessorSubtaskWorker worker = new PipeProcessorSubtaskWorker(new LinkedHashSet<>()); worker.setSleepingTimeInMilliSecond(60000L); @@ -351,14 +360,14 @@ public void testNotificationBeforeIdleWaitIsNotLost() throws Exception { final ExecutorService executor = Executors.newSingleThreadExecutor(); try { wakeUpCallback.get().run(); - executor.submit(() -> worker.sleepIfNecessary(true)).get(3, TimeUnit.SECONDS); + executor.submit(() -> worker.sleepIfNecessary(true)).get(30, TimeUnit.SECONDS); } finally { executor.shutdownNow(); - Assert.assertTrue(executor.awaitTermination(3, TimeUnit.SECONDS)); + Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS)); } } - @Test(timeout = 15000) + @Test(timeout = 150000) public void testBusyFirstWorkerDoesNotSerializeOtherEligibleWorker() throws Exception { final CommonConfig config = CommonDescriptor.getInstance().getConfig(); final PipeMemoryManager manager = PipeDataNodeResourceManager.memory(); @@ -394,7 +403,7 @@ public void testBusyFirstWorkerDoesNotSerializeOtherEligibleWorker() throws Exce ignored -> { if (blockFirstWorker.get()) { firstWorkerBusy.countDown(); - Assert.assertTrue(unblockFirstWorker.await(5, TimeUnit.SECONDS)); + unblockFirstWorker.await(); } return false; }); @@ -448,7 +457,7 @@ public void testBusyFirstWorkerDoesNotSerializeOtherEligibleWorker() throws Exce Assert.assertTrue(secondWorker.runSubtasks()); blockFirstWorker.set(true); final Future busyRound = executor.submit(firstWorker::runSubtasks); - Assert.assertTrue(firstWorkerBusy.await(3, TimeUnit.SECONDS)); + Assert.assertTrue(firstWorkerBusy.await(30, TimeUnit.SECONDS)); for (int i = 0; i < reservedBlockers.length; i++) { manager.releaseTsFileParserMemory("blocker" + i, 0, "0"); reservedBlockers[i] = false; @@ -459,27 +468,36 @@ public void testBusyFirstWorkerDoesNotSerializeOtherEligibleWorker() throws Exce secondWorker.sleepIfNecessary(secondWorker.runSubtasks()); } }); - Assert.assertTrue(secondAdmitted.await(3, TimeUnit.SECONDS)); + Assert.assertTrue(secondAdmitted.await(30, TimeUnit.SECONDS)); verify(firstSubtask, times(1)).call(); Assert.assertFalse(busyRound.isDone()); unblockFirstWorker.countDown(); - busyRound.get(3, TimeUnit.SECONDS); + busyRound.get(30, TimeUnit.SECONDS); Assert.assertEquals(0, manager.getWaitingTsFileParserCount()); Assert.assertEquals(0, manager.getReservedTsFileParserCount()); } finally { unblockFirstWorker.countDown(); executor.shutdownNow(); - Assert.assertTrue(executor.awaitTermination(3, TimeUnit.SECONDS)); - manager.cancelTsFileParserMemoryReservation("first", 0, "0", first); - manager.cancelTsFileParserMemoryReservation("second", 0, "0", second); - for (int i = 0; i < reservedBlockers.length; i++) { - if (reservedBlockers[i]) { - manager.releaseTsFileParserMemory("blocker" + i, 0, "0"); + try { + Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS)); + } finally { + try { + manager.cancelTsFileParserMemoryReservation("first", 0, "0", first); + manager.cancelTsFileParserMemoryReservation("second", 0, "0", second); + } finally { + try { + for (int i = 0; i < reservedBlockers.length; i++) { + if (reservedBlockers[i]) { + manager.releaseTsFileParserMemory("blocker" + i, 0, "0"); + } + } + } finally { + config.setPipeTsFileParserMemory(originalParserMemory); + config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit); + config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit); + } } } - config.setPipeTsFileParserMemory(originalParserMemory); - config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit); - config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskTest.java index 2b99cccbe9a90..86bf4c5b25f8b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskTest.java @@ -45,9 +45,11 @@ import org.mockito.Mockito; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; @@ -114,31 +116,44 @@ public void testExternalSinkDiscardEventsOfPipeNotBlockedByConnectionRetry() thr final Thread failureThread = new Thread(() -> subtask.onFailure(new PipeConnectionException("connection broken"))); - failureThread.start(); - Assert.assertTrue(handshakeEntered.await(5, TimeUnit.SECONDS)); - - final CountDownLatch discardReturned = new CountDownLatch(1); - final Thread discardThread = - new Thread( - () -> { - try { - subtask.discardEventsOfPipe(new CommitterKey("pipe", 1L, 1, -1)); - } finally { - discardReturned.countDown(); - } - }); - discardThread.start(); + failureThread.setDaemon(true); + Thread discardThread = null; try { - Assert.assertTrue(discardReturned.await(5, TimeUnit.SECONDS)); + failureThread.start(); + Assert.assertTrue(handshakeEntered.await(30, TimeUnit.SECONDS)); + + final CountDownLatch discardReturned = new CountDownLatch(1); + discardThread = + new Thread( + () -> { + try { + subtask.discardEventsOfPipe(new CommitterKey("pipe", 1L, 1, -1)); + } finally { + discardReturned.countDown(); + } + }); + discardThread.setDaemon(true); + discardThread.start(); + + Assert.assertTrue(discardReturned.await(30, TimeUnit.SECONDS)); Assert.assertEquals(1L, discardEntered.getCount()); Assert.assertFalse(discardDuringHandshake.get()); } finally { releaseHandshake.countDown(); - failureThread.join(5000); - Assert.assertTrue(discardEntered.await(1, TimeUnit.SECONDS)); - Assert.assertFalse(discardDuringHandshake.get()); - subtask.close(); + try { + failureThread.join(30_000); + Assert.assertFalse(failureThread.isAlive()); + if (discardThread != null) { + discardThread.join(30_000); + Assert.assertFalse(discardThread.isAlive()); + // Closing the subtask also drains pending discards. Check the handshake path first. + Assert.assertTrue(discardEntered.await(30, TimeUnit.SECONDS)); + Assert.assertFalse(discardDuringHandshake.get()); + } + } finally { + subtask.close(); + } } } @@ -171,9 +186,7 @@ public void testBuiltinSinkDiscardEventsOfPipeWaitsForConnectionRetry() throws E final Thread failureThread = new Thread(() -> subtask.onFailure(new PipeConnectionException("connection broken"))); - failureThread.start(); - Assert.assertTrue(handshakeEntered.await(5, TimeUnit.SECONDS)); - + failureThread.setDaemon(true); final CountDownLatch discardReturned = new CountDownLatch(1); final Thread discardThread = new Thread( @@ -184,22 +197,40 @@ public void testBuiltinSinkDiscardEventsOfPipeWaitsForConnectionRetry() throws E discardReturned.countDown(); } }); - discardThread.start(); + discardThread.setDaemon(true); try { - Assert.assertFalse(discardReturned.await(100, TimeUnit.MILLISECONDS)); + failureThread.start(); + Assert.assertTrue(handshakeEntered.await(30, TimeUnit.SECONDS)); + + discardThread.start(); + await() + .atMost(30, TimeUnit.SECONDS) + .until( + () -> + discardThread.getState() == Thread.State.WAITING + || discardReturned.getCount() == 0); + Assert.assertEquals(1L, discardReturned.getCount()); Assert.assertEquals(1L, discardEntered.getCount()); Assert.assertFalse(discardDuringHandshake.get()); + + releaseHandshake.countDown(); + discardThread.join(30_000); + failureThread.join(30_000); + Assert.assertFalse(discardThread.isAlive()); + Assert.assertFalse(failureThread.isAlive()); + // Closing the subtask also drains pending discards. Check normal completion first. + Assert.assertTrue(discardEntered.await(30, TimeUnit.SECONDS)); + Assert.assertFalse(discardDuringHandshake.get()); } finally { releaseHandshake.countDown(); - discardThread.join(5000); - failureThread.join(5000); - subtask.close(); + try { + discardThread.join(30_000); + failureThread.join(30_000); + } finally { + subtask.close(); + } } - - Assert.assertFalse(discardThread.isAlive()); - Assert.assertTrue(discardEntered.await(1, TimeUnit.SECONDS)); - Assert.assertFalse(discardDuringHandshake.get()); } @Test @@ -233,24 +264,45 @@ public void testExternalSinkCloseNotConcurrentWithConnectionRetry() throws Excep final Thread failureThread = new Thread(() -> subtask.onFailure(new PipeConnectionException("connection broken"))); - failureThread.start(); + failureThread.setDaemon(true); + final FutureTask closeTask = + new FutureTask<>( + () -> { + final long startTime = System.nanoTime(); + subtask.close(); + return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); + }); + final Thread closeThread = new Thread(closeTask); + closeThread.setDaemon(true); + boolean closeStarted = false; try { - Assert.assertTrue(handshakeEntered.await(5, TimeUnit.SECONDS)); + failureThread.start(); + Assert.assertTrue(handshakeEntered.await(30, TimeUnit.SECONDS)); - final long startTime = System.currentTimeMillis(); - subtask.close(); + closeThread.start(); + closeStarted = true; - Assert.assertTrue(System.currentTimeMillis() - startTime < 1000); + Assert.assertTrue(closeTask.get(30, TimeUnit.SECONDS) < 1000); Assert.assertFalse(closeEntered.await(100, TimeUnit.MILLISECONDS)); Assert.assertFalse(closeDuringHandshake.get()); } finally { releaseHandshake.countDown(); - Assert.assertTrue(closeEntered.await(1, TimeUnit.SECONDS)); - failureThread.join(5000); - Assert.assertFalse(closeDuringHandshake.get()); - CommonDescriptor.getInstance().getConfig().setDnConnectionTimeoutInMS(originalTimeout); + try { + if (!closeStarted) { + subtask.close(); + } + closeThread.join(30_000); + failureThread.join(30_000); + Assert.assertFalse(closeThread.isAlive()); + Assert.assertTrue(closeEntered.await(30, TimeUnit.SECONDS)); + Assert.assertFalse(closeDuringHandshake.get()); + } finally { + CommonDescriptor.getInstance().getConfig().setDnConnectionTimeoutInMS(originalTimeout); + } } + + Assert.assertFalse(failureThread.isAlive()); } @Test @@ -285,9 +337,10 @@ public void testExternalSinkCloseDoesNotWaitForeverForConnectorClose() throws Ex try { final long startTime = System.currentTimeMillis(); subtask.close(); + final long closeDuration = System.currentTimeMillis() - startTime; - Assert.assertTrue(closeEntered.await(1, TimeUnit.SECONDS)); - Assert.assertTrue(System.currentTimeMillis() - startTime < 1000); + Assert.assertTrue(closeDuration < 1000); + Assert.assertTrue(closeEntered.await(30, TimeUnit.SECONDS)); } finally { releaseClose.countDown(); CommonDescriptor.getInstance().getConfig().setDnConnectionTimeoutInMS(originalTimeout); @@ -308,7 +361,7 @@ public void testBuiltinSinkCloseWaitsForConnectorClose() throws Exception { doAnswer( invocation -> { closeEntered.countDown(); - releaseClose.await(5, TimeUnit.SECONDS); + releaseClose.await(); return null; }) .when(connector) @@ -334,15 +387,19 @@ public void testBuiltinSinkCloseWaitsForConnectorClose() throws Exception { closeReturned.countDown(); } }); + closeThread.setDaemon(true); try { closeThread.start(); - Assert.assertTrue(closeEntered.await(5, TimeUnit.SECONDS)); + Assert.assertTrue(closeEntered.await(30, TimeUnit.SECONDS)); Assert.assertFalse(closeReturned.await(100, TimeUnit.MILLISECONDS)); } finally { releaseClose.countDown(); - closeThread.join(5000); - CommonDescriptor.getInstance().getConfig().setDnConnectionTimeoutInMS(originalTimeout); + try { + closeThread.join(30_000); + } finally { + CommonDescriptor.getInstance().getConfig().setDnConnectionTimeoutInMS(originalTimeout); + } } Assert.assertFalse(closeThread.isAlive()); @@ -566,7 +623,7 @@ public void handshake() throws Exception { handshaking = true; handshakeEntered.countDown(); try { - releaseHandshake.await(5, TimeUnit.SECONDS); + releaseHandshake.await(); } finally { handshaking = false; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/sink/PipeDataRegionSinkMetricsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/sink/PipeDataRegionSinkMetricsTest.java index e31e6c188ef7d..17e381083be65 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/sink/PipeDataRegionSinkMetricsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/sink/PipeDataRegionSinkMetricsTest.java @@ -32,9 +32,10 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import static org.awaitility.Awaitility.await; import static org.mockito.Mockito.when; public class PipeDataRegionSinkMetricsTest { @@ -52,7 +53,7 @@ public void testDeregisterWaitsForBind() throws Exception { invocation -> { if (blockBindOnce.compareAndSet(true, false)) { bindEntered.countDown(); - Assert.assertTrue(continueBind.await(5, TimeUnit.SECONDS)); + continueBind.await(); } return "pipe"; }); @@ -71,28 +72,35 @@ public void testDeregisterWaitsForBind() throws Exception { final ExecutorService executor = Executors.newFixedThreadPool(2); final Future bindFuture = executor.submit(() -> metrics.bindTo(metricService)); - Future deregisterFuture = null; try { - Assert.assertTrue(bindEntered.await(5, TimeUnit.SECONDS)); - deregisterFuture = executor.submit(() -> metrics.deregister(taskId)); + Assert.assertTrue(bindEntered.await(30, TimeUnit.SECONDS)); + final AtomicReference deregisterThread = new AtomicReference<>(); + final Future deregisterFuture = + executor.submit( + () -> { + deregisterThread.set(Thread.currentThread()); + metrics.deregister(taskId); + }); try { - try { - deregisterFuture.get(200, TimeUnit.MILLISECONDS); - Assert.fail("Deregistration must wait until metric binding completes"); - } catch (final TimeoutException expected) { - // Expected: bindTo holds the lifecycle lock until all metrics are created. - } + await() + .atMost(30, TimeUnit.SECONDS) + .until( + () -> + deregisterFuture.isDone() + || (deregisterThread.get() != null + && deregisterThread.get().getState() == Thread.State.BLOCKED)); + Assert.assertFalse( + "Deregistration must wait until metric binding completes", deregisterFuture.isDone()); } finally { continueBind.countDown(); } - bindFuture.get(5, TimeUnit.SECONDS); - Assert.assertNotNull(deregisterFuture); - deregisterFuture.get(5, TimeUnit.SECONDS); + bindFuture.get(30, TimeUnit.SECONDS); + deregisterFuture.get(30, TimeUnit.SECONDS); } finally { continueBind.countDown(); executor.shutdownNow(); - Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS)); } } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerTest.java index c945e10fe2b8b..fe354fece06aa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerTest.java @@ -777,7 +777,8 @@ public void testConcurrentTsFilesFromMultiplePipesAreNotStarved() throws Excepti boolean acquired = tryAcquireWithoutTracking(reservation); enqueued.countDown(); - final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + // Allow time for all contenders to enqueue before the blocker is released. + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60); while (!acquired && System.nanoTime() < deadline) { reservation.key.await( Math.max( @@ -797,20 +798,20 @@ public void testConcurrentTsFilesFromMultiplePipesAreNotStarved() throws Excepti })); } - Assert.assertTrue(ready.await(5, TimeUnit.SECONDS)); + Assert.assertTrue(ready.await(30, TimeUnit.SECONDS)); start.countDown(); - Assert.assertTrue(enqueued.await(5, TimeUnit.SECONDS)); + Assert.assertTrue(enqueued.await(30, TimeUnit.SECONDS)); release(blocker); for (final Future future : futures) { - Assert.assertTrue(future.get(15, TimeUnit.SECONDS)); + Assert.assertTrue(future.get(30, TimeUnit.SECONDS)); } Assert.assertEquals(waitingTsFiles.size(), acquisitionOrder.size()); Assert.assertEquals(3, new HashSet<>(acquisitionOrder.subList(0, 3)).size()); } finally { release(blocker); executor.shutdownNow(); - Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS)); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTrackableHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTrackableHandlerTest.java index 522a1d06fe5d0..eb142a5500d4e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTrackableHandlerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTrackableHandlerTest.java @@ -321,15 +321,15 @@ public void clearEventsReferenceCount() { final Thread closeThread = new Thread(sink::close, "pipe-sink-close"); closeThread.setDaemon(true); - callbackThread.start(); - Assert.assertTrue(callbackEntered.await(5, TimeUnit.SECONDS)); - closeThread.start(); - Assert.assertTrue(sink.closeEliminationStarted.await(5, TimeUnit.SECONDS)); - try { + callbackThread.start(); + Assert.assertTrue(callbackEntered.await(30, TimeUnit.SECONDS)); + closeThread.start(); + Assert.assertTrue(sink.closeEliminationStarted.await(30, TimeUnit.SECONDS)); + allowCallbackToFinish.countDown(); - callbackThread.join(TimeUnit.SECONDS.toMillis(5)); - closeThread.join(TimeUnit.SECONDS.toMillis(5)); + callbackThread.join(TimeUnit.SECONDS.toMillis(30)); + closeThread.join(TimeUnit.SECONDS.toMillis(30)); Assert.assertFalse(callbackThread.isAlive()); Assert.assertFalse(closeThread.isAlive()); Assert.assertTrue(sink.isClosed()); @@ -337,6 +337,8 @@ public void clearEventsReferenceCount() { allowCallbackToFinish.countDown(); callbackThread.interrupt(); closeThread.interrupt(); + callbackThread.join(TimeUnit.SECONDS.toMillis(30)); + closeThread.join(TimeUnit.SECONDS.toMillis(30)); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/DisruptorShutdownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/DisruptorShutdownTest.java index ef57ea625aef6..7d18ddf7d3f7b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/DisruptorShutdownTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/DisruptorShutdownTest.java @@ -45,13 +45,15 @@ public void testBatchEventProcessorDrainsPublishedEventsOnShutdownInterrupt() th (event, sequence, endOfBatch) -> handledEventCount.incrementAndGet()); final Thread processorThread = new Thread(processor, "pipe-batch-event-processor-test"); - processorThread.start(); - - Assert.assertTrue(barrier.awaitWaitForCall()); - processor.halt(); - barrier.interruptWait(); - - processorThread.join(TimeUnit.SECONDS.toMillis(5)); + processorThread.setDaemon(true); + try { + processorThread.start(); + Assert.assertTrue(barrier.awaitWaitForCall()); + } finally { + processor.halt(); + barrier.interruptWait(); + processorThread.join(TimeUnit.SECONDS.toMillis(30)); + } Assert.assertFalse(processorThread.isAlive()); Assert.assertEquals(1, handledEventCount.get()); @@ -83,12 +85,18 @@ public void testBatchEventProcessorDrainsEventsPublishedAfterCurrentBatchWhenHal final Thread processorThread = new Thread(processor, "pipe-batch-event-processor-snapshot-test"); - processorThread.start(); - processorThread.join(TimeUnit.SECONDS.toMillis(5)); - - Assert.assertFalse(processorThread.isAlive()); - Assert.assertEquals(2, handledEventCount.get()); - Assert.assertEquals(1L, processor.getSequence().get()); + processorThread.setDaemon(true); + try { + processorThread.start(); + processorThread.join(TimeUnit.SECONDS.toMillis(30)); + Assert.assertFalse(processorThread.isAlive()); + Assert.assertEquals(2, handledEventCount.get()); + Assert.assertEquals(1L, processor.getSequence().get()); + } finally { + processor.halt(); + processorThread.interrupt(); + processorThread.join(TimeUnit.SECONDS.toMillis(30)); + } } @Test @@ -97,6 +105,7 @@ public void testDisruptorShutdownInterruptsWaitingProcessor() throws Exception { final ThreadFactory threadFactory = runnable -> { final Thread thread = new Thread(runnable, "pipe-disruptor-shutdown-test"); + thread.setDaemon(true); processorThreadReference.set(thread); return thread; }; @@ -106,10 +115,12 @@ public void testDisruptorShutdownInterruptsWaitingProcessor() throws Exception { disruptor.start(); final Thread processorThread = processorThreadReference.get(); - Assert.assertNotNull(processorThread); - - TimeUnit.MILLISECONDS.sleep(50); - disruptor.shutdown(); + try { + Assert.assertNotNull(processorThread); + TimeUnit.MILLISECONDS.sleep(50); + } finally { + disruptor.shutdown(); + } Assert.assertFalse(processorThread.isAlive()); } @@ -120,6 +131,7 @@ public void testUnexpectedInterruptDoesNotStopProcessor() throws Exception { final ThreadFactory threadFactory = runnable -> { final Thread thread = new Thread(runnable, "pipe-disruptor-unexpected-interrupt-test"); + thread.setDaemon(true); processorThreadReference.set(thread); return thread; }; @@ -130,16 +142,17 @@ public void testUnexpectedInterruptDoesNotStopProcessor() throws Exception { disruptor.handleEventsWith((event, sequence, endOfBatch) -> handled.countDown()).start(); final Thread processorThread = processorThreadReference.get(); - Assert.assertNotNull(processorThread); - - TimeUnit.MILLISECONDS.sleep(50); - processorThread.interrupt(); - - ringBuffer.publishEvent((event, sequence, value) -> event.value = value, 1); - Assert.assertTrue(handled.await(5, TimeUnit.SECONDS)); - Assert.assertTrue(processorThread.isAlive()); - - disruptor.shutdown(); + try { + Assert.assertNotNull(processorThread); + TimeUnit.MILLISECONDS.sleep(50); + processorThread.interrupt(); + + ringBuffer.publishEvent((event, sequence, value) -> event.value = value, 1); + Assert.assertTrue(handled.await(30, TimeUnit.SECONDS)); + Assert.assertTrue(processorThread.isAlive()); + } finally { + disruptor.shutdown(); + } Assert.assertFalse(processorThread.isAlive()); } @@ -160,10 +173,14 @@ public void testPublishEventCanAbortWhenClosingWhileBufferIsFull() throws Except (event, sequence, value) -> event.value = value, 2, isClosed::get)), "pipe-disruptor-publish-abort-test"); - publisherThread.start(); - TimeUnit.MILLISECONDS.sleep(50); - isClosed.set(true); - publisherThread.join(TimeUnit.SECONDS.toMillis(5)); + publisherThread.setDaemon(true); + try { + publisherThread.start(); + TimeUnit.MILLISECONDS.sleep(50); + } finally { + isClosed.set(true); + publisherThread.join(TimeUnit.SECONDS.toMillis(30)); + } Assert.assertFalse(publisherThread.isAlive()); Assert.assertFalse(published.get()); @@ -202,7 +219,7 @@ public long getHighestPublishedSequence(final long lowerBound, final long availa } private boolean awaitWaitForCall() throws InterruptedException { - return waitForCalled.await(5, TimeUnit.SECONDS); + return waitForCalled.await(30, TimeUnit.SECONDS); } private void interruptWait() {