Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@
public abstract class AbstractEnv implements BaseEnv {
private static final Logger logger = IoTDBTestLogger.logger;
private static final int DEFAULT_CLUSTER_READY_RETRY_COUNT = 30;
// Shutdown hooks may wait for Pipe events, progress persistence, and Ratis snapshots.
private static final int DATA_NODE_SHUTDOWN_TIMEOUT_SECONDS = 60;
private static final long READINESS_RETRY_INTERVAL_NANOS = TimeUnit.MILLISECONDS.toNanos(100);
private static final String CLUSTER_READY_RETRY_COUNT_PROPERTY =
"integrationTest.clusterReadyRetryCount";
Expand Down Expand Up @@ -1660,12 +1662,14 @@ public void startAllDataNodes() {

@Override
public void shutdownDataNode(final int index) {
dataNodeWrapperList.get(index).stop();
dataNodeWrapperList.get(index).stop(DATA_NODE_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS);
}

@Override
public void shutdownAllDataNodes() {
dataNodeWrapperList.forEach(AbstractNodeWrapper::stop);
dataNodeWrapperList.forEach(
dataNodeWrapper ->
dataNodeWrapper.stop(DATA_NODE_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -556,13 +556,17 @@ public void start() {

@Override
public void stop() {
stop(20, TimeUnit.SECONDS);
}

public void stop(final long timeout, final TimeUnit unit) {
if (this.instance == null) {
return;
}
this.instance.destroy();
try {
if (!this.instance.waitFor(20, TimeUnit.SECONDS)) {
logger.warn("Node {} does not exit within 20s, killing it", getId());
if (!this.instance.waitFor(timeout, unit)) {
logger.warn("Node {} does not exit within {} {}, killing it", getId(), timeout, unit);
if (!this.instance.destroyForcibly().waitFor(10, TimeUnit.SECONDS)) {
logger.error("Cannot forcibly stop node {}", getId());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -289,11 +289,11 @@ public void testReadOnlyDataNode() throws Exception {
public void testStoppedDataNode() throws Exception {
// Shutdown a DataNode, the ConfigNode should still be able to create RegionGroup
NodeStatus stoppedStatus = EnvUtils.getNodeStatusAfterLocalStop();
DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(testDataNodeId);
EnvFactory.getEnv().shutdownDataNode(testDataNodeId);
EnvFactory.getEnv()
.ensureNodeStatus(
Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(testDataNodeId)),
Collections.singletonList(stoppedStatus));
Collections.singletonList(dataNodeWrapper), Collections.singletonList(stoppedStatus));

try (SyncConfigNodeIServiceClient client =
(SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ public void tryIfTheServerIsRestart() throws InterruptedException {
BaseNodeWrapper node = EnvFactory.getEnv().getDataNodeWrapper(0);
try {
wrapper = pool.executeQueryStatement("select * from " + database + ".d1 where time > 1");
node.stop();
EnvFactory.getEnv().shutdownDataNode(0);
EnvFactory.getEnv()
.ensureNodeStatus(
// A gracefully stopped DataNode is reported as Stopped by its shutdown hook (on
Expand All @@ -304,7 +304,7 @@ public void tryIfTheServerIsRestart() throws InterruptedException {
} catch (IoTDBConnectionException e) {
pool.closeResultSet(wrapper);
pool.close();
node.stop();
EnvFactory.getEnv().shutdownDataNode(0);
EnvFactory.getEnv()
.ensureNodeStatus(
// A gracefully stopped DataNode is reported as Stopped by its shutdown hook (on
Expand Down Expand Up @@ -333,7 +333,7 @@ public void tryIfTheServerIsRestart() throws InterruptedException {
} catch (IoTDBConnectionException ec) {
pool.closeResultSet(wrapper);
pool.close();
node.stop();
EnvFactory.getEnv().shutdownDataNode(0);
EnvFactory.getEnv()
.ensureNodeStatus(
// A gracefully stopped DataNode is reported as Stopped by its shutdown hook (on
Expand Down Expand Up @@ -394,7 +394,7 @@ public void restart() throws InterruptedException {
// stop the server.
pool.close();
BaseNodeWrapper node = EnvFactory.getEnv().getDataNodeWrapper(0);
node.stop();
EnvFactory.getEnv().shutdownDataNode(0);
EnvFactory.getEnv()
.ensureNodeStatus(
// A gracefully stopped DataNode is reported as Stopped by its shutdown hook (on
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ public void testSinkCriticalExceptionIsPropagatedOnlyWithinItsPipe() throws Exce
taskAgent.stopAllPipesWithCriticalExceptionAndTrackException(failedTaskMeta, exception);

Awaitility.await()
.atMost(5, TimeUnit.SECONDS)
.atMost(30, TimeUnit.SECONDS)
.until(
() ->
PipeStatus.STOPPED.equals(failedPipeMeta.getRuntimeMeta().getStatus().get())
Expand Down Expand Up @@ -462,7 +462,7 @@ public void testDetachedTaskMetaMustIdentifyOnePipeUniquely() throws Exception {
taskAgent.stopAllPipesWithCriticalExceptionAndTrackException(
detachedTaskMeta, new PipeRuntimeCriticalException("ambiguous task meta", 1L));

Awaitility.await().atMost(2, TimeUnit.SECONDS).until(detachedTaskMeta::hasExceptionMessages);
Awaitility.await().atMost(30, TimeUnit.SECONDS).until(detachedTaskMeta::hasExceptionMessages);
pipeMetaKeeper.acquireWriteLock();
pipeMetaKeeper.releaseWriteLock();

Expand Down Expand Up @@ -499,7 +499,7 @@ public void testDetachedTaskMetaIsRecordedOnIdentifiedPipe() throws Exception {
"detachedPipe", 2L, detachedTaskMeta, exception);

Awaitility.await()
.atMost(5, TimeUnit.SECONDS)
.atMost(30, TimeUnit.SECONDS)
.until(
() ->
PipeStatus.STOPPED.equals(pipeMeta.getRuntimeMeta().getStatus().get())
Expand Down Expand Up @@ -533,7 +533,7 @@ public void testCriticalExceptionWithNullTaskMetaStopsIdentifiedPipe() throws Ex
"nullTaskMetaPipe", 3L, null, exception);

Awaitility.await()
.atMost(5, TimeUnit.SECONDS)
.atMost(30, TimeUnit.SECONDS)
.until(
() ->
PipeStatus.STOPPED.equals(failedPipeMeta.getRuntimeMeta().getStatus().get())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ protected boolean executeOnce() {
if (current == 1) {
firstCallEntered.countDown();
try {
firstCallReleased.await(5, TimeUnit.SECONDS);
firstCallReleased.await();
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
}
Expand All @@ -171,13 +171,14 @@ protected boolean executeOnce() {
};

executor.register(blockedSubtask);
executor.start(blockedSubtask.getTaskID());
Assert.assertTrue(firstCallEntered.await(5, TimeUnit.SECONDS));

executor.stop(blockedSubtask.getTaskID());
executor.start(blockedSubtask.getTaskID());

try {
executor.start(blockedSubtask.getTaskID());
Assert.assertTrue(firstCallEntered.await(30, TimeUnit.SECONDS));

executor.stop(blockedSubtask.getTaskID());
executor.start(blockedSubtask.getTaskID());

Assert.assertFalse(
"The subtask must not be executed in parallel with itself",
secondCallEntered.await(1, TimeUnit.SECONDS));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ public void testParserAdmissionYieldsWithoutBlockingAndResumes() throws Exceptio
}
}

@Test(timeout = 60000)
@Test(timeout = 120000)
public void testCloseCancelsReservationEnqueuedByAWorkerInvocation() throws Exception {
final CommonConfig config = CommonDescriptor.getInstance().getConfig();
final PipeMemoryManager manager = PipeDataNodeResourceManager.memory();
Expand Down Expand Up @@ -226,7 +226,7 @@ public void testCloseCancelsReservationEnqueuedByAWorkerInvocation() throws Exce
eventToClose.close();
});
await()
.atMost(5, TimeUnit.SECONDS)
.atMost(30, TimeUnit.SECONDS)
.until(
() ->
closingThread.get() != null
Expand All @@ -241,27 +241,39 @@ public void testCloseCancelsReservationEnqueuedByAWorkerInvocation() throws Exce
Assert.assertEquals(1, manager.getWaitingTsFileParserCount());
Assert.assertTrue(outputQueue.isEmpty());
}
closing.get(5, TimeUnit.SECONDS);
closing.get(30, TimeUnit.SECONDS);
Assert.assertFalse(event.isWaitingForTsFileParserMemory());
Assert.assertEquals(0, manager.getWaitingTsFileParserCount());
Assert.assertEquals(1, manager.getReservedTsFileParserCount());
} finally {
executor.shutdownNow();
Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
if (subtask != null) {
subtask.disallowSubmittingSelf();
subtask.close();
}
if (event != null) {
event.close();
}
if (blockerReserved) {
manager.releaseTsFileParserMemory("blocker", 0, "0");
try {
Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS));
} finally {
try {
if (subtask != null) {
subtask.disallowSubmittingSelf();
subtask.close();
}
} finally {
try {
if (event != null) {
event.close();
}
} finally {
try {
if (blockerReserved) {
manager.releaseTsFileParserMemory("blocker", 0, "0");
}
} finally {
config.setPipeTsFileParserMemory(originalParserMemory);
config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit);
config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit);
FileUtils.deleteFileOrDirectory(tempDir);
}
}
}
}
config.setPipeTsFileParserMemory(originalParserMemory);
config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit);
config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit);
FileUtils.deleteFileOrDirectory(tempDir);
}
}

Expand Down
Loading
Loading