Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@
public abstract class AbstractEnv implements BaseEnv {
private static final Logger logger = IoTDBTestLogger.logger;
private static final int DEFAULT_CLUSTER_READY_RETRY_COUNT = 30;
// Shutdown hooks may wait for Pipe events, progress persistence, and Ratis snapshots.
private static final int DATA_NODE_SHUTDOWN_TIMEOUT_SECONDS = 60;
private static final String CLUSTER_READY_RETRY_COUNT_PROPERTY =
"integrationTest.clusterReadyRetryCount";

Expand Down Expand Up @@ -1319,12 +1321,14 @@ public void startAllDataNodes() {

@Override
public void shutdownDataNode(final int index) {
dataNodeWrapperList.get(index).stop();
dataNodeWrapperList.get(index).stop(DATA_NODE_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS);
}

@Override
public void shutdownAllDataNodes() {
dataNodeWrapperList.forEach(AbstractNodeWrapper::stop);
dataNodeWrapperList.forEach(
dataNodeWrapper ->
dataNodeWrapper.stop(DATA_NODE_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -488,13 +488,20 @@ public void start() {

@Override
public void stop() {
stop(20, TimeUnit.SECONDS);
}

public void stop(final long timeout, final TimeUnit unit) {
if (this.instance == null) {
return;
}
this.instance.destroy();
try {
if (!this.instance.waitFor(20, TimeUnit.SECONDS)) {
this.instance.destroyForcibly().waitFor(10, TimeUnit.SECONDS);
if (!this.instance.waitFor(timeout, unit)) {
logger.warn("Node {} does not exit within {} {}, killing it", getId(), timeout, unit);
if (!this.instance.destroyForcibly().waitFor(10, TimeUnit.SECONDS)) {
logger.error("Cannot forcibly stop node {}", getId());
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -274,10 +274,11 @@ public void testReadOnlyDataNode() throws Exception {
@Test
public void testUnknownDataNode() throws Exception {
// Shutdown a DataNode, the ConfigNode should still be able to create RegionGroup
DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(testDataNodeId);
EnvFactory.getEnv().shutdownDataNode(testDataNodeId);
EnvFactory.getEnv()
.ensureNodeStatus(
Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(testDataNodeId)),
Collections.singletonList(dataNodeWrapper),
Collections.singletonList(NodeStatus.Unknown));

try (SyncConfigNodeIServiceClient client =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ public void tryIfTheServerIsRestart() {
BaseNodeWrapper node = EnvFactory.getEnv().getDataNodeWrapper(0);
try {
wrapper = pool.executeQueryStatement("select * from root.sg1.d1 where time > 1");
node.stop();
EnvFactory.getEnv().shutdownDataNode(0);
EnvFactory.getEnv()
.ensureNodeStatus(
Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown));
Expand All @@ -259,7 +259,7 @@ public void tryIfTheServerIsRestart() {
} catch (IoTDBConnectionException e) {
pool.closeResultSet(wrapper);
pool.close();
node.stop();
EnvFactory.getEnv().shutdownDataNode(0);
EnvFactory.getEnv()
.ensureNodeStatus(
Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown));
Expand All @@ -285,7 +285,7 @@ public void tryIfTheServerIsRestart() {
} catch (IoTDBConnectionException ec) {
pool.closeResultSet(wrapper);
pool.close();
node.stop();
EnvFactory.getEnv().shutdownDataNode(0);
EnvFactory.getEnv()
.ensureNodeStatus(
Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown));
Expand Down Expand Up @@ -341,7 +341,7 @@ public void restart() {
// stop the server.
pool.close();
BaseNodeWrapper node = EnvFactory.getEnv().getDataNodeWrapper(0);
node.stop();
EnvFactory.getEnv().shutdownDataNode(0);
EnvFactory.getEnv()
.ensureNodeStatus(
Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ public void testSinkCriticalExceptionIsPropagatedOnlyWithinItsPipe() throws Exce
taskAgent.stopAllPipesWithCriticalExceptionAndTrackException(failedTaskMeta, exception);

Awaitility.await()
.atMost(5, TimeUnit.SECONDS)
.atMost(30, TimeUnit.SECONDS)
.until(
() ->
PipeStatus.STOPPED.equals(failedPipeMeta.getRuntimeMeta().getStatus().get())
Expand Down Expand Up @@ -249,7 +249,7 @@ public void testDetachedTaskMetaMustIdentifyOnePipeUniquely() throws Exception {
taskAgent.stopAllPipesWithCriticalExceptionAndTrackException(
detachedTaskMeta, new PipeRuntimeCriticalException("ambiguous task meta", 1L));

Awaitility.await().atMost(2, TimeUnit.SECONDS).until(detachedTaskMeta::hasExceptionMessages);
Awaitility.await().atMost(30, TimeUnit.SECONDS).until(detachedTaskMeta::hasExceptionMessages);
pipeMetaKeeper.acquireWriteLock();
pipeMetaKeeper.releaseWriteLock();

Expand Down Expand Up @@ -286,7 +286,7 @@ public void testDetachedTaskMetaIsRecordedOnIdentifiedPipe() throws Exception {
"detachedPipe", 2L, detachedTaskMeta, exception);

Awaitility.await()
.atMost(5, TimeUnit.SECONDS)
.atMost(30, TimeUnit.SECONDS)
.until(
() ->
PipeStatus.STOPPED.equals(pipeMeta.getRuntimeMeta().getStatus().get())
Expand Down Expand Up @@ -320,7 +320,7 @@ public void testCriticalExceptionWithNullTaskMetaStopsIdentifiedPipe() throws Ex
"nullTaskMetaPipe", 3L, null, exception);

Awaitility.await()
.atMost(5, TimeUnit.SECONDS)
.atMost(30, TimeUnit.SECONDS)
.until(
() ->
PipeStatus.STOPPED.equals(failedPipeMeta.getRuntimeMeta().getStatus().get())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ public void testParserAdmissionYieldsWithoutBlockingAndResumes() throws Exceptio
}
}

@Test(timeout = 60000)
@Test(timeout = 120000)
public void testCloseCancelsReservationEnqueuedByAWorkerInvocation() throws Exception {
final CommonConfig config = CommonDescriptor.getInstance().getConfig();
final PipeMemoryManager manager = PipeDataNodeResourceManager.memory();
Expand Down Expand Up @@ -225,7 +225,7 @@ public void testCloseCancelsReservationEnqueuedByAWorkerInvocation() throws Exce
eventToClose.close();
});
await()
.atMost(5, TimeUnit.SECONDS)
.atMost(30, TimeUnit.SECONDS)
.until(
() ->
closingThread.get() != null
Expand All @@ -240,27 +240,39 @@ public void testCloseCancelsReservationEnqueuedByAWorkerInvocation() throws Exce
Assert.assertEquals(1, manager.getWaitingTsFileParserCount());
Assert.assertTrue(outputQueue.isEmpty());
}
closing.get(5, TimeUnit.SECONDS);
closing.get(30, TimeUnit.SECONDS);
Assert.assertFalse(event.isWaitingForTsFileParserMemory());
Assert.assertEquals(0, manager.getWaitingTsFileParserCount());
Assert.assertEquals(1, manager.getReservedTsFileParserCount());
} finally {
executor.shutdownNow();
Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
if (subtask != null) {
subtask.disallowSubmittingSelf();
subtask.close();
}
if (event != null) {
event.close();
}
if (blockerReserved) {
manager.releaseTsFileParserMemory("blocker", 0, "0");
try {
Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS));
} finally {
try {
if (subtask != null) {
subtask.disallowSubmittingSelf();
subtask.close();
}
} finally {
try {
if (event != null) {
event.close();
}
} finally {
try {
if (blockerReserved) {
manager.releaseTsFileParserMemory("blocker", 0, "0");
}
} finally {
config.setPipeTsFileParserMemory(originalParserMemory);
config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit);
config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit);
FileUtils.deleteFileOrDirectory(tempDir);
}
}
}
}
config.setPipeTsFileParserMemory(originalParserMemory);
config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit);
config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit);
FileUtils.deleteFileOrDirectory(tempDir);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ public void testRequestsOnOneWorkerDoNotStrandOtherWorkersParserSlots() throws E
}
}

@Test(timeout = 30000)
@Test(timeout = 120000)
public void testOneHundredWaitingPipesDoNotWaitForWorkerPolling() throws Exception {
final CommonConfig config = CommonDescriptor.getInstance().getConfig();
final PipeMemoryManager memoryManager = PipeDataNodeResourceManager.memory();
Expand Down Expand Up @@ -232,8 +232,8 @@ public void testOneHundredWaitingPipesDoNotWaitForWorkerPolling() throws Excepti
final AtomicReference<Thread> workerThread = workerThreads.get(i);
// Consume the scheduling notification before all workers become idle on admission.
worker.sleepIfNecessary(true);
// Make a missed wake-up observable without relying on a tight performance assertion.
worker.setSleepingTimeInMilliSecond(60000L);
// Keep polling beyond the whole test timeout so it cannot hide a missed wake-up.
worker.setSleepingTimeInMilliSecond(180000L);
futures.add(
executor.submit(
() -> {
Expand All @@ -244,7 +244,7 @@ public void testOneHundredWaitingPipesDoNotWaitForWorkerPolling() throws Excepti
}));
}
await()
.atMost(5, TimeUnit.SECONDS)
.atMost(30, TimeUnit.SECONDS)
.until(
() ->
workerThreads.stream()
Expand All @@ -258,25 +258,34 @@ public void testOneHundredWaitingPipesDoNotWaitForWorkerPolling() throws Excepti
}
Assert.assertTrue(
"Admission notifications must wake every queue head",
processed.await(5, TimeUnit.SECONDS));
processed.await(30, TimeUnit.SECONDS));
Assert.assertEquals(0, memoryManager.getWaitingTsFileParserCount());
Assert.assertEquals(0, memoryManager.getReservedTsFileParserCount());
} finally {
futures.forEach(future -> future.cancel(true));
executor.shutdownNow();
Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
for (int i = 0; i < requests.size(); i++) {
memoryManager.cancelTsFileParserMemoryReservation(
"waitingPipe" + i, 0, "0", requests.get(i));
}
for (int i = 0; i < reservedBlockers.length; i++) {
if (reservedBlockers[i]) {
memoryManager.releaseTsFileParserMemory("blocker" + i, 0, "0");
try {
Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS));
} finally {
try {
for (int i = 0; i < requests.size(); i++) {
memoryManager.cancelTsFileParserMemoryReservation(
"waitingPipe" + i, 0, "0", requests.get(i));
}
} finally {
try {
for (int i = 0; i < reservedBlockers.length; i++) {
if (reservedBlockers[i]) {
memoryManager.releaseTsFileParserMemory("blocker" + i, 0, "0");
}
}
} finally {
config.setPipeTsFileParserMemory(originalParserMemory);
config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit);
config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit);
}
}
}
config.setPipeTsFileParserMemory(originalParserMemory);
config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit);
config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit);
}
}

Expand Down Expand Up @@ -333,7 +342,7 @@ public void testWaitingParserIsSkippedUntilAdmissionNotification() throws Except
}
}

@Test(timeout = 10000)
@Test(timeout = 90000)
public void testNotificationBeforeIdleWaitIsNotLost() throws Exception {
final PipeProcessorSubtaskWorker worker = new PipeProcessorSubtaskWorker(new LinkedHashSet<>());
worker.setSleepingTimeInMilliSecond(60000L);
Expand All @@ -351,14 +360,14 @@ public void testNotificationBeforeIdleWaitIsNotLost() throws Exception {
final ExecutorService executor = Executors.newSingleThreadExecutor();
try {
wakeUpCallback.get().run();
executor.submit(() -> worker.sleepIfNecessary(true)).get(3, TimeUnit.SECONDS);
executor.submit(() -> worker.sleepIfNecessary(true)).get(30, TimeUnit.SECONDS);
} finally {
executor.shutdownNow();
Assert.assertTrue(executor.awaitTermination(3, TimeUnit.SECONDS));
Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS));
}
}

@Test(timeout = 15000)
@Test(timeout = 150000)
public void testBusyFirstWorkerDoesNotSerializeOtherEligibleWorker() throws Exception {
final CommonConfig config = CommonDescriptor.getInstance().getConfig();
final PipeMemoryManager manager = PipeDataNodeResourceManager.memory();
Expand Down Expand Up @@ -394,7 +403,7 @@ public void testBusyFirstWorkerDoesNotSerializeOtherEligibleWorker() throws Exce
ignored -> {
if (blockFirstWorker.get()) {
firstWorkerBusy.countDown();
Assert.assertTrue(unblockFirstWorker.await(5, TimeUnit.SECONDS));
unblockFirstWorker.await();
}
return false;
});
Expand Down Expand Up @@ -448,7 +457,7 @@ public void testBusyFirstWorkerDoesNotSerializeOtherEligibleWorker() throws Exce
Assert.assertTrue(secondWorker.runSubtasks());
blockFirstWorker.set(true);
final Future<?> busyRound = executor.submit(firstWorker::runSubtasks);
Assert.assertTrue(firstWorkerBusy.await(3, TimeUnit.SECONDS));
Assert.assertTrue(firstWorkerBusy.await(30, TimeUnit.SECONDS));
for (int i = 0; i < reservedBlockers.length; i++) {
manager.releaseTsFileParserMemory("blocker" + i, 0, "0");
reservedBlockers[i] = false;
Expand All @@ -459,27 +468,36 @@ public void testBusyFirstWorkerDoesNotSerializeOtherEligibleWorker() throws Exce
secondWorker.sleepIfNecessary(secondWorker.runSubtasks());
}
});
Assert.assertTrue(secondAdmitted.await(3, TimeUnit.SECONDS));
Assert.assertTrue(secondAdmitted.await(30, TimeUnit.SECONDS));
verify(firstSubtask, times(1)).call();
Assert.assertFalse(busyRound.isDone());
unblockFirstWorker.countDown();
busyRound.get(3, TimeUnit.SECONDS);
busyRound.get(30, TimeUnit.SECONDS);
Assert.assertEquals(0, manager.getWaitingTsFileParserCount());
Assert.assertEquals(0, manager.getReservedTsFileParserCount());
} finally {
unblockFirstWorker.countDown();
executor.shutdownNow();
Assert.assertTrue(executor.awaitTermination(3, TimeUnit.SECONDS));
manager.cancelTsFileParserMemoryReservation("first", 0, "0", first);
manager.cancelTsFileParserMemoryReservation("second", 0, "0", second);
for (int i = 0; i < reservedBlockers.length; i++) {
if (reservedBlockers[i]) {
manager.releaseTsFileParserMemory("blocker" + i, 0, "0");
try {
Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS));
} finally {
try {
manager.cancelTsFileParserMemoryReservation("first", 0, "0", first);
manager.cancelTsFileParserMemoryReservation("second", 0, "0", second);
} finally {
try {
for (int i = 0; i < reservedBlockers.length; i++) {
if (reservedBlockers[i]) {
manager.releaseTsFileParserMemory("blocker" + i, 0, "0");
}
}
} finally {
config.setPipeTsFileParserMemory(originalParserMemory);
config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit);
config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit);
}
}
}
config.setPipeTsFileParserMemory(originalParserMemory);
config.setPipeTsFileParserInFlightMaxNum(originalGlobalLimit);
config.setPipeTsFileParserInFlightMaxNumPerPipeRegion(originalRegionLimit);
}
}

Expand Down
Loading
Loading