Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions integration-test/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,18 @@ test; it may be omitted for the default `SimpleIT` profile.
- Report the tested revision, edition, profile, and result accurately. Passing an upstream build
does not establish that the corresponding downstream edition was tested.

### Startup diagnostics

- ConfigNode leader discovery, cluster readiness, and JDBC readiness retry every 100ms. The existing
retry count still determines a time budget in seconds (leader discovery allows that budget per
ConfigNode); socket/RPC timeouts remain separate. Cluster status checks share their deadline with
leader discovery rather than restarting a nested wait.
- Add `-DintegrationTest.configNodeStartupTrace=true` to a targeted IT command to record JVM startup
and class initialization before ConfigNode's first application log. The default is disabled to
avoid tracing overhead. Each ConfigNode writes `<node-log>.startup-<pid>.log` alongside its normal
log. JVM startup events include nested phases; class initialization events mark their start and
do not measure exclusive class execution time.

### Node templates and IDE runs

Build the node templates before an IDE run or a direct Failsafe invocation, and rebuild them after
Expand Down
5 changes: 5 additions & 0 deletions integration-test/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,11 @@
</dependencies>
<build>
<plugins>
<!-- Compile readiness diagnostics in the selected locale, using the parent source mapping. -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
</plugin>
<!-- skip default-test -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.it.env.cluster.env;

/** Localized diagnostics for the integration-test readiness deadlines. */
public final class IntegrationTestEnvMessages {

private IntegrationTestEnvMessages() {}

public static final String EXCEPTION_CLUSTER_STATUS_CHECK_FAILED_RETRY_BUDGET_ARG_SECONDS_AEB4010E =
"Cluster status check failed (retry budget: %d seconds)";

public static final String EXCEPTION_JDBC_CONNECTIONS_TO_DATANODES_ARE_NOT_READY_RETRY_BUDGET_ARG_SECONDS_ENDPOINTS_ARG_LASTCONNECTIONFAILURES_ARG_LOGDIRS_ARG_D822C35C =
"JDBC connections to DataNodes are not ready (retry budget: %d seconds). endpoints=%s, lastConnectionFailures=%s, logDirs=%s";

public static final String EXCEPTION_FAILED_TO_GET_CONNECTION_TO_CONFIGNODE_ARG_DF03F50F =
"Failed to get connection to ConfigNode %s";
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.it.env.cluster.env;

/** Localized diagnostics for the integration-test readiness deadlines. */
public final class IntegrationTestEnvMessages {

private IntegrationTestEnvMessages() {}

public static final String EXCEPTION_CLUSTER_STATUS_CHECK_FAILED_RETRY_BUDGET_ARG_SECONDS_AEB4010E =
"集群状态检查失败(重试时间预算:%d 秒)";

public static final String EXCEPTION_JDBC_CONNECTIONS_TO_DATANODES_ARE_NOT_READY_RETRY_BUDGET_ARG_SECONDS_ENDPOINTS_ARG_LASTCONNECTIONFAILURES_ARG_LOGDIRS_ARG_D822C35C =
"连接 DataNode 的 JDBC 连接尚未就绪(重试时间预算:%d 秒)。endpoints=%s, lastConnectionFailures=%s, logDirs=%s";

public static final String EXCEPTION_FAILED_TO_GET_CONNECTION_TO_CONFIGNODE_ARG_DF03F50F =
"无法连接 ConfigNode %s";
}
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,12 @@ public DataNodeConfig setMqttPayloadFormatter(String mqttPayloadFormatter) {
return this;
}

@Override
public DataNodeConfig setMqttHandlerPoolSize(int mqttHandlerPoolSize) {
setProperty("mqtt_handler_pool_size", String.valueOf(mqttHandlerPoolSize));
return this;
}

@Override
public DataNodeConfig setLoadLastCacheStrategy(String strategyName) {
setProperty("last_cache_operation_on_load", strategyName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@
public abstract class AbstractEnv implements BaseEnv {
private static final Logger logger = IoTDBTestLogger.logger;
private static final int DEFAULT_CLUSTER_READY_RETRY_COUNT = 30;
private static final long READINESS_RETRY_INTERVAL_NANOS = TimeUnit.MILLISECONDS.toNanos(100);
private static final String CLUSTER_READY_RETRY_COUNT_PROPERTY =
"integrationTest.clusterReadyRetryCount";

Expand Down Expand Up @@ -380,8 +381,8 @@ public void checkClusterStatusOneUnknownOtherRunning() {
}

/**
* check whether all nodes' status match the provided predicate with RPC. after retryCount times,
* if the status of all nodes still not match the predicate, throw AssertionError.
* check whether all nodes' status match the provided predicate with RPC. after retryCount
* seconds, if the status of all nodes still not match the predicate, throw AssertionError.
*
* @param nodeStatusCheck the predicate to test the status of nodes
*/
Expand All @@ -400,10 +401,14 @@ public void checkClusterStatus(
int actualNodeSize = 0;
Map<Integer, String> lastNodeStatus = null;
Map<AbstractNodeWrapper, Integer> processStatusMap = new HashMap<>();
// retryCount historically allowed one second per retry. Preserve that time budget while
// checking more frequently, and share the deadline with leader discovery to avoid nested waits.
final long deadlineNanos = readinessDeadline(retryCount);

for (int i = 0; i < retryCount; i++) {
int attempt = 0;
do {
try (final SyncConfigNodeIServiceClient client =
(SyncConfigNodeIServiceClient) getLeaderConfigNodeConnection()) {
getLeaderConfigNodeConnection(deadlineNanos)) {
passed = true;
showClusterPassed = true;
nodeSizePassed = true;
Expand Down Expand Up @@ -478,23 +483,33 @@ public void checkClusterStatus(
logger.info("The cluster is now ready for testing!");
return;
}
logger.info(
"Retry {}: showClusterPassed={}, nodeSizePassed={}, nodeStatusPassed={}, processStatusPassed={}",
i,
showClusterPassed,
nodeSizePassed,
nodeStatusPassed,
processStatusPassed);
if (attempt % 10 == 0) {
logger.info(
"Retry {}: showClusterPassed={}, nodeSizePassed={}, nodeStatusPassed={}, processStatusPassed={}",
attempt,
showClusterPassed,
nodeSizePassed,
nodeStatusPassed,
processStatusPassed);
}
} catch (final InterruptedException e) {
lastException = e;
Thread.currentThread().interrupt();
break;
} catch (final Exception e) {
lastException = e;
}
attempt++;
try {
TimeUnit.SECONDS.sleep(1L);
if (!waitForNextReadinessCheck(deadlineNanos)) {
break;
}
} catch (final InterruptedException e) {
lastException = e;
Thread.currentThread().interrupt();
break;
}
}
} while (true);
if (lastException != null) {
logger.error(
"exception in test Cluster with RPC, message: {}",
Expand Down Expand Up @@ -547,7 +562,10 @@ private String buildClusterStatusFailureMessage(
final Exception lastException) {
final StringBuilder builder =
new StringBuilder(
String.format("After %d times retry, the cluster status check failed", retryCount));
String.format(
IntegrationTestEnvMessages
.EXCEPTION_CLUSTER_STATUS_CHECK_FAILED_RETRY_BUDGET_ARG_SECONDS_AEB4010E,
retryCount));
builder
.append(": showClusterPassed=")
.append(showClusterPassed)
Expand Down Expand Up @@ -1164,7 +1182,7 @@ protected List<NodeConnection> getReadConnections(
// because it is hard to add retry and handle exception when getting jdbc connections in
// getWriteConnectionWithSpecifiedDataNode and getReadConnections.
// so use this function to add retry when cluster is ready.
// after retryCount times, if the jdbc can't connect, throw
// after retryCount seconds, if the jdbc can't connect, throw
// AssertionError.
protected void testJDBCConnection() {
logger.info("Testing JDBC connection...");
Expand All @@ -1180,26 +1198,26 @@ protected void testJDBCConnection() {
testDelegate.addRequest(
() -> {
Exception lastException = null;
for (int i = 0; i < retryCount; i++) {
final long deadlineNanos = readinessDeadline(retryCount);
do {
try (final IoTDBConnection ignored =
(IoTDBConnection)
DriverManager.getConnection(
Config.IOTDB_URL_PREFIX
+ dataNodeEndpoint
+ getParam(null, NODE_NETWORK_TIMEOUT_MS, ZERO_TIME_ZONE),
constructConnectionProperties(
System.getProperty("User", "root"),
System.getProperty("Password", "root"),
System.getProperty("User", SessionConfig.DEFAULT_USER),
System.getProperty("Password", SessionConfig.DEFAULT_PASSWORD),
TREE_SQL_DIALECT))) {
logger.info("Successfully connecting to DataNode: {}.", dataNodeEndpoint);
return null;
} catch (final Exception e) {
lastException = e;
lastConnectionFailures.put(
dataNodeEndpoint, e.getClass().getName() + ": " + e.getMessage());
TimeUnit.SECONDS.sleep(1L);
}
}
} while (waitForNextReadinessCheck(deadlineNanos));
if (lastException != null) {
throw lastException;
}
Expand All @@ -1213,8 +1231,12 @@ protected void testJDBCConnection() {
dumpTestJVMSnapshotQuietly("JDBC connection check failed");
throw new AssertionError(
String.format(
"After %d times retry, JDBC connections to DataNodes are not ready. endpoints=%s, lastConnectionFailures=%s, logDirs=%s",
retryCount, endpoints, lastConnectionFailures, getClusterLogDirs()));
IntegrationTestEnvMessages
.EXCEPTION_JDBC_CONNECTIONS_TO_DATANODES_ARE_NOT_READY_RETRY_BUDGET_ARG_SECONDS_ENDPOINTS_ARG_LASTCONNECTIONFAILURES_ARG_LOGDIRS_ARG_D822C35C,
retryCount,
endpoints,
lastConnectionFailures,
getClusterLogDirs()));
}
}

Expand Down Expand Up @@ -1287,28 +1309,39 @@ public List<DataNodeWrapper> getDataNodeWrapperList() {
@Override
public IConfigNodeRPCService.Iface getLeaderConfigNodeConnection()
throws IOException, InterruptedException {
// Previously each ConfigNode incurred a one-second delay in every retry round.
return getLeaderConfigNodeConnection(
readinessDeadline((long) retryCount * Math.max(1, configNodeWrapperList.size())));
}

private SyncConfigNodeIServiceClient getLeaderConfigNodeConnection(final long deadlineNanos)
throws IOException, InterruptedException {
Exception lastException = null;
ConfigNodeWrapper lastErrorNode = null;
for (int i = 0; i < retryCount; i++) {
if (configNodeWrapperList.isEmpty()) {
throw new IOException("Empty configNode set");
}
do {
for (final ConfigNodeWrapper configNodeWrapper : configNodeWrapperList) {
lastErrorNode = configNodeWrapper;
SyncConfigNodeIServiceClient client = null;
boolean leaderFound = false;
try {
if (!configNodeWrapper.getInstance().isAlive()) {
throw new IOException("ConfigNode " + configNodeWrapper.getId() + " is not alive");
}

lastErrorNode = configNodeWrapper;
final SyncConfigNodeIServiceClient client =
client =
clientManager.borrowClient(
new TEndPoint(configNodeWrapper.getIp(), configNodeWrapper.getPort()));
final TShowClusterResp resp = client.showCluster();

if (resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
// Only the ConfigNodeClient who connects to the ConfigNode-leader
// will respond the SUCCESS_STATUS
leaderFound = true;
return client;
} else {
// Return client otherwise
client.close();
throw new Exception(
"Bad status: "
+ resp.getStatus().getCode()
Expand All @@ -1317,38 +1350,58 @@ public IConfigNodeRPCService.Iface getLeaderConfigNodeConnection()
}
} catch (final Exception e) {
lastException = e;
} finally {
// The caller owns the leader client; return every failed probe, including RPC failures.
if (!leaderFound && client != null) {
client.close();
}
}
if (deadlineNanos - System.nanoTime() <= 0) {
break;
}

// Sleep 1s before next retry
TimeUnit.SECONDS.sleep(1);
}
}
if (lastErrorNode != null) {
throw new IOException(
"Failed to get connection to ConfigNode-Leader. Last error configNode: "
+ lastErrorNode.getIpAndPortString(),
lastException);
} else {
throw new IOException("Empty configNode set");
}
// Probe all candidates before waiting, so a follower does not delay the next leader probe.
} while (waitForNextReadinessCheck(deadlineNanos));
throw new IOException(
"Failed to get connection to ConfigNode-Leader. Last error configNode: "
+ lastErrorNode.getIpAndPortString(),
lastException);
}

@Override
public IConfigNodeRPCService.Iface getConfigNodeConnection(int index) throws Exception {
Exception lastException = null;
final ConfigNodeWrapper configNodeWrapper = configNodeWrapperList.get(index);
for (int i = 0; i < 30; i++) {
final long deadlineNanos = readinessDeadline(30);
do {
try {
return clientManager.borrowClient(
new TEndPoint(configNodeWrapper.getIp(), configNodeWrapper.getPort()));
} catch (final Exception e) {
lastException = e;
}
// Sleep 1s before next retry
TimeUnit.SECONDS.sleep(1);
}
} while (waitForNextReadinessCheck(deadlineNanos));
throw new IOException(
"Failed to get connection to this ConfigNode. Last error: " + lastException);
String.format(
IntegrationTestEnvMessages
.EXCEPTION_FAILED_TO_GET_CONNECTION_TO_CONFIGNODE_ARG_DF03F50F,
configNodeWrapper.getIpAndPortString()),
lastException);
}

private static long readinessDeadline(final long timeoutSeconds) {
return System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds);
}

private static boolean waitForNextReadinessCheck(final long deadlineNanos)
throws InterruptedException {
final long remainingNanos = deadlineNanos - System.nanoTime();
if (remainingNanos <= 0) {
return false;
}
// Keep the network timeout independent: the deadline bounds retries, not an in-flight RPC.
TimeUnit.NANOSECONDS.sleep(Math.min(READINESS_RETRY_INTERVAL_NANOS, remainingNanos));
return deadlineNanos - System.nanoTime() > 0;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -848,6 +848,10 @@ public int[] getPortList() {
return portList;
}

public int getJmxPort() {
return jmxPort;
}

public void clearLogContent() throws IOException {
Files.newOutputStream(Paths.get(getLogPath()), StandardOpenOption.TRUNCATE_EXISTING).close();
}
Expand Down
Loading
Loading