diff --git a/integration-test/AGENTS.md b/integration-test/AGENTS.md index d121b388f3859..b26846495113d 100644 --- a/integration-test/AGENTS.md +++ b/integration-test/AGENTS.md @@ -203,6 +203,18 @@ test; it may be omitted for the default `SimpleIT` profile. - Report the tested revision, edition, profile, and result accurately. Passing an upstream build does not establish that the corresponding downstream edition was tested. +### Startup diagnostics + +- ConfigNode leader discovery, cluster readiness, and JDBC readiness retry every 100ms. The existing + retry count still determines a time budget in seconds (leader discovery allows that budget per + ConfigNode); socket/RPC timeouts remain separate. Cluster status checks share their deadline with + leader discovery rather than restarting a nested wait. +- Add `-DintegrationTest.configNodeStartupTrace=true` to a targeted IT command to record JVM startup + and class initialization before ConfigNode's first application log. The default is disabled to + avoid tracing overhead. Each ConfigNode writes `.startup-.log` alongside its normal + log. JVM startup events include nested phases; class initialization events mark their start and + do not measure exclusive class execution time. + ### Node templates and IDE runs Build the node templates before an IDE run or a direct Failsafe invocation, and rebuild them after diff --git a/integration-test/pom.xml b/integration-test/pom.xml index 779c059e2d649..0cdf1b898b835 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -267,6 +267,11 @@ + + + org.codehaus.mojo + build-helper-maven-plugin + org.apache.maven.plugins diff --git a/integration-test/src/main/i18n/en/org/apache/iotdb/it/env/cluster/env/IntegrationTestEnvMessages.java b/integration-test/src/main/i18n/en/org/apache/iotdb/it/env/cluster/env/IntegrationTestEnvMessages.java new file mode 100644 index 0000000000000..b7a7c51d86c28 --- /dev/null +++ b/integration-test/src/main/i18n/en/org/apache/iotdb/it/env/cluster/env/IntegrationTestEnvMessages.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.it.env.cluster.env; + +/** Localized diagnostics for the integration-test readiness deadlines. */ +public final class IntegrationTestEnvMessages { + + private IntegrationTestEnvMessages() {} + + public static final String EXCEPTION_CLUSTER_STATUS_CHECK_FAILED_RETRY_BUDGET_ARG_SECONDS_AEB4010E = + "Cluster status check failed (retry budget: %d seconds)"; + + public static final String EXCEPTION_JDBC_CONNECTIONS_TO_DATANODES_ARE_NOT_READY_RETRY_BUDGET_ARG_SECONDS_ENDPOINTS_ARG_LASTCONNECTIONFAILURES_ARG_LOGDIRS_ARG_D822C35C = + "JDBC connections to DataNodes are not ready (retry budget: %d seconds). endpoints=%s, lastConnectionFailures=%s, logDirs=%s"; + + public static final String EXCEPTION_FAILED_TO_GET_CONNECTION_TO_CONFIGNODE_ARG_DF03F50F = + "Failed to get connection to ConfigNode %s"; +} diff --git a/integration-test/src/main/i18n/zh/org/apache/iotdb/it/env/cluster/env/IntegrationTestEnvMessages.java b/integration-test/src/main/i18n/zh/org/apache/iotdb/it/env/cluster/env/IntegrationTestEnvMessages.java new file mode 100644 index 0000000000000..5909e594dfe9f --- /dev/null +++ b/integration-test/src/main/i18n/zh/org/apache/iotdb/it/env/cluster/env/IntegrationTestEnvMessages.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.it.env.cluster.env; + +/** Localized diagnostics for the integration-test readiness deadlines. */ +public final class IntegrationTestEnvMessages { + + private IntegrationTestEnvMessages() {} + + public static final String EXCEPTION_CLUSTER_STATUS_CHECK_FAILED_RETRY_BUDGET_ARG_SECONDS_AEB4010E = + "集群状态检查失败(重试时间预算:%d 秒)"; + + public static final String EXCEPTION_JDBC_CONNECTIONS_TO_DATANODES_ARE_NOT_READY_RETRY_BUDGET_ARG_SECONDS_ENDPOINTS_ARG_LASTCONNECTIONFAILURES_ARG_LOGDIRS_ARG_D822C35C = + "连接 DataNode 的 JDBC 连接尚未就绪(重试时间预算:%d 秒)。endpoints=%s, lastConnectionFailures=%s, logDirs=%s"; + + public static final String EXCEPTION_FAILED_TO_GET_CONNECTION_TO_CONFIGNODE_ARG_DF03F50F = + "无法连接 ConfigNode %s"; +} diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java index 90174941e54e4..95efb88b754ca 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java @@ -139,6 +139,12 @@ public DataNodeConfig setMqttPayloadFormatter(String mqttPayloadFormatter) { return this; } + @Override + public DataNodeConfig setMqttHandlerPoolSize(int mqttHandlerPoolSize) { + setProperty("mqtt_handler_pool_size", String.valueOf(mqttHandlerPoolSize)); + return this; + } + @Override public DataNodeConfig setLoadLastCacheStrategy(String strategyName) { setProperty("last_cache_operation_on_load", strategyName); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java index 0b8dee66334ac..43b16fc4a84f3 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java @@ -97,6 +97,7 @@ public abstract class AbstractEnv implements BaseEnv { private static final Logger logger = IoTDBTestLogger.logger; private static final int DEFAULT_CLUSTER_READY_RETRY_COUNT = 30; + private static final long READINESS_RETRY_INTERVAL_NANOS = TimeUnit.MILLISECONDS.toNanos(100); private static final String CLUSTER_READY_RETRY_COUNT_PROPERTY = "integrationTest.clusterReadyRetryCount"; @@ -380,8 +381,8 @@ public void checkClusterStatusOneUnknownOtherRunning() { } /** - * check whether all nodes' status match the provided predicate with RPC. after retryCount times, - * if the status of all nodes still not match the predicate, throw AssertionError. + * check whether all nodes' status match the provided predicate with RPC. after retryCount + * seconds, if the status of all nodes still not match the predicate, throw AssertionError. * * @param nodeStatusCheck the predicate to test the status of nodes */ @@ -400,10 +401,14 @@ public void checkClusterStatus( int actualNodeSize = 0; Map lastNodeStatus = null; Map processStatusMap = new HashMap<>(); + // retryCount historically allowed one second per retry. Preserve that time budget while + // checking more frequently, and share the deadline with leader discovery to avoid nested waits. + final long deadlineNanos = readinessDeadline(retryCount); - for (int i = 0; i < retryCount; i++) { + int attempt = 0; + do { try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) getLeaderConfigNodeConnection()) { + getLeaderConfigNodeConnection(deadlineNanos)) { passed = true; showClusterPassed = true; nodeSizePassed = true; @@ -478,23 +483,33 @@ public void checkClusterStatus( logger.info("The cluster is now ready for testing!"); return; } - logger.info( - "Retry {}: showClusterPassed={}, nodeSizePassed={}, nodeStatusPassed={}, processStatusPassed={}", - i, - showClusterPassed, - nodeSizePassed, - nodeStatusPassed, - processStatusPassed); + if (attempt % 10 == 0) { + logger.info( + "Retry {}: showClusterPassed={}, nodeSizePassed={}, nodeStatusPassed={}, processStatusPassed={}", + attempt, + showClusterPassed, + nodeSizePassed, + nodeStatusPassed, + processStatusPassed); + } + } catch (final InterruptedException e) { + lastException = e; + Thread.currentThread().interrupt(); + break; } catch (final Exception e) { lastException = e; } + attempt++; try { - TimeUnit.SECONDS.sleep(1L); + if (!waitForNextReadinessCheck(deadlineNanos)) { + break; + } } catch (final InterruptedException e) { lastException = e; Thread.currentThread().interrupt(); + break; } - } + } while (true); if (lastException != null) { logger.error( "exception in test Cluster with RPC, message: {}", @@ -547,7 +562,10 @@ private String buildClusterStatusFailureMessage( final Exception lastException) { final StringBuilder builder = new StringBuilder( - String.format("After %d times retry, the cluster status check failed", retryCount)); + String.format( + IntegrationTestEnvMessages + .EXCEPTION_CLUSTER_STATUS_CHECK_FAILED_RETRY_BUDGET_ARG_SECONDS_AEB4010E, + retryCount)); builder .append(": showClusterPassed=") .append(showClusterPassed) @@ -1164,7 +1182,7 @@ protected List getReadConnections( // because it is hard to add retry and handle exception when getting jdbc connections in // getWriteConnectionWithSpecifiedDataNode and getReadConnections. // so use this function to add retry when cluster is ready. - // after retryCount times, if the jdbc can't connect, throw + // after retryCount seconds, if the jdbc can't connect, throw // AssertionError. protected void testJDBCConnection() { logger.info("Testing JDBC connection..."); @@ -1180,7 +1198,8 @@ protected void testJDBCConnection() { testDelegate.addRequest( () -> { Exception lastException = null; - for (int i = 0; i < retryCount; i++) { + final long deadlineNanos = readinessDeadline(retryCount); + do { try (final IoTDBConnection ignored = (IoTDBConnection) DriverManager.getConnection( @@ -1188,8 +1207,8 @@ protected void testJDBCConnection() { + dataNodeEndpoint + getParam(null, NODE_NETWORK_TIMEOUT_MS, ZERO_TIME_ZONE), constructConnectionProperties( - System.getProperty("User", "root"), - System.getProperty("Password", "root"), + System.getProperty("User", SessionConfig.DEFAULT_USER), + System.getProperty("Password", SessionConfig.DEFAULT_PASSWORD), TREE_SQL_DIALECT))) { logger.info("Successfully connecting to DataNode: {}.", dataNodeEndpoint); return null; @@ -1197,9 +1216,8 @@ protected void testJDBCConnection() { lastException = e; lastConnectionFailures.put( dataNodeEndpoint, e.getClass().getName() + ": " + e.getMessage()); - TimeUnit.SECONDS.sleep(1L); } - } + } while (waitForNextReadinessCheck(deadlineNanos)); if (lastException != null) { throw lastException; } @@ -1213,8 +1231,12 @@ protected void testJDBCConnection() { dumpTestJVMSnapshotQuietly("JDBC connection check failed"); throw new AssertionError( String.format( - "After %d times retry, JDBC connections to DataNodes are not ready. endpoints=%s, lastConnectionFailures=%s, logDirs=%s", - retryCount, endpoints, lastConnectionFailures, getClusterLogDirs())); + IntegrationTestEnvMessages + .EXCEPTION_JDBC_CONNECTIONS_TO_DATANODES_ARE_NOT_READY_RETRY_BUDGET_ARG_SECONDS_ENDPOINTS_ARG_LASTCONNECTIONFAILURES_ARG_LOGDIRS_ARG_D822C35C, + retryCount, + endpoints, + lastConnectionFailures, + getClusterLogDirs())); } } @@ -1287,17 +1309,29 @@ public List getDataNodeWrapperList() { @Override public IConfigNodeRPCService.Iface getLeaderConfigNodeConnection() throws IOException, InterruptedException { + // Previously each ConfigNode incurred a one-second delay in every retry round. + return getLeaderConfigNodeConnection( + readinessDeadline((long) retryCount * Math.max(1, configNodeWrapperList.size()))); + } + + private SyncConfigNodeIServiceClient getLeaderConfigNodeConnection(final long deadlineNanos) + throws IOException, InterruptedException { Exception lastException = null; ConfigNodeWrapper lastErrorNode = null; - for (int i = 0; i < retryCount; i++) { + if (configNodeWrapperList.isEmpty()) { + throw new IOException("Empty configNode set"); + } + do { for (final ConfigNodeWrapper configNodeWrapper : configNodeWrapperList) { + lastErrorNode = configNodeWrapper; + SyncConfigNodeIServiceClient client = null; + boolean leaderFound = false; try { if (!configNodeWrapper.getInstance().isAlive()) { throw new IOException("ConfigNode " + configNodeWrapper.getId() + " is not alive"); } - lastErrorNode = configNodeWrapper; - final SyncConfigNodeIServiceClient client = + client = clientManager.borrowClient( new TEndPoint(configNodeWrapper.getIp(), configNodeWrapper.getPort())); final TShowClusterResp resp = client.showCluster(); @@ -1305,10 +1339,9 @@ public IConfigNodeRPCService.Iface getLeaderConfigNodeConnection() if (resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // Only the ConfigNodeClient who connects to the ConfigNode-leader // will respond the SUCCESS_STATUS + leaderFound = true; return client; } else { - // Return client otherwise - client.close(); throw new Exception( "Bad status: " + resp.getStatus().getCode() @@ -1317,38 +1350,58 @@ public IConfigNodeRPCService.Iface getLeaderConfigNodeConnection() } } catch (final Exception e) { lastException = e; + } finally { + // The caller owns the leader client; return every failed probe, including RPC failures. + if (!leaderFound && client != null) { + client.close(); + } + } + if (deadlineNanos - System.nanoTime() <= 0) { + break; } - - // Sleep 1s before next retry - TimeUnit.SECONDS.sleep(1); } - } - if (lastErrorNode != null) { - throw new IOException( - "Failed to get connection to ConfigNode-Leader. Last error configNode: " - + lastErrorNode.getIpAndPortString(), - lastException); - } else { - throw new IOException("Empty configNode set"); - } + // Probe all candidates before waiting, so a follower does not delay the next leader probe. + } while (waitForNextReadinessCheck(deadlineNanos)); + throw new IOException( + "Failed to get connection to ConfigNode-Leader. Last error configNode: " + + lastErrorNode.getIpAndPortString(), + lastException); } @Override public IConfigNodeRPCService.Iface getConfigNodeConnection(int index) throws Exception { Exception lastException = null; final ConfigNodeWrapper configNodeWrapper = configNodeWrapperList.get(index); - for (int i = 0; i < 30; i++) { + final long deadlineNanos = readinessDeadline(30); + do { try { return clientManager.borrowClient( new TEndPoint(configNodeWrapper.getIp(), configNodeWrapper.getPort())); } catch (final Exception e) { lastException = e; } - // Sleep 1s before next retry - TimeUnit.SECONDS.sleep(1); - } + } while (waitForNextReadinessCheck(deadlineNanos)); throw new IOException( - "Failed to get connection to this ConfigNode. Last error: " + lastException); + String.format( + IntegrationTestEnvMessages + .EXCEPTION_FAILED_TO_GET_CONNECTION_TO_CONFIGNODE_ARG_DF03F50F, + configNodeWrapper.getIpAndPortString()), + lastException); + } + + private static long readinessDeadline(final long timeoutSeconds) { + return System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); + } + + private static boolean waitForNextReadinessCheck(final long deadlineNanos) + throws InterruptedException { + final long remainingNanos = deadlineNanos - System.nanoTime(); + if (remainingNanos <= 0) { + return false; + } + // Keep the network timeout independent: the deadline bounds retries, not an in-flight RPC. + TimeUnit.NANOSECONDS.sleep(Math.min(READINESS_RETRY_INTERVAL_NANOS, remainingNanos)); + return deadlineNanos - System.nanoTime() > 0; } @Override diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index 1b9bf2f9fc97b..2047309582e65 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -848,6 +848,10 @@ public int[] getPortList() { return portList; } + public int getJmxPort() { + return jmxPort; + } + public void clearLogContent() throws IOException { Files.newOutputStream(Paths.get(getLogPath()), StandardOpenOption.TRUNCATE_EXISTING).close(); } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java index f819ad481965a..0f344364eba66 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java @@ -26,6 +26,7 @@ import org.apache.iotdb.it.env.cluster.config.MppJVMConfig; import java.io.File; +import java.nio.file.Paths; import java.util.Arrays; import java.util.List; @@ -131,6 +132,20 @@ public final String getId() { protected void addStartCmdParams(final List params) { final String workDir = getNodePath(); final String confDir = workDir + File.separator + "conf"; + if (Boolean.getBoolean("integrationTest.configNodeStartupTrace")) { + // Capture JVM/management-agent and static class initialization before ConfigNode's logger + // exists. Keep this opt-in: class tracing adds I/O and should not affect normal IT timings. + // A relative path avoids the Windows drive colon being parsed as an -Xlog option separator. + final String tracePath = + Paths.get(System.getProperty(USER_DIR)) + .relativize(Paths.get(getLogPath() + ".startup-%p.log")) + .toString() + .replace(File.separatorChar, '/'); + params.add( + "-Xlog:startuptime=debug,class+init=info:file=" + + tracePath + + ":time,uptimemillis,level,tags:filecount=0"); + } params.addAll( Arrays.asList( "-Dlogback.configurationFile=" + confDir + File.separator + "logback-confignode.xml", diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java index 95e4b23b3e73f..d8350577f572d 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java @@ -91,6 +91,11 @@ public DataNodeConfig setMqttPayloadFormatter(String mqttPayloadFormatter) { return this; } + @Override + public DataNodeConfig setMqttHandlerPoolSize(int mqttHandlerPoolSize) { + return this; + } + @Override public DataNodeConfig setLoadLastCacheStrategy(String strategyName) { return this; diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java index 6f65e18cdeac8..99c360d2678cd 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java @@ -52,6 +52,8 @@ DataNodeConfig setLoadActiveListeningCheckIntervalSeconds( DataNodeConfig setMqttPayloadFormatter(String mqttPayloadFormatter); + DataNodeConfig setMqttHandlerPoolSize(int mqttHandlerPoolSize); + DataNodeConfig setLoadLastCacheStrategy(String strategyName); DataNodeConfig setCacheLastValuesForLoad(boolean cacheLastValuesForLoad); diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeErrorStartUpIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeErrorStartUpIT.java index 3f477c1c95fad..3af81205a9d98 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeErrorStartUpIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeErrorStartUpIT.java @@ -37,6 +37,8 @@ import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.env.cluster.config.MppBaseConfig; import org.apache.iotdb.it.env.cluster.config.MppCommonConfig; +import org.apache.iotdb.it.env.cluster.env.AbstractEnv; +import org.apache.iotdb.it.env.cluster.node.AbstractNodeWrapper; import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -44,16 +46,24 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; +import org.awaitility.Awaitility; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Arrays; +import java.util.List; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; @RunWith(IoTDBTestRunner.class) @Category({ClusterIT.class}) @@ -66,10 +76,10 @@ public class IoTDBClusterNodeErrorStartUpIT { private static final String TEST_CLUSTER_NAME = "defaultCluster"; private static final String ERROR_CLUSTER_NAME = "errorCluster"; - private static final int START_RETRY_NUM = 10; + private static final int START_TIMEOUT_SECONDS = 10; - @Before - public void setUp() throws Exception { + @BeforeClass + public static void setUp() throws Exception { EnvFactory.getEnv() .getConfig() .getCommonConfig() @@ -79,11 +89,47 @@ public void setUp() throws Exception { EnvFactory.getEnv().initClusterEnvironment(testConfigNodeNum, testDataNodeNum); } - @After - public void tearDown() { + @AfterClass + public static void tearDown() { EnvFactory.getEnv().cleanClusterEnvironment(); } + @After + public void resetCluster() throws IOException { + // Generated wrappers are tracked even when registration fails. Remove them before checking + // readiness so later tests see only the original cluster and use its reachable DataNode. + try { + removeExtraNodes(EnvFactory.getEnv().getConfigNodeWrapperList(), testConfigNodeNum); + } finally { + removeExtraNodes(EnvFactory.getEnv().getDataNodeWrapperList(), testDataNodeNum); + } + // An assertion can interrupt the restart scenario while original nodes are stopped. + for (ConfigNodeWrapper node : EnvFactory.getEnv().getConfigNodeWrapperList()) { + if (!node.isAlive()) { + node.start(); + } + } + for (DataNodeWrapper node : EnvFactory.getEnv().getDataNodeWrapperList()) { + if (!node.isAlive()) { + node.start(); + } + } + ((AbstractEnv) EnvFactory.getEnv()).checkClusterStatusWithoutUnknown(); + } + + private static void removeExtraNodes(List nodes, int originalSize) + throws IOException { + while (nodes.size() > originalSize) { + AbstractNodeWrapper node = nodes.get(nodes.size() - 1); + node.stopForcibly(); + node.destroyDir(); + Files.deleteIfExists(Paths.get(EnvUtils.getLockFilePath(node.getPort()))); + // Keep a wrapper tracked until cleanup succeeds so @AfterClass can retry on failure. + nodes.remove(nodes.size() - 1); + } + } + + /** Reject ConfigNode and DataNode registration requests with an inconsistent cluster name. */ @Test public void testIllegalNodeRegistration() throws ClientManagerException, IOException, InterruptedException, TException { @@ -117,6 +163,7 @@ public void testIllegalNodeRegistration() } } + /** Reject ports that conflict with registered nodes without persisting their node identities. */ @Test public void testConflictNodeRegistration() throws ClientManagerException, InterruptedException, TException, IOException { @@ -183,6 +230,10 @@ public void testConflictNodeRegistration() Assert.assertFalse(systemProperties.exists()); } + /** + * Reject an incorrect cluster name, node ID and internal endpoint, then restart the original + * nodes and verify that both return to Running. + */ @Test public void testIllegalNodeRestart() throws ClientManagerException, IOException, InterruptedException, TException { @@ -222,26 +273,24 @@ public void testIllegalNodeRestart() .ensureNodeStatus( Arrays.asList(EnvFactory.getEnv().getDataNodeWrapper(0)), Arrays.asList(EnvUtils.getNodeStatusAfterLocalStop())); - boolean isConfigNodeDown = false; - for (int retry = 0; retry < 30; retry++) { - TShowClusterResp showClusterResp = client.showCluster(); - for (TConfigNodeLocation configNodeLocation : showClusterResp.getConfigNodeList()) { - if (configNodeLocation.getConsensusEndPoint().getPort() - == registeredConfigNodeWrapper.getConsensusPort()) { - String configNodeStatus = - showClusterResp.getNodeStatus().get(configNodeLocation.getConfigNodeId()); - if (NodeStatus.Stopped.getStatus().equals(configNodeStatus) - || NodeStatus.Unknown.getStatus().equals(configNodeStatus)) { - isConfigNodeDown = true; - } - } - } - if (isConfigNodeDown) { - break; - } - Thread.sleep(1000); - } - Assert.assertTrue(isConfigNodeDown); + Awaitility.await() + .pollDelay(0, MILLISECONDS) + .pollInterval(100, MILLISECONDS) + .atMost(30, SECONDS) + .until( + () -> { + TShowClusterResp cluster = client.showCluster(); + for (TConfigNodeLocation configNodeLocation : cluster.getConfigNodeList()) { + if (configNodeLocation.getConsensusEndPoint().getPort() + == registeredConfigNodeWrapper.getConsensusPort()) { + String nodeStatus = + cluster.getNodeStatus().get(configNodeLocation.getConfigNodeId()); + return NodeStatus.Stopped.getStatus().equals(nodeStatus) + || NodeStatus.Unknown.getStatus().equals(nodeStatus); + } + } + return false; + }); /* Restart and updatePeer */ // TODO: Delete this IT after enable modify internal TEndPoints @@ -270,17 +319,20 @@ public void testIllegalNodeRestart() } Assert.assertNotEquals(-1, registeredDataNodeId); originPort = registeredDataNodeWrapper.getInternalPort(); - registeredDataNodeWrapper.setInternalPort(-12345); - dataNodeRestartReq = - ConfigNodeTestUtils.generateTDataNodeRestartReq( - TEST_CLUSTER_NAME, registeredDataNodeId, registeredDataNodeWrapper); - dataNodeRestartResp = client.restartDataNode(dataNodeRestartReq); - Assert.assertEquals( - TSStatusCode.REJECT_NODE_START.getStatusCode(), - dataNodeRestartResp.getStatus().getCode()); - Assert.assertTrue( - dataNodeRestartResp.getStatus().getMessage().contains("the internal TEndPoints")); - registeredDataNodeWrapper.setInternalPort(originPort); + try { + registeredDataNodeWrapper.setInternalPort(-12345); + dataNodeRestartReq = + ConfigNodeTestUtils.generateTDataNodeRestartReq( + TEST_CLUSTER_NAME, registeredDataNodeId, registeredDataNodeWrapper); + dataNodeRestartResp = client.restartDataNode(dataNodeRestartReq); + Assert.assertEquals( + TSStatusCode.REJECT_NODE_START.getStatusCode(), + dataNodeRestartResp.getStatus().getCode()); + Assert.assertTrue( + dataNodeRestartResp.getStatus().getMessage().contains("the internal TEndPoints")); + } finally { + registeredDataNodeWrapper.setInternalPort(originPort); + } // Restart and check EnvFactory.getEnv().startConfigNode(1); @@ -294,6 +346,9 @@ public void testIllegalNodeRestart() } } + /** + * Duplicate ports within a node must terminate startup without adding either node to the cluster. + */ @Test public void testIllegalNodeStartUp() throws IOException, ClientManagerException, InterruptedException, TException { @@ -313,13 +368,8 @@ public void testIllegalNodeStartUp() (MppCommonConfig) EnvFactory.getEnv().getConfig().getConfigNodeCommonConfig(), null); portConflictConfigNodeWrapper.start(); - int afterStartConfigNodes; - for (int i = 0; i < START_RETRY_NUM; ++i) { - showClusterResp = client.showCluster(); - afterStartConfigNodes = showClusterResp.getConfigNodeListSize(); - Assert.assertEquals(beforeStartConfigNodes, afterStartConfigNodes); - Thread.sleep(1000); - } + awaitRejectedStartup( + portConflictConfigNodeWrapper, client, beforeStartConfigNodes, beforeStartDataNodes); // set datanode port repeat portConflictDataNodeWrapper.setMppDataExchangePort( @@ -329,13 +379,29 @@ public void testIllegalNodeStartUp() (MppCommonConfig) EnvFactory.getEnv().getConfig().getDataNodeCommonConfig(), null); portConflictDataNodeWrapper.start(); - int afterStartDataNodes; - for (int i = 0; i < START_RETRY_NUM; ++i) { - showClusterResp = client.showCluster(); - afterStartDataNodes = showClusterResp.getDataNodeListSize(); - Assert.assertEquals(beforeStartDataNodes, afterStartDataNodes); - Thread.sleep(1000); - } + awaitRejectedStartup( + portConflictDataNodeWrapper, client, beforeStartConfigNodes, beforeStartDataNodes); } } + + private static void awaitRejectedStartup( + AbstractNodeWrapper node, + SyncConfigNodeIServiceClient client, + int expectedConfigNodes, + int expectedDataNodes) { + // Check membership until the invalid process exits. It cannot register after exiting, so the + // final membership check replaces the remaining fixed observation window. + Awaitility.await() + .pollDelay(0, MILLISECONDS) + .pollInterval(100, MILLISECONDS) + .atMost(START_TIMEOUT_SECONDS, SECONDS) + .until( + () -> { + boolean exited = !node.isAlive(); + TShowClusterResp cluster = client.showCluster(); + Assert.assertEquals(expectedConfigNodes, cluster.getConfigNodeListSize()); + Assert.assertEquals(expectedDataNodes, cluster.getDataNodeListSize()); + return exited; + }); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java index 6f94e0e16f46f..da7c9a7ec2d0b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java @@ -44,6 +44,7 @@ import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.rpc.TSStatusCode; +import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -58,7 +59,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({ClusterIT.class}) @@ -208,6 +208,8 @@ public void showClusterAndNodesTest() throws Exception { } } + // After removing the second ConfigNode, wait for the registry to report one node before stopping + // the removed process; the membership count must not be accepted while the procedure is pending. @Test public void removeAndStopConfigNodeTest() throws Exception { TShowClusterResp showClusterResp; @@ -222,21 +224,18 @@ public void removeAndStopConfigNodeTest() throws Exception { status = client.removeConfigNode(removedConfigNodeLocation); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - // Waiting for RemoveConfigNodeProcedure - boolean isRemoved = false; - for (int retry = 0; retry < 10; retry++) { - showClusterResp = client.showCluster(); - if (showClusterResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() - && showClusterResp.getConfigNodeListSize() == testConfigNodeNum - 1) { - isRemoved = true; - break; - } - - TimeUnit.SECONDS.sleep(1); - } - if (!isRemoved) { - fail("Remove ConfigNode failed"); - } + // Keep the original 10-second budget while observing procedure completion promptly. + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + TShowClusterResp currentClusterResp = client.showCluster(); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + currentClusterResp.getStatus().getCode()); + assertEquals(testConfigNodeNum - 1, currentClusterResp.getConfigNodeListSize()); + }); // Test stop ConfigNode status = client.stopAndClearConfigNode(removedConfigNodeLocation); @@ -244,6 +243,8 @@ public void removeAndStopConfigNodeTest() throws Exception { } } + // After querying both DataNode registrations and their configuration, remove one node and wait + // for membership to shrink before considering the procedure complete. @Test public void queryAndRemoveDataNodeTest() throws Exception { @@ -305,17 +306,18 @@ public void queryAndRemoveDataNodeTest() throws Exception { assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), dataNodeRemoveResp.getStatus().getCode()); - // Waiting for RemoveDataNodeProcedure - for (int retry = 0; retry < 60; retry++) { - showDataNodesResp = client.showDataNodes(); - if (showDataNodesResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() - && showDataNodesResp.getDataNodesInfoListSize() == testDataNodeNum - 1) { - return; - } - - TimeUnit.SECONDS.sleep(1); - } - fail("Remove DataNode failed, last showDataNodesResp: " + showDataNodesResp); + // Keep the original 60-second budget while observing procedure completion promptly. + Awaitility.await() + .atMost(60, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + TShowDataNodesResp currentDataNodesResp = client.showDataNodes(); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + currentDataNodesResp.getStatus().getCode()); + assertEquals(testDataNodeNum - 1, currentDataNodesResp.getDataNodesInfoListSize()); + }); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseSetAndDeleteIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseSetAndDeleteIT.java index 959f37d594d51..71988e24b5c03 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseSetAndDeleteIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/database/IoTDBDatabaseSetAndDeleteIT.java @@ -37,9 +37,9 @@ import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.rpc.TSStatusCode; -import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -54,22 +54,25 @@ @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBDatabaseSetAndDeleteIT { - @Before - public void setUp() throws Exception { + // Each scenario owns a separate database prefix so the cluster can be reused across methods. + @BeforeClass + public static void setUp() throws Exception { // Init 1C1D environment EnvFactory.getEnv().initClusterEnvironment(1, 1); } - @After - public void tearDown() { + @AfterClass + public static void tearDown() { EnvFactory.getEnv().cleanClusterEnvironment(); } + // Exact and wildcard lookups return default/custom settings; duplicate registration fails and + // later queries reflect replication-factor and partition-interval updates. @Test public void testSetAndQueryDatabase() throws Exception { TSStatus status; - final String sg0 = "root.sg0"; - final String sg1 = "root.sg1"; + final String sg0 = "root.query0"; + final String sg1 = "root.query1"; try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { @@ -90,7 +93,7 @@ public void testSetAndQueryDatabase() throws Exception { // test count all Databases TCountDatabaseResp countResp = client.countMatchedDatabases( - new TGetDatabaseReq(Arrays.asList("root", "sg*"), ALL_MATCH_SCOPE_BINARY)); + new TGetDatabaseReq(Arrays.asList("root", "query*"), ALL_MATCH_SCOPE_BINARY)); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), countResp.getStatus().getCode()); Assert.assertEquals(2, countResp.getCount()); @@ -98,7 +101,7 @@ public void testSetAndQueryDatabase() throws Exception { // test count one Database countResp = client.countMatchedDatabases( - new TGetDatabaseReq(Arrays.asList("root", "sg0"), ALL_MATCH_SCOPE_BINARY)); + new TGetDatabaseReq(Arrays.asList("root", "query0"), ALL_MATCH_SCOPE_BINARY)); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), countResp.getStatus().getCode()); Assert.assertEquals(1, countResp.getCount()); @@ -106,7 +109,7 @@ public void testSetAndQueryDatabase() throws Exception { // test query all DatabaseSchemas TDatabaseSchemaResp getResp = client.getMatchedDatabaseSchemas( - new TGetDatabaseReq(Arrays.asList("root", "sg*"), ALL_MATCH_SCOPE_BINARY)); + new TGetDatabaseReq(Arrays.asList("root", "query*"), ALL_MATCH_SCOPE_BINARY)); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), getResp.getStatus().getCode()); Map schemaMap = getResp.getDatabaseSchemaMap(); @@ -144,7 +147,7 @@ public void testSetAndQueryDatabase() throws Exception { // test setter results getResp = client.getMatchedDatabaseSchemas( - new TGetDatabaseReq(Arrays.asList("root", "sg1"), ALL_MATCH_SCOPE_BINARY)); + new TGetDatabaseReq(Arrays.asList("root", "query1"), ALL_MATCH_SCOPE_BINARY)); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), getResp.getStatus().getCode()); schemaMap = getResp.getDatabaseSchemaMap(); @@ -158,11 +161,12 @@ public void testSetAndQueryDatabase() throws Exception { } } + // Delete two registered databases and verify that no schema remains under their database prefix. @Test public void testDeleteDatabase() throws Exception { TSStatus status; - final String sg0 = "root.sg0"; - final String sg1 = "root.sg1"; + final String sg0 = "root.delete0"; + final String sg1 = "root.delete1"; try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { @@ -180,7 +184,7 @@ public void testDeleteDatabase() throws Exception { TSStatus deleteSgStatus = client.deleteDatabases(deleteDatabasesReq); TDatabaseSchemaResp root = client.getMatchedDatabaseSchemas( - new TGetDatabaseReq(Arrays.asList("root", "sg*"), ALL_MATCH_SCOPE_BINARY)); + new TGetDatabaseReq(Arrays.asList("root", "delete*"), ALL_MATCH_SCOPE_BINARY)); Assert.assertTrue(root.getDatabaseSchemaMap().isEmpty()); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), deleteSgStatus.getCode()); } diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/DataPartitionTableIntegrityCheckProcedureIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/DataPartitionTableIntegrityCheckProcedureIT.java index f2390a6c6080c..71dbf227ab484 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/DataPartitionTableIntegrityCheckProcedureIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/DataPartitionTableIntegrityCheckProcedureIT.java @@ -33,9 +33,11 @@ import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.TSStatusCode; -import org.junit.After; +import org.awaitility.Awaitility; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -67,6 +69,7 @@ public class DataPartitionTableIntegrityCheckProcedureIT { private static final String TABLE_DATABASE = "repair_table_db"; private static final String TABLE_NAME = "table1"; + private static final long POLL_INTERVAL_IN_MS = 100; private static final long TIME_PARTITION_INTERVAL = 604_800_000L; private static final long TABLE_TTL = 7 * TIME_PARTITION_INTERVAL; private static final long CURRENT_TIME_PARTITION_START = @@ -74,8 +77,9 @@ public class DataPartitionTableIntegrityCheckProcedureIT { private static final long EXPIRED_TIME_PARTITION_START = CURRENT_TIME_PARTITION_START - TABLE_TTL * 2; - @Before - public void setUp() { + // Only one scenario creates data, in its own database; all three can reuse the same cluster. + @BeforeClass + public static void setUp() { EnvFactory.getEnv() .getConfig() .getCommonConfig() @@ -95,11 +99,21 @@ public void setUp() { EnvFactory.getEnv().initClusterEnvironment(1, 1); } - @After - public void tearDown() throws Exception { + @Before + public void waitForPreviousRepairCompletion() throws Exception { + // Repair is cluster-wide and asynchronous, so database isolation alone cannot isolate tasks. + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + waitForRepairCompletion(statement); + } + } + + @AfterClass + public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + // Ten concurrent repair requests must accept exactly one procedure and reject the other nine. @Test public void testConcurrentSubmitDataPartitionTableIntegrityCheckProcedure() throws InterruptedException { @@ -159,6 +173,7 @@ public void testConcurrentSubmitDataPartitionTableIntegrityCheckProcedure() "The other concurrent submissions should be rejected", threadCount - 1, failCount.get()); } + // An idle cluster reports zero progress; a submitted repair reports a known state and 0-100%. @Test public void testShowRepairDataPartitionTableProgress() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); @@ -170,6 +185,7 @@ public void testShowRepairDataPartitionTableProgress() throws Exception { } } + // Repair must preserve current table partitions and must not restore those removed by table TTL. @Test public void testRepairDataPartitionTableIgnoresTableModelDatabase() throws Exception { final TDataPartitionReq dataPartitionReq = new TDataPartitionReq(); @@ -241,30 +257,29 @@ private static boolean containsTimePartition( private static void waitUntilTimePartitionRemoved( final TDataPartitionReq request, final long timePartitionStart) throws Exception { - for (int retry = 0; retry < 120; retry++) { - if (!containsTimePartition(request, timePartitionStart)) { - return; - } - TimeUnit.SECONDS.sleep(1); - } - Assert.fail("The expired table-model time partition was not cleaned within the timeout"); + Awaitility.await("expired table-model time partition cleanup") + .atMost(120, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL_IN_MS, TimeUnit.MILLISECONDS) + .until(() -> !containsTimePartition(request, timePartitionStart)); } private static void waitForRepairCompletion(final Statement statement) throws Exception { - TimeUnit.SECONDS.sleep(1); - for (int retry = 0; retry < 120; retry++) { - try (final ResultSet resultSet = - statement.executeQuery("SHOW REPAIR DATA PARTITION TABLE PROGRESS")) { - Assert.assertTrue(resultSet.next()); - if (RepairDataPartitionTableProgressState.IDLE - .name() - .equals(resultSet.getString("Status"))) { - return; - } - } - TimeUnit.SECONDS.sleep(1); - } - Assert.fail("The data partition table repair did not complete within the timeout"); + // Submission registers the procedure before returning, so IDLE can be checked immediately. + Awaitility.await("data partition table repair completion") + .atMost(120, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL_IN_MS, TimeUnit.MILLISECONDS) + .until( + () -> { + try (final ResultSet resultSet = + statement.executeQuery("SHOW REPAIR DATA PARTITION TABLE PROGRESS")) { + Assert.assertTrue(resultSet.next()); + return RepairDataPartitionTableProgressState.IDLE + .name() + .equals(resultSet.getString("Status")); + } + }); } private static void assertRepairProgress( diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java index 3e5195586bebb..07f9806576f76 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java @@ -46,6 +46,7 @@ import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.rpc.TSStatusCode; +import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -114,6 +115,8 @@ public void tearDown() { EnvFactory.getEnv().cleanClusterEnvironment(); } + // Partition allocation excludes Removing and ReadOnly nodes, permits an offline node, and + // restores all regions after that node restarts. @Test public void testPartitionAllocation() throws Exception { try (SyncConfigNodeIServiceClient client = @@ -347,9 +350,24 @@ public void testPartitionAllocation() throws Exception { // Status: Running, Running, Removing, ReadOnly, Stopped (Unknown on Windows), Running. // RegionGroup: [0, 1, 2, 3], [0, 1, 2], [0], [1], [2, 3], [3] EnvFactory.getEnv().registerNewDataNode(false); - // Use thread sleep to replace verifying because the offline DataNode can not pass the - // connection check - TimeUnit.SECONDS.sleep(25); + DataNodeWrapper registeredDataNode = EnvFactory.getEnv().getDataNodeWrapper(5); + // Check only the new node because a cluster-wide readiness check includes the offline node. + Awaitility.await("new DataNode registration") + .atMost(60, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .until( + () -> { + TShowDataNodesResp showDataNodesResp = client.showDataNodes(); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + showDataNodesResp.getStatus().getCode()); + return showDataNodesResp.getDataNodesInfoList().stream() + .anyMatch( + dataNodeInfo -> + registeredDataNode.getIp().equals(dataNodeInfo.getRpcAddresss()) + && registeredDataNode.getPort() == dataNodeInfo.getRpcPort() + && NodeStatus.Running.getStatus().equals(dataNodeInfo.getStatus())); + }); partitionSlotsMap = ConfigNodeTestUtils.constructPartitionSlotsMap( sg, diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java index d150348c09601..6fc6a5e13eb64 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java @@ -29,9 +29,10 @@ import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.TSStatusCode; -import org.junit.After; +import org.awaitility.Awaitility; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -45,12 +46,14 @@ @Category({ClusterIT.class}) public class IoTDBPartitionTableAutoCleanIT { - private static final String TREE_DATABASE_PREFIX = "root.db.g_"; - private static final String TABLE_DATABASE_PREFIX = "database_"; + private static final String TREE_DATABASE_PREFIX = "root.db.auto_clean_"; + private static final String TREE_DEVICE_TTL_DATABASE_PREFIX = "root.db.device_ttl_"; + private static final String TABLE_DATABASE_NAME = "database_auto_clean"; private static final int TEST_REPLICATION_FACTOR = 1; private static final long TEST_TIME_PARTITION_INTERVAL = 604800000; private static final long TEST_TTL_CHECK_INTERVAL_IN_MS = 5_00; + private static final long AUTO_CLEAN_POLL_INTERVAL_IN_MS = 100; private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT = new TTimePartitionSlot() @@ -60,8 +63,9 @@ public class IoTDBPartitionTableAutoCleanIT { * TEST_TIME_PARTITION_INTERVAL); private static final long TEST_TTL = 7 * TEST_TIME_PARTITION_INTERVAL; - @Before - public void setUp() throws Exception { + // All scenarios use isolated database names so one initialized 1C1D cluster can serve the class. + @BeforeClass + public static void setUp() throws Exception { EnvFactory.getEnv() .getConfig() .getCommonConfig() @@ -74,11 +78,12 @@ public void setUp() throws Exception { EnvFactory.getEnv().initClusterEnvironment(1, 1); } - @After - public void tearDown() { + @AfterClass + public static void tearDown() { EnvFactory.getEnv().cleanClusterEnvironment(); } + // Expired partitions are eventually removed for database and table TTL rules. @Test public void testAutoCleanPartitionTableForTreeModel() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); @@ -117,31 +122,18 @@ public void testAutoCleanPartitionTableForTreeModel() throws Exception { } try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - for (int retry = 0; retry < 120; retry++) { - boolean partitionTableAutoCleaned = true; - TDataPartitionTableResp resp = client.getDataPartitionTable(req); - if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == resp.getStatus().getCode()) { - partitionTableAutoCleaned = - resp.getDataPartitionTable().entrySet().stream() - .flatMap(e1 -> e1.getValue().entrySet().stream()) - .allMatch(e2 -> e2.getValue().size() == 1); - } - if (partitionTableAutoCleaned) { - return; - } - TimeUnit.SECONDS.sleep(1); - } + awaitPartitionTableAutoCleaned(client, req); } - Assert.fail("The PartitionTable in the ConfigNode is not auto cleaned!"); } + // Device-level TTL alone must not clean expired partitions when the database has no TTL. @Test public void testAutoCleanTakesNoEffectsForTreeDeviceTTL() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); Statement statement = connection.createStatement()) { // Create databases and insert test data for (int i = 0; i < 3; i++) { - String databaseName = String.format("%s%d", TREE_DATABASE_PREFIX, i); + String databaseName = String.format("%s%d", TREE_DEVICE_TTL_DATABASE_PREFIX, i); statement.execute(String.format("CREATE DATABASE %s", databaseName)); statement.execute( String.format( @@ -168,7 +160,8 @@ public void testAutoCleanTakesNoEffectsForTreeDeviceTTL() throws Exception { TDataPartitionReq req = new TDataPartitionReq(); for (int i = 0; i < 3; i++) { - req.putToPartitionSlotsMap(String.format("%s%d", TREE_DATABASE_PREFIX, i), new TreeMap<>()); + req.putToPartitionSlotsMap( + String.format("%s%d", TREE_DEVICE_TTL_DATABASE_PREFIX, i), new TreeMap<>()); } try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { @@ -188,13 +181,14 @@ public void testAutoCleanTakesNoEffectsForTreeDeviceTTL() throws Exception { } } + // Expired partitions are eventually removed when a table-model database has a TTL. @Test public void testAutoCleanPartitionTableForTableModel() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { // Create databases and insert test data - String databaseName = TABLE_DATABASE_PREFIX; + String databaseName = TABLE_DATABASE_NAME; statement.execute(String.format("CREATE DATABASE IF NOT EXISTS %s", databaseName)); statement.execute(String.format("USE %s", databaseName)); statement.execute("CREATE TABLE tb (time TIMESTAMP TIME, s int64 FIELD)"); @@ -207,29 +201,31 @@ public void testAutoCleanPartitionTableForTableModel() throws Exception { statement.execute( String.format( "INSERT INTO tb(time, s) VALUES (%d, %d)", TEST_CURRENT_TIME_SLOT.getStartTime(), 1)); - statement.execute(String.format("USE %s", TABLE_DATABASE_PREFIX)); + statement.execute(String.format("USE %s", databaseName)); statement.execute(String.format("ALTER TABLE tb SET PROPERTIES TTL=%d", TEST_TTL)); } TDataPartitionReq req = new TDataPartitionReq(); - req.putToPartitionSlotsMap(TABLE_DATABASE_PREFIX, new TreeMap<>()); + req.putToPartitionSlotsMap(TABLE_DATABASE_NAME, new TreeMap<>()); try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - for (int retry = 0; retry < 120; retry++) { - boolean partitionTableAutoCleaned = true; - TDataPartitionTableResp resp = client.getDataPartitionTable(req); - if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == resp.getStatus().getCode()) { - partitionTableAutoCleaned = - resp.getDataPartitionTable().entrySet().stream() - .flatMap(e1 -> e1.getValue().entrySet().stream()) - .allMatch(e2 -> e2.getValue().size() == 1); - } - if (partitionTableAutoCleaned) { - return; - } - TimeUnit.SECONDS.sleep(1); - } + awaitPartitionTableAutoCleaned(client, req); } - Assert.fail("The PartitionTable in the ConfigNode is not auto cleaned!"); + } + + private static void awaitPartitionTableAutoCleaned( + SyncConfigNodeIServiceClient client, TDataPartitionReq req) { + // TTL checks run asynchronously; poll promptly while retaining the existing two-minute bound. + Awaitility.await("partition table auto cleanup") + .atMost(120, TimeUnit.SECONDS) + .pollInterval(AUTO_CLEAN_POLL_INTERVAL_IN_MS, TimeUnit.MILLISECONDS) + .until( + () -> { + TDataPartitionTableResp resp = client.getDataPartitionTable(req); + return TSStatusCode.SUCCESS_STATUS.getStatusCode() == resp.getStatus().getCode() + && resp.getDataPartitionTable().entrySet().stream() + .flatMap(e1 -> e1.getValue().entrySet().stream()) + .allMatch(e2 -> e2.getValue().size() == 1); + }); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java index 69ab3a03d2832..68a108ee4cc15 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java @@ -29,9 +29,9 @@ import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.TSStatusCode; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; +import org.awaitility.Awaitility; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -50,7 +50,11 @@ public class IoTDBPartitionTableAutoCleanUSIT { private static final int TEST_REPLICATION_FACTOR = 1; private static final long TEST_TIME_PARTITION_INTERVAL_IN_MS = 604800_000; - private static final long TEST_TTL_CHECK_INTERVAL = 5_000; + private static final long TEST_TIME_PARTITION_INTERVAL_IN_US = + TimeUnit.MILLISECONDS.toMicros(TEST_TIME_PARTITION_INTERVAL_IN_MS); + // Exercise TTL cleanup promptly without changing the microsecond timestamps or TTL rules. + private static final long TEST_TTL_CHECK_INTERVAL = 500; + private static final long AUTO_CLEAN_POLL_INTERVAL_IN_MS = 100; private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT = new TTimePartitionSlot() @@ -61,8 +65,17 @@ public class IoTDBPartitionTableAutoCleanUSIT { * TEST_TIME_PARTITION_INTERVAL_IN_MS); private static final long TEST_TTL_IN_MS = 7 * TEST_TIME_PARTITION_INTERVAL_IN_MS; - @Before - public void setUp() throws Exception { + // Inserts use microseconds, while the configured partition interval is converted from + // milliseconds. + private static final TTimePartitionSlot EXPECTED_CURRENT_TIME_SLOT = + new TTimePartitionSlot( + TEST_CURRENT_TIME_SLOT.getStartTime() + / TEST_TIME_PARTITION_INTERVAL_IN_US + * TEST_TIME_PARTITION_INTERVAL_IN_US); + + // Tree and table scenarios use distinct database names and can share the same 1C1D cluster. + @BeforeClass + public static void setUp() throws Exception { EnvFactory.getEnv() .getConfig() .getCommonConfig() @@ -77,11 +90,13 @@ public void setUp() throws Exception { EnvFactory.getEnv().initClusterEnvironment(1, 1); } - @After - public void tearDown() { + @AfterClass + public static void tearDown() { EnvFactory.getEnv().cleanClusterEnvironment(); } + // With microsecond timestamps, database/device TTL precedence must retain only current + // partitions. @Test public void testAutoCleanPartitionTableForTreeModel() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); @@ -120,24 +135,12 @@ public void testAutoCleanPartitionTableForTreeModel() throws Exception { } try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - for (int retry = 0; retry < 120; retry++) { - boolean partitionTableAutoCleaned = true; - TDataPartitionTableResp resp = client.getDataPartitionTable(req); - if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == resp.getStatus().getCode()) { - partitionTableAutoCleaned = - resp.getDataPartitionTable().entrySet().stream() - .flatMap(e1 -> e1.getValue().entrySet().stream()) - .allMatch(e2 -> e2.getValue().size() == 1); - } - if (partitionTableAutoCleaned) { - return; - } - TimeUnit.SECONDS.sleep(1); - } + awaitPartitionTableAutoCleaned(client, req); } - Assert.fail("The PartitionTable in the ConfigNode is not auto cleaned!"); } + // With microsecond timestamps, table TTL must remove expired partitions and retain the current + // one. @Test public void testAutoCleanPartitionTableForTableModel() throws Exception { try (final Connection connection = @@ -165,21 +168,34 @@ public void testAutoCleanPartitionTableForTableModel() throws Exception { req.putToPartitionSlotsMap(TABLE_DATABASE_PREFIX, new TreeMap<>()); try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - for (int retry = 0; retry < 120; retry++) { - boolean partitionTableAutoCleaned = true; - TDataPartitionTableResp resp = client.getDataPartitionTable(req); - if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == resp.getStatus().getCode()) { - partitionTableAutoCleaned = - resp.getDataPartitionTable().entrySet().stream() - .flatMap(e1 -> e1.getValue().entrySet().stream()) - .allMatch(e2 -> e2.getValue().size() == 1); - } - if (partitionTableAutoCleaned) { - return; - } - TimeUnit.SECONDS.sleep(1); - } + awaitPartitionTableAutoCleaned(client, req); } - Assert.fail("The PartitionTable in the ConfigNode is not auto cleaned!"); + } + + private static void awaitPartitionTableAutoCleaned( + SyncConfigNodeIServiceClient client, TDataPartitionReq req) { + // Keep the two-minute deadline and require real, successful results rather than an empty match. + Awaitility.await("microsecond partition table auto cleanup") + .atMost(120, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(AUTO_CLEAN_POLL_INTERVAL_IN_MS, TimeUnit.MILLISECONDS) + .until( + () -> { + TDataPartitionTableResp resp = client.getDataPartitionTable(req); + return TSStatusCode.SUCCESS_STATUS.getStatusCode() == resp.getStatus().getCode() + && resp.getDataPartitionTable() + .keySet() + .equals(req.getPartitionSlotsMap().keySet()) + && resp.getDataPartitionTable().values().stream() + .allMatch( + seriesPartitions -> + !seriesPartitions.isEmpty() + && seriesPartitions.values().stream() + .allMatch( + timePartitions -> + timePartitions.size() == 1 + && timePartitions.containsKey( + EXPECTED_CURRENT_TIME_SLOT))); + }); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java index 8f8ec7daa64b6..c6d7975694161 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java @@ -289,15 +289,22 @@ public void testPartialPathRangeDelete() throws SQLException { cleanData(6); } + // Insert 10,000 five-column rows with a 10KB memtable threshold and 1,000-timestamp + // partitions to exercise automatic flushing. Deleting (1500, 9000] must leave 2,500 s0 rows. @Test public void testDelFlushingMemtable() throws SQLException { int testNum = 7; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { + // Batch the original single-row SQLs to reduce RPCs while retaining their order and + // per-row automatic-flush checks on the server. for (int i = 1; i <= 10000; i++) { - statement.execute( + statement.addBatch( String.format(insertTemplate, testNum, i, i, i, (double) i, "'" + i + "'", i % 2 == 0)); + if (i % 1000 == 0) { + statement.executeBatch(); + } } statement.execute("DELETE FROM root.vehicle7.d0.s0 WHERE time > 1500 and time <= 9000"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBIPv6ExternalServiceIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBIPv6ExternalServiceIT.java index e63d428992c74..3899a133aad2e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBIPv6ExternalServiceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBIPv6ExternalServiceIT.java @@ -93,6 +93,7 @@ public static void tearDown() { } } + /** REST ping and writes to an IPv6 loopback endpoint remain reachable and queryable. */ @Test public void restServiceCanCommunicateThroughIPv6Loopback() throws Exception { final DataNodeWrapper dataNode = EnvFactory.getEnv().getDataNodeWrapper(0); @@ -103,7 +104,8 @@ public void restServiceCanCommunicateThroughIPv6Loopback() throws Exception { try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) { Awaitility.await() .atMost(30, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .ignoreExceptions() .untilAsserted(() -> assertRestPingSucceeds(httpClient, restAddress)); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java index a5978558751dc..dfcc91dc8cb86 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java @@ -25,7 +25,8 @@ import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.junit.After; -import org.junit.Before; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -50,17 +51,27 @@ @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBInsertWithQueryIT { - @Before - public void setUp() throws Exception { + // All scenarios use the same startup configuration, so only their root.fans schema needs cleanup. + @BeforeClass + public static void setUp() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecisionCheckEnabled(false); EnvFactory.getEnv().initClusterEnvironment(); } @After public void tearDown() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("DELETE TIMESERIES root.fans.**"); + } + } + + @AfterClass + public static void cleanCluster() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + // Insert [0,1000) and [1000,2000) in order; each query must return the cumulative row count. @Test public void insertWithQueryTest() { // insert @@ -76,6 +87,8 @@ public void insertWithQueryTest() { selectAndCount(2000); } + // Insert consecutive ranges while five readers run; every read must return the expected row + // count. @Test public void insertWithQueryMultiThreadTest() throws InterruptedException { // insert @@ -90,6 +103,7 @@ public void insertWithQueryMultiThreadTest() throws InterruptedException { selectWithMultiThread(2000); } + // Overlap [0,1000) with [500,1500), then add [2000,3000); counts must deduplicate timestamps. @Test public void insertWithQueryUnsequenceTest() { // insert @@ -111,6 +125,7 @@ public void insertWithQueryUnsequenceTest() { selectAndCount(2500); } + // Repeat overlapping writes with five readers; each query must return unique timestamp counts. @Test public void insertWithQueryMultiThreadUnsequenceTest() throws InterruptedException { // insert @@ -131,6 +146,7 @@ public void insertWithQueryMultiThreadUnsequenceTest() throws InterruptedExcepti selectWithMultiThread(2500); } + // Flush between ordered ranges and verify later writes remain visible with the expected total. @Test public void insertWithQueryFlushTest() { // insert @@ -148,6 +164,8 @@ public void insertWithQueryFlushTest() { selectAndCount(2000); } + // Check negative-time ordering and aggregation boundaries, including + // the Long.MIN_VALUE + 1 timestamp value. @Test public void insertNegativeTimestampWithQueryTest() { // insert @@ -169,6 +187,8 @@ public void insertNegativeTimestampWithQueryTest() { queryNegativeTimestampTypeDataTest(); } + // Run five readers alongside flushes between ordered writes; each reader must see the expected + // row count. @Test public void flushWithQueryTest() throws InterruptedException { // insert @@ -184,6 +204,8 @@ public void flushWithQueryTest() throws InterruptedException { selectWithMultiThreadAndFlush(1500); } + // Flush while querying out-of-order writes, then add a missing interval and verify the merged + // count. @Test public void flushWithQueryUnorderTest() throws InterruptedException { // insert @@ -202,6 +224,8 @@ public void flushWithQueryUnorderTest() throws InterruptedException { selectWithMultiThread(1000); } + // Merge out-of-order ranges across concurrent flushes; checked snapshots must have expected + // counts. @Test public void flushWithQueryUnorderLargerTest() throws InterruptedException { // insert @@ -228,6 +252,7 @@ public void flushWithQueryUnorderLargerTest() throws InterruptedException { selectWithMultiThreadAndFlush(1500); } + // Interleave writers, readers, and a flush; each reader must see strictly increasing timestamps. @Test public void insertWithQueryTogetherTest() throws InterruptedException { // insert diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadLastCacheIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadLastCacheIT.java index 4fd1f6fab9139..628247251f401 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadLastCacheIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadLastCacheIT.java @@ -25,7 +25,6 @@ import org.apache.iotdb.it.utils.TsFileGenerator; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; -import org.apache.iotdb.jdbc.IoTDBSQLException; import com.google.common.util.concurrent.RateLimiter; import org.apache.tsfile.enums.ColumnCategory; @@ -94,9 +93,10 @@ public IoTDBLoadLastCacheIT(LastCacheLoadStrategy lastCacheLoadStrategy) { this.lastCacheLoadStrategy = lastCacheLoadStrategy; } - @Before - public void setUp() throws Exception { - tmpDir = new File(Files.createTempDirectory("load").toUri()); + // The load strategy is read at startup, so only tests with the same parameter share a cluster. + @Parameterized.BeforeParam + public static void setUpEnvironment(LastCacheLoadStrategy lastCacheLoadStrategy) + throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setTimePartitionInterval(PARTITION_INTERVAL); EnvFactory.getEnv() .getConfig() @@ -111,13 +111,25 @@ public void setUp() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); } - @After - public void tearDown() throws Exception { - deleteSG(); + @Parameterized.AfterParam + public static void tearDownEnvironment() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); + } - if (!deleteDir()) { - LOGGER.error("Can not delete tmp dir for loading tsfile."); + @Before + public void setUp() throws Exception { + tmpDir = new File(Files.createTempDirectory("load-" + lastCacheLoadStrategy.name()).toUri()); + } + + @After + public void tearDown() throws Exception { + try { + // Drop both models' databases to clear their schemas and last caches between tests. + deleteDatabases(); + } finally { + if (!deleteDir()) { + LOGGER.error("Can not delete tmp dir for loading tsfile."); + } } } @@ -177,17 +189,30 @@ private String convert2AlignedSQL(final String device, final List i).average().orElse(0.0) / 1000000); } + /** + * Load 100000 table TsFiles while concurrent rate-limited readers query last values until the + * final timestamp is visible; keep this large performance workload excluded from ordinary IT + * runs. + */ @Ignore("Performance") @Test public void testTableLoadPerformance() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java index 73d3e9ccaed25..99cfba2520ee4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java @@ -142,7 +142,10 @@ public static void setUpClass() { .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) .setSchemaReplicationFactor(REPLICATION_FACTOR) - .setDataReplicationFactor(REPLICATION_FACTOR); + .setDataReplicationFactor(REPLICATION_FACTOR) + // Keep this functional fixture's WAL/Region budget independent of the host CPU count. + .setDataRegionPerDataNode(1) + .setDefaultDataRegionGroupNumPerDatabase(1); // Keep every kind of merge off for this class: the restart case hard-links the replicas' // TsFiles out of the DataNode directory and compares their points while the restarted DataNode // catches up, so compaction must not rewrite those files in the meantime. @@ -234,7 +237,7 @@ public void testLoadTsFileInCluster() throws Exception { } } - /** Concurrent LOADs on separate files must not interfere with each other. */ + /** Four concurrent LOADs on separate time partitions must expose their combined point count. */ @Test public void testLoadConcurrently() throws Exception { final int fileCount = 4; @@ -286,15 +289,11 @@ public void testLoadConcurrently() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { final long expected = expectedPointCount; - long actualPointCount = queryPointCount(statement); - for (int retry = 0; actualPointCount < expected && retry < 30; retry++) { - Thread.sleep(1_000L); - actualPointCount = queryPointCount(statement); - } - Assert.assertEquals(expected, actualPointCount); + Assert.assertEquals(expected, awaitPointCountAtLeast(statement, expected)); } } + /** A load through the remaining DataNodes preserves data while one DataNode is stopped. */ @Test public void testLoadAfterOneDataNodeDown() throws Exception { final long firstPointCount = generateTsFile(tmpDir, "load-first.tsfile", 5_000, 1L); @@ -323,12 +322,8 @@ public void testLoadAfterOneDataNodeDown() throws Exception { statement.execute("load \"" + secondDir.getAbsolutePath() + "\""); final long expectedPointCount = firstPointCount + secondPointCount; - long actualPointCount = queryPointCount(statement); - for (int retry = 0; actualPointCount < expectedPointCount && retry < 30; retry++) { - Thread.sleep(1_000L); - actualPointCount = queryPointCount(statement); - } - Assert.assertEquals(expectedPointCount, actualPointCount); + Assert.assertEquals( + expectedPointCount, awaitPointCountAtLeast(statement, expectedPointCount)); } } @@ -1070,6 +1065,17 @@ private static void waitUntilClusterAnswers() { }); } + // Keep the JDBC statement on the calling thread while preserving the 30-second retry window. + private long awaitPointCountAtLeast(final Statement statement, final long expectedPointCount) + throws Exception { + return Awaitility.await() + .pollInSameThread() + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .atMost(30, TimeUnit.SECONDS) + .until(() -> queryPointCount(statement), pointCount -> pointCount >= expectedPointCount); + } + private static boolean deleteRecursively(final File file) { final File[] children = file.listFiles(); if (children != null) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java index d3778b5d528e8..b17c6a19b57a1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java @@ -49,8 +49,10 @@ import org.apache.tsfile.write.schema.MeasurementSchema; import org.awaitility.Awaitility; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -93,10 +95,18 @@ public class IoTDBLoadTsFileIT { private static final long loadTsFileAnalyzeSchemaMemorySizeInBytes = 10 * 1024L; private File tmpDir; + private boolean templateCreated; + private boolean testUserCreated; @Before public void setUp() throws Exception { tmpDir = new File(Files.createTempDirectory("load").toUri()); + templateCreated = false; + testUserCreated = false; + } + + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setTimePartitionInterval(PARTITION_INTERVAL); EnvFactory.getEnv().getConfig().getCommonConfig().setEnforceStrongPassword(false); EnvFactory.getEnv().getConfig().getCommonConfig().setPipeMemoryManagementEnabled(false); @@ -112,14 +122,21 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - deleteSG(); - EnvFactory.getEnv().cleanClusterEnvironment(); - - if (!deleteDir()) { - LOGGER.error("Can not delete tmp dir for loading tsfile."); + try { + // The tests reuse fixed database names, so clear test-owned metadata before the next method. + deleteSG(); + } finally { + if (!deleteDir()) { + LOGGER.error("Can not delete tmp dir for loading tsfile."); + } } } + @AfterClass + public static void tearDownClass() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + private void registerSchema() throws SQLException { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { @@ -180,14 +197,40 @@ private void deleteSG() throws SQLException { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute(String.format("delete database %s", SchemaConfig.DATABASE_0)); - statement.execute(String.format("delete database %s", SchemaConfig.DATABASE_1)); - } catch (final IoTDBSQLException ignored) { + deleteDatabaseIfExists(statement, SchemaConfig.DATABASE_0); + deleteDatabaseIfExists(statement, SchemaConfig.DATABASE_1); + if (templateCreated) { + statement.execute("drop schema template t1"); + templateCreated = false; + } + if (testUserCreated) { + statement.execute("drop user test"); + testUserCreated = false; + } + } + } + + private void deleteDatabaseIfExists(final Statement statement, final String database) + throws SQLException { + try (final ResultSet resultSet = statement.executeQuery("show databases " + database)) { + if (resultSet.next()) { + statement.execute("delete database " + database); + } } } private boolean deleteDir() { - for (final File file : tmpDir.listFiles()) { + if (tmpDir == null) { + return true; + } + final File[] files = tmpDir.listFiles(); + if (files == null) { + return false; + } + for (final File file : files) { + if (file.isDirectory() && !deleteDirectoryContents(file)) { + return false; + } if (!file.delete()) { return false; } @@ -195,6 +238,22 @@ private boolean deleteDir() { return tmpDir.delete(); } + private boolean deleteDirectoryContents(final File directory) { + final File[] files = directory.listFiles(); + if (files == null) { + return false; + } + for (final File file : files) { + if (file.isDirectory() && !deleteDirectoryContents(file)) { + return false; + } + if (!file.delete()) { + return false; + } + } + return true; + } + @Test public void testQueryFallbackHonorsDisabledSchemaCreation() throws Exception { final String database = "root.load.fallback_disabled_schema"; @@ -572,6 +631,7 @@ public void testLoadAcrossMultipleTimePartitions() throws Exception { } } + // Loading aligned and non-aligned devices extends the bound template and preserves every point. @Test public void testLoadWithExtendTemplate() throws Exception { final long writtenPoint1; @@ -610,6 +670,7 @@ public void testLoadWithExtendTemplate() throws Exception { statement.execute("create database root.sg.test_0"); statement.execute( "create device template t1 (lat FLOAT encoding=Gorilla, lon FLOAT encoding=Gorilla)"); + templateCreated = true; statement.execute(" set device template t1 to root.sg.test_0.d_0"); statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath())); @@ -738,9 +799,11 @@ public void testLoadWithAutoCreate() throws Exception { } } + // LOAD must reject missing SYSTEM/schema privileges and succeed after both grants are present. @Test public void testAuth() throws Exception { - createUser("test", "test123123456"); + createUser("test", "IoTDB!Test2024"); + testUserCreated = true; // device 0, device 1, sg 0 try (final TsFileGenerator generator = @@ -789,7 +852,7 @@ public void testAuth() throws Exception { String.format("load \"%s\" sgLevel=2", tmpDir.getAbsolutePath()), "No permissions for this operation, please add privilege WRITE_DATA", "test", - "test123123456"); + "IoTDB!Test2024"); grantUserSeriesPrivilege("test", PrivilegeType.WRITE_DATA, "root.**"); @@ -797,7 +860,7 @@ public void testAuth() throws Exception { String.format("load \"%s\" sgLevel=2", tmpDir.getAbsolutePath()), "No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456"); + "IoTDB!Test2024"); grantUserSystemPrivileges("test", PrivilegeType.SYSTEM); @@ -805,12 +868,12 @@ public void testAuth() throws Exception { String.format("load \"%s\" sgLevel=2", tmpDir.getAbsolutePath()), "Auto create or verify schema error when executing statement LoadTsFileStatement", "test", - "test123123456"); + "IoTDB!Test2024"); grantUserSystemPrivileges("test", PrivilegeType.WRITE_SCHEMA); executeNonQuery( - String.format("load \"%s\" sgLevel=2", tmpDir.getAbsolutePath()), "test", "test123123456"); + String.format("load \"%s\" sgLevel=2", tmpDir.getAbsolutePath()), "test", "IoTDB!Test2024"); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java index 757b514ff13ef..3b72df20e9221 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java @@ -26,8 +26,10 @@ import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -36,8 +38,8 @@ import java.sql.Connection; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; -import java.util.Locale; import static org.apache.iotdb.db.utils.constant.TestConstant.count; import static org.apache.iotdb.db.utils.constant.TestConstant.maxValue; @@ -85,17 +87,34 @@ public class IoTDBRecoverIT { private final String d0s2 = "root.vehicle.d0.s2"; private final String d0s3 = "root.vehicle.d0.s3"; - @Before - public void setUp() throws Exception { + @BeforeClass + public static void setUpCluster() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDownCluster() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Before + public void setUp() throws SQLException { + // Recreate the same file layout before each recovery on the shared cluster. prepareData(); } @After - public void tearDown() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); + public void tearDown() throws SQLException { + // Remove this class's databases so neither recovered data nor schemas reach the next test. + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("DELETE DATABASE root.vehicle.d0, root.vehicle.d1, root.ln.wf01.wt01"); + } } + /** + * Restart all nodes and verify filtered aggregates over recovered sequence and unsequence data. + */ @Test public void recoverTest1() { // stop cluster @@ -172,6 +191,7 @@ public void recoverTest1() { } } + /** Restart all nodes and verify recovered counts for overlapping writes and a time range. */ @Test public void recoverTest2() { // stop cluster @@ -227,7 +247,7 @@ public void recoverTest2() { } } - private void prepareData() { + private void prepareData() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -240,55 +260,52 @@ private void prepareData() { } // prepare BufferWrite file - String insertTemplate = - "INSERT INTO root.vehicle.d0(timestamp,s0,s1,s2,s3,s4)" + " VALUES(%d,%d,%d,%f,%s,%s)"; - for (int i = 5000; i < 7000; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); - } - statement.executeBatch(); + insertRange(statement, 5000, 7000, true); statement.execute("flush"); - for (int i = 7500; i < 8500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); - } - statement.executeBatch(); + insertRange(statement, 7500, 8500, false); statement.execute("flush"); // prepare Unseq-File - for (int i = 500; i < 1500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); - } - statement.executeBatch(); + insertRange(statement, 500, 1500, true); statement.execute("flush"); - for (int i = 3000; i < 6500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); - } - statement.executeBatch(); + insertRange(statement, 3000, 6500, false); statement.execute("flush"); // prepare BufferWrite cache - for (int i = 9000; i < 10000; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); - } - statement.executeBatch(); + insertRange(statement, 9000, 10000, true); // prepare Overflow cache - for (int i = 2000; i < 2500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); - } - statement.executeBatch(); + insertRange(statement, 2000, 2500, false); statement.execute("flush"); - } catch (Exception e) { - e.printStackTrace(); + } + } + + private static void insertRange( + Statement statement, int startTime, int endTime, boolean booleanValue) throws SQLException { + // Reduce SQL parsing and execution overhead without changing rows, write order or flush + // boundaries. + final int rowsPerInsert = 500; + for (int batchStart = startTime; batchStart < endTime; batchStart += rowsPerInsert) { + StringBuilder sql = + new StringBuilder("INSERT INTO root.vehicle.d0(timestamp,s0,s1,s2,s3,s4) VALUES"); + int batchEnd = Math.min(batchStart + rowsPerInsert, endTime); + for (int time = batchStart; time < batchEnd; time++) { + if (time > batchStart) { + sql.append(','); + } + sql.append('(') + .append(time) + .append(',') + .append(time) + .append(',') + .append(time) + .append(',') + .append((double) time) + .append(",'") + .append(time) + .append("',") + .append(booleanValue) + .append(')'); + } + statement.execute(sql.toString()); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverUnclosedIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverUnclosedIT.java index 6ea26e7bc7008..75b782e41c080 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverUnclosedIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverUnclosedIT.java @@ -33,12 +33,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.Locale; import static org.apache.iotdb.db.utils.constant.TestConstant.count; import static org.apache.iotdb.db.utils.constant.TestConstant.maxValue; @@ -94,8 +92,12 @@ public void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + /** + * Write overlapping ranges with synchronous WAL and no flush, restart all nodes, and verify the + * recovered 7500 timestamps and filtered extrema from single-row and batched writes. + */ @Test - public void test() throws SQLException, IOException { + public void test() throws SQLException { String[] retArray = new String[] {"2", "4", "3"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -147,13 +149,14 @@ public void test() throws SQLException, IOException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { String selectSql = "select count(*) from root.vehicle.d0"; - ResultSet tempResultSet = statement.executeQuery(selectSql); - assertNotNull(tempResultSet); - tempResultSet.next(); String d0s0 = "root.vehicle.d0.s0"; String d0s1 = "root.vehicle.d0.s1"; String d0s2 = "root.vehicle.d0.s2"; - assertEquals(7500, tempResultSet.getInt("count(" + d0s0 + ")")); + try (ResultSet tempResultSet = statement.executeQuery(selectSql)) { + assertNotNull(tempResultSet); + tempResultSet.next(); + assertEquals(7500, tempResultSet.getInt("count(" + d0s0 + ")")); + } // test max, min value retArray = new String[] {"8499,500.0", "2499,500.0"}; @@ -182,7 +185,7 @@ public void test() throws SQLException, IOException { } } - private void prepareData() { + private void prepareData() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { for (String sql : creationSqls) { @@ -192,59 +195,51 @@ private void prepareData() { for (String sql : dataSet2) { statement.execute(sql); } - - } catch (Exception e) { - e.printStackTrace(); } } - private void insertMoreData() { + private void insertMoreData() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { + // Keep every point and the original range order, without flushing before recovery. + insertRange(statement, 5000, 7000, true); + insertRange(statement, 7500, 8500, false); + insertRange(statement, 500, 1500, true); + insertRange(statement, 3000, 6500, false); + insertRange(statement, 9000, 10000, true); + insertRange(statement, 2000, 2500, false); + } + } - // prepare BufferWrite file - String insertTemplate = - "INSERT INTO root.vehicle.d0(timestamp,s0,s1,s2,s3,s4)" + " VALUES(%d,%d,%d,%f,%s,%s)"; - for (int i = 5000; i < 7000; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); - } - statement.executeBatch(); - for (int i = 7500; i < 8500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); + private static void insertRange( + Statement statement, int startTime, int endTime, boolean booleanValue) throws SQLException { + // Retain each range's first single-row INSERT to exercise the original WAL replay path. + // Batch the remaining rows to reduce SQL execution and synchronous WAL flush overhead. + final int rowsPerInsert = 500; + for (int batchStart = startTime; batchStart < endTime; ) { + int batchEnd = Math.min(batchStart + (batchStart == startTime ? 1 : rowsPerInsert), endTime); + StringBuilder sql = + new StringBuilder("INSERT INTO root.vehicle.d0(timestamp,s0,s1,s2,s3,s4) VALUES"); + for (int time = batchStart; time < batchEnd; time++) { + if (time > batchStart) { + sql.append(','); + } + sql.append('(') + .append(time) + .append(',') + .append(time) + .append(',') + .append(time) + .append(',') + .append((double) time) + .append(",'") + .append(time) + .append("',") + .append(booleanValue) + .append(')'); } - statement.executeBatch(); - // prepare Unseq-File - for (int i = 500; i < 1500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); - } - for (int i = 3000; i < 6500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); - } - statement.executeBatch(); - // prepare BufferWrite cache - for (int i = 9000; i < 10000; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); - } - statement.executeBatch(); - // prepare Overflow cache - for (int i = 2000; i < 2500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); - } - statement.executeBatch(); - } catch (Exception e) { - e.printStackTrace(); + statement.execute(sql.toString()); + batchStart = batchEnd; } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java index 302d059959a5a..c788bf94d9ad6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java @@ -34,7 +34,8 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; -import org.junit.Before; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -48,6 +49,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -66,8 +68,9 @@ public class IoTDBRestartIT { private final Logger logger = LoggerFactory.getLogger(IoTDBRestartIT.class); - @Before - public void setUp() throws Exception { + // Reuse the fixed recovery-test configuration while retaining each scenario's DataNode restarts. + @BeforeClass + public static void setUp() throws Exception { EnvFactory.getEnv() .getConfig() .getCommonConfig() @@ -87,8 +90,8 @@ public void setUp() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); } - @After - public void tearDown() throws Exception { + @AfterClass + public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); EnvFactory.getEnv().getConfig().getCommonConfig().setWalMode("ASYNC"); EnvFactory.getEnv() @@ -97,6 +100,24 @@ public void tearDown() throws Exception { .setSchemaRegionConsensusProtocolClass("org.apache.iotdb.consensus.simple.SimpleConsensus"); } + @After + public void cleanDatabases() throws SQLException { + // Clear only databases owned by this class so recovered schemas cannot leak between scenarios. + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + for (String database : Arrays.asList("root.turbine", "root.turbine1", "root.sg")) { + final boolean exists; + try (ResultSet resultSet = statement.executeQuery("show databases " + database)) { + exists = resultSet.next(); + } + if (exists) { + statement.execute("delete database " + database); + } + } + } + } + + /** Write flushed and unflushed points across two restarts, then verify all three timestamps. */ @Test public void testRestart() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -139,6 +160,9 @@ public void testRestart() throws SQLException { } } + /** + * Recover three points, delete the first and then the second, and verify only the third remains. + */ @Test public void testRestartDelete() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -183,6 +207,9 @@ public void testRestartDelete() throws SQLException { } } + /** + * Write points around two restarts and verify a query beyond the end time returns only time 4. + */ @Test public void testDeleteAndRewriteSurviveForcedRestart() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -334,6 +361,7 @@ public void testRestartQueryLargerThanEndTime() throws SQLException { } } + /** Add another series at earlier timestamps after recovery and verify both values survive. */ @Test public void testRestartEndTime() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -377,6 +405,9 @@ public void testRestartEndTime() throws SQLException { } } + /** + * Recreate a WAL-backed series with a different type and verify the other series' value recovers. + */ @Test public void testRecoverWALMismatchDataType() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -407,6 +438,9 @@ public void testRecoverWALMismatchDataType() throws Exception { } } + /** + * Delete one WAL-backed series before restart and verify the remaining series and row recover. + */ @Test public void testRecoverWALDeleteSchema() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -434,6 +468,7 @@ public void testRecoverWALDeleteSchema() throws Exception { } } + /** Flush one series, leave another in the WAL, and verify the flushed values after restart. */ @Test public void testRecoverWALDeleteSchemaCheckResourceTime() throws Exception { IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); @@ -474,6 +509,7 @@ public void testRecoverWALDeleteSchemaCheckResourceTime() throws Exception { config.setUnSeqTsFileSize(unFsFileSize); } + /** Restart after inserting two unflushed values and verify both values are recovered. */ @Test public void testRecoverFromFlushMemTableError() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -501,6 +537,9 @@ public void testRecoverFromFlushMemTableError() throws Exception { } } + /** + * Load overlapping TsFile data, force-stop DataNodes, and verify both timestamps after recovery. + */ @Test public void testInsertLoadAndRecover() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartRatisIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartRatisIT.java index 22f5234ee9239..4a3f05242128d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartRatisIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartRatisIT.java @@ -28,7 +28,8 @@ import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.junit.After; -import org.junit.Before; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -39,6 +40,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; import static org.apache.iotdb.db.utils.constant.TestConstant.TIMESTAMP_STR; import static org.junit.Assert.*; @@ -49,8 +51,9 @@ public class IoTDBRestartRatisIT { private final Logger logger = LoggerFactory.getLogger(IoTDBRestartRatisIT.class); - @Before - public void setUp() throws Exception { + // Share the fixed Ratis/WAL configuration; each test still performs its own recovery restarts. + @BeforeClass + public static void setUp() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setWalMode("SYNC"); EnvFactory.getEnv() .getConfig() @@ -60,8 +63,8 @@ public void setUp() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); } - @After - public void tearDown() throws Exception { + @AfterClass + public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); EnvFactory.getEnv().getConfig().getCommonConfig().setWalMode("ASYNC"); EnvFactory.getEnv() @@ -70,6 +73,27 @@ public void tearDown() throws Exception { .setSchemaRegionConsensusProtocolClass("org.apache.iotdb.consensus.simple.SimpleConsensus"); } + @After + public void cleanDatabases() throws SQLException { + // Remove old schemas and data so each recovery scenario starts from empty databases. + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + for (String database : Arrays.asList("root.turbine", "root.turbine1")) { + final boolean exists; + try (ResultSet resultSet = statement.executeQuery("show databases " + database)) { + exists = resultSet.next(); + } + if (exists) { + statement.execute("delete database " + database); + } + } + } + } + + /** + * Write flushed and unflushed points across two DataNode restarts, then verify that querying the + * series still returns the values at all three timestamps. + */ @Test public void testRestart() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -112,6 +136,10 @@ public void testRestart() throws SQLException { } } + /** + * Recover three unflushed points, delete the earliest point, then flush and delete another point; + * verify that each query returns only the remaining timestamps and values. + */ @Test public void testRestartDelete() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -156,6 +184,10 @@ public void testRestartDelete() throws SQLException { } } + /** + * Insert points on both sides of a restart, restart again, and query beyond the earlier end time; + * verify that only the point at timestamp four is returned. + */ @Test public void testRestartQueryLargerThanEndTime() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -203,6 +235,10 @@ public void testRestartQueryLargerThanEndTime() throws SQLException { } } + /** + * Add a second measurement at existing timestamps between two DataNode restarts, then verify that + * its recovered end time includes both points at timestamps one and two. + */ @Test public void testRestartEndTime() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -246,6 +282,10 @@ public void testRestartEndTime() throws SQLException { } } + /** + * Delete a populated measurement and recreate it with an incompatible INT32 type before restart; + * verify that recovery leaves it null while preserving the other measurement's value. + */ @Test public void testRecoverWALMismatchDataType() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -276,6 +316,10 @@ public void testRecoverWALMismatchDataType() throws Exception { } } + /** + * Delete one of two populated measurements before restarting DataNodes, then verify that recovery + * excludes the deleted column and preserves the surviving measurement and timestamp. + */ @Test public void testRecoverWALDeleteSchema() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -303,6 +347,10 @@ public void testRecoverWALDeleteSchema() throws Exception { } } + /** + * Flush an INT64 measurement and append a BOOLEAN measurement at later times before restarting; + * verify that a query bounded below time three recovers exactly the two original timestamps. + */ @Test public void testRecoverWALDeleteSchemaCheckResourceTime() throws Exception { IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); @@ -343,6 +391,10 @@ public void testRecoverWALDeleteSchemaCheckResourceTime() throws Exception { config.setUnSeqTsFileSize(unFsFileSize); } + /** + * Restart DataNodes after an unflushed row with two measurements, then verify that recovery + * preserves the row's timestamp and both values without requiring an explicit flush. + */ @Test public void testRecoverFromFlushMemTableError() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java index 599f2cb86113a..b179d7165a34d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java @@ -19,13 +19,16 @@ package org.apache.iotdb.db.it; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -50,6 +53,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -1238,26 +1242,72 @@ public void testUnseqUnsealedDeleteQuery() throws SQLException { } } + // Query the unwritten s0 after flushing data from 9,999 sibling series in a device with 10,000 + // series. Loading its metadata must succeed and return an empty result. @Test - public void testTimeseriesMetadataCache() { + public void testTimeseriesMetadataCache() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { try { statement.execute("CREATE DATABASE root.sg1"); - for (int i = 0; i < 10000; i++) { - statement.execute( - "CREATE TIMESERIES root.sg1.d0.s" + i + " WITH DATATYPE=INT32,ENCODING=PLAIN"); + final int timeseriesCount = 10000; + final int batchSize = 1000; + // Bulk schema creation preserves the full non-aligned schema needed by the cache + // regression. + // Reuse one Session for bounded batches with the original SQL's default compressor. + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + for (int start = 0; start < timeseriesCount; start += batchSize) { + List paths = new ArrayList<>(batchSize); + for (int i = start; i < Math.min(start + batchSize, timeseriesCount); i++) { + paths.add("root.sg1.d0.s" + i); + } + session.createMultiTimeseries( + paths, + Collections.nCopies(paths.size(), TSDataType.INT32), + Collections.nCopies(paths.size(), TSEncoding.PLAIN), + Collections.nCopies( + paths.size(), + TSFileDescriptor.getInstance().getConfig().getCompressor(TSDataType.INT32)), + null, + null, + null, + null); + } } - for (int i = 1; i < 10000; i++) { - statement.execute("INSERT INTO root.sg1.d0(timestamp, s" + i + ") VALUES (1000, 1)"); + try (ResultSet resultSet = statement.executeQuery("count timeseries root.sg1.d0.*")) { + assertTrue(resultSet.next()); + assertEquals(timeseriesCount, resultSet.getLong(1)); + assertFalse(resultSet.next()); + } + // Every written series has one point at the same timestamp. Grouping these distinct + // measurements preserves all 9,999 points and the original flush boundary. + for (int start = 1; start < timeseriesCount; start += batchSize) { + StringBuilder insert = new StringBuilder("INSERT INTO root.sg1.d0(timestamp"); + StringBuilder values = new StringBuilder(" VALUES (1000"); + for (int i = start; i < Math.min(start + batchSize, timeseriesCount); i++) { + insert.append(",s").append(i); + values.append(",1"); + } + statement.execute(insert.append(')').append(values).append(')').toString()); } statement.execute("flush"); - statement.executeQuery("select s0 from root.sg1.d0"); + try (ResultSet resultSet = statement.executeQuery("select s0 from root.sg1.d0")) { + assertFalse(resultSet.next()); + } + try (ResultSet resultSet = + statement.executeQuery("select s1, s1000, s1001, s9999 from root.sg1.d0")) { + assertTrue(resultSet.next()); + assertEquals(1000, resultSet.getLong(1)); + for (int i = 2; i <= 5; i++) { + assertEquals(1, resultSet.getInt(i)); + } + assertFalse(resultSet.next()); + } } finally { executeQuietly(statement, "DELETE DATABASE root.sg1"); } } catch (SQLException e) { - fail(); + fail(e.getMessage()); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBTtlIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBTtlIT.java index d2dd50ebcf5c9..278c2aa654658 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBTtlIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBTtlIT.java @@ -25,6 +25,7 @@ import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.rpc.TSStatusCode; +import org.awaitility.Awaitility; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -35,6 +36,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.*; @@ -52,8 +54,10 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + // TTL rejects expired inserts and filters old rows; after UNSET, the same old timestamps become + // writable on both database paths. @Test - public void testTTL() throws SQLException, InterruptedException { + public void testTTL() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { try { @@ -138,14 +142,11 @@ public void testTTL() throws SQLException, InterruptedException { } } - Thread.sleep(1000); - try (ResultSet resultSet = statement.executeQuery("SELECT s1 FROM root.TTL_SG1")) { - int cnt = 0; - while (resultSet.next()) { - cnt++; - } - assertEquals(100, cnt); - } + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> assertEquals(100, countRows(statement, "SELECT s1 FROM root.TTL_SG1"))); statement.execute("UNSET TTL TO root.TTL_SG1"); // UNSET TTL waits for the procedure to update all DataNode TTL caches. @@ -271,4 +272,14 @@ public void testTTL() throws SQLException, InterruptedException { } } } + + private static int countRows(Statement statement, String sql) throws SQLException { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int count = 0; + while (resultSet.next()) { + count++; + } + return count; + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBWriteMemoryLifecycleIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBWriteMemoryLifecycleIT.java index 6f743ee307397..4785f3377491d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBWriteMemoryLifecycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBWriteMemoryLifecycleIT.java @@ -33,7 +33,9 @@ import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -58,12 +60,16 @@ @Category(LocalStandaloneIT.class) public class IoTDBWriteMemoryLifecycleIT { private static final String DATABASE = "root.write_memory"; + private static final String TABLE_DATABASE = "write_memory_table"; private static final String[] DEVICES = {DATABASE + ".plain", DATABASE + ".aligned"}; private static final String OVERSIZED_TEXT = "x".repeat(32 * 1024); private final Map expected = new TreeMap<>(); + // Tree-mode DELETE DATABASE requires an existing database created by this fixture. + private boolean treeDatabaseCreated; - @Before - public void setUp() throws Exception { + @BeforeClass + public static void setUpClass() throws Exception { + // Node settings are shared by all scenarios; each test resets its databases in @Before. EnvFactory.getEnv() .getConfig() .getCommonConfig() @@ -95,8 +101,14 @@ public void setUp() throws Exception { .getDataNodeConfig() .setMetricReporterType(Collections.singletonList("PROMETHEUS")); EnvFactory.getEnv().initClusterEnvironment(); + } + + @Before + public void setUp() throws Exception { + cleanTestDatabases(); try (ISession session = EnvFactory.getEnv().getSessionConnection()) { session.executeNonQueryStatement("CREATE DATABASE " + DATABASE); + treeDatabaseCreated = true; session.executeNonQueryStatement("CREATE TIMESERIES " + DEVICES[0] + ".n INT64"); session.executeNonQueryStatement("CREATE TIMESERIES " + DEVICES[0] + ".t TEXT"); session.executeNonQueryStatement( @@ -106,9 +118,27 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { + cleanTestDatabases(); + } + + @AfterClass + public static void tearDownClass() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + private void cleanTestDatabases() throws Exception { + if (treeDatabaseCreated) { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("DELETE DATABASE " + DATABASE); + } + treeDatabaseCreated = false; + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("DROP DATABASE IF EXISTS " + TABLE_DATABASE); + } + } + @Test public void testArrayBoundariesSparseColumnsAndRepeatedFlush() throws Exception { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { @@ -241,13 +271,14 @@ public void testCrashRecoveryThenWriteAndRelease() throws Exception { } } + // Two-device writes cross a partition boundary; overwrites retain values and flush releases + // memory. @Test public void testTableTabletSplitAcrossDevicesAndTimePartitions() throws Exception { - String database = "write_memory_table"; try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { - statement.execute("CREATE DATABASE " + database); - statement.execute("USE " + database); + statement.execute("CREATE DATABASE " + TABLE_DATABASE); + statement.execute("USE " + TABLE_DATABASE); statement.execute("CREATE TABLE readings(device STRING TAG, n INT64 FIELD, t STRING FIELD)"); // Each request contains two devices and spans the 1000 ms partition boundary. for (int cycle = 0; cycle < 2; cycle++) { @@ -269,7 +300,8 @@ public void testTableTabletSplitAcrossDevicesAndTimePartitions() throws Exceptio } } statement.execute(sql.toString()); - assertTrue("All split processors must account for live writes", memoryUsage(database) > 0); + assertTrue( + "All split processors must account for live writes", memoryUsage(TABLE_DATABASE) > 0); // The second cycle overwrites flushed data, exercising unsequence ranges in both // partitions. for (int phase = 0; phase < 2; phase++) { @@ -291,7 +323,7 @@ public void testTableTabletSplitAcrossDevicesAndTimePartitions() throws Exceptio statement.execute("FLUSH"); await() .atMost(30, TimeUnit.SECONDS) - .untilAsserted(() -> assertEquals(0, memoryUsage(database))); + .untilAsserted(() -> assertEquals(0, memoryUsage(TABLE_DATABASE))); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java index 524d0770a6c07..2985721e019cf 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java @@ -19,11 +19,18 @@ package org.apache.iotdb.db.it.aggregation; +import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -35,7 +42,9 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.Locale; +import java.time.LocalDate; +import java.util.HexFormat; +import java.util.List; import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualWithDescOrderTest; @@ -115,9 +124,6 @@ public class IoTDBAggregationIT { private final String d0s6 = "root.vehicle.d0.s6"; private final String d0s7 = "root.vehicle.d0.s7"; private final String d0s8 = "root.vehicle.d0.s8"; - private static final String insertTemplate = - "INSERT INTO root.vehicle.d0(timestamp,s0,s1,s2,s3,s4,s5,s6,s7,s8)" - + " VALUES(%d,%d,%d,%f,%s,%s,%s,%s,%s,%d)"; @BeforeClass public static void setUp() throws Exception { @@ -991,154 +997,51 @@ public void descAggregationWithUnseqData() { } private static void prepareData() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + // Reuse one session for schema creation, typed batch writes and flushes. These tests exercise + // aggregation queries, so preparing every row through the SQL parser adds no query coverage. + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { for (String sql : creationSqls) { - statement.execute(sql); + session.executeNonQueryStatement(sql); } + Tablet tablet = + new Tablet( + "root.vehicle.d0", + List.of( + new MeasurementSchema("s0", TSDataType.INT32), + new MeasurementSchema("s1", TSDataType.INT64), + new MeasurementSchema("s2", TSDataType.FLOAT), + new MeasurementSchema("s3", TSDataType.TEXT), + new MeasurementSchema("s4", TSDataType.BOOLEAN), + new MeasurementSchema("s5", TSDataType.STRING), + new MeasurementSchema("s6", TSDataType.BLOB), + new MeasurementSchema("s7", TSDataType.DATE), + new MeasurementSchema("s8", TSDataType.TIMESTAMP)), + 500); + // prepare BufferWrite file - for (int i = 5000; i < 7000; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, - insertTemplate, - i, - i, - i, - (double) i, - "'" + i + "'", - "true", - "'" + i + "'", - "X'" + i + "'", - "'" + i + "-01-01'", - i)); - } - statement.executeBatch(); - statement.clearBatch(); - statement.execute("flush"); - - for (int i = 7500; i < 8500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, - insertTemplate, - i, - i, - i, - (double) i, - "'" + i + "'", - "false", - "'" + i + "'", - "X'" + i + "'", - "'" + i + "-01-01'", - i)); - } - statement.executeBatch(); - statement.clearBatch(); - statement.execute("flush"); + insertRange(session, tablet, 5000, 7000, true); + session.executeNonQueryStatement("flush"); + + insertRange(session, tablet, 7500, 8500, false); + session.executeNonQueryStatement("flush"); // prepare Unseq-File - for (int i = 500; i < 1500; i++) { - if (i < 1000) { - statement.addBatch( - String.format( - Locale.ENGLISH, - insertTemplate, - i, - i, - i, - (double) i, - "'" + i + "'", - "true", - "'" + i + "'", - "X'0" + i + "'", - "'1" + i + "-01-01'", - i)); - } else { - statement.addBatch( - String.format( - Locale.ENGLISH, - insertTemplate, - i, - i, - i, - (double) i, - "'" + i + "'", - "true", - "'" + i + "'", - "X'" + i + "'", - "'" + i + "-01-01'", - i)); - } - } - statement.executeBatch(); - statement.clearBatch(); - statement.execute("flush"); - for (int i = 3000; i < 6500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, - insertTemplate, - i, - i, - i, - (double) i, - "'" + i + "'", - "false", - "'" + i + "'", - "X'" + i + "'", - "'" + i + "-01-01'", - i)); - } - statement.executeBatch(); - statement.clearBatch(); + insertRange(session, tablet, 500, 1500, true); + session.executeNonQueryStatement("flush"); + insertRange(session, tablet, 3000, 6500, false); // prepare BufferWrite cache - for (int i = 9000; i < 10000; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, - insertTemplate, - i, - i, - i, - (double) i, - "'" + i + "'", - "true", - "'" + i + "'", - "X'" + i + "'", - "'" + i + "-01-01'", - i)); - } - statement.executeBatch(); - statement.clearBatch(); + insertRange(session, tablet, 9000, 10000, true); // prepare Overflow cache - for (int i = 2000; i < 2500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, - insertTemplate, - i, - i, - i, - (double) i, - "'" + i + "'", - "false", - "'" + i + "'", - "X'" + i + "'", - "'" + i + "-01-01'", - i)); - } - statement.executeBatch(); - statement.clearBatch(); + insertRange(session, tablet, 2000, 2500, false); for (String sql : dataSet3) { - statement.execute(sql); + session.executeNonQueryStatement(sql); } for (String sql : dataSet2) { - statement.execute(sql); + session.executeNonQueryStatement(sql); } } catch (Exception e) { e.printStackTrace(); @@ -1146,6 +1049,36 @@ private static void prepareData() { } } + private static void insertRange( + ISession session, Tablet tablet, int startTime, int endTime, boolean booleanValue) + throws IoTDBConnectionException, StatementExecutionException { + // Reuse the bounded tablet while retaining row order and the callers' flush boundaries. + for (int time = startTime; time < endTime; time++) { + int row = tablet.getRowSize(); + tablet.addTimestamp(row, time); + tablet.addValue("s0", row, time); + tablet.addValue("s1", row, (long) time); + tablet.addValue("s2", row, (float) time); + String text = Integer.toString(time); + tablet.addValue("s3", row, text); + tablet.addValue("s4", row, booleanValue); + tablet.addValue("s5", row, text); + // SQL used decimal digits as hex, e.g. X'0500', and DATE years 1500-1999 for times 500-999. + tablet.addValue( + "s6", row, new Binary(HexFormat.of().parseHex((time < 1000 ? "0" : "") + text))); + tablet.addValue("s7", row, LocalDate.of(time < 1000 ? time + 1000 : time, 1, 1)); + tablet.addValue("s8", row, (long) time); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insertTablet(tablet); + tablet.reset(); + } + } + if (tablet.getRowSize() > 0) { + session.insertTablet(tablet); + tablet.reset(); + } + } + @Test public void noDataRegionTest() { String[] expectedHeader = diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedDataDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedDataDeletionIT.java index 0dfb916b37895..884ec83c6769b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedDataDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedDataDeletionIT.java @@ -26,8 +26,10 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -46,9 +48,14 @@ @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBAlignedDataDeletionIT { + // Missing paths make DELETE TIMESERIES fail, so cleanup tracks which non-fixture paths each test + // creates. + private boolean nestedDatabaseCreated; + private boolean lnSeriesCreated; + private boolean lzSeriesCreated; + private static String[] creationSqls = new String[] { - "CREATE DATABASE root.vehicle", "CREATE ALIGNED TIMESERIES root.vehicle.d0(s0 INT32 ENCODING=RLE, s1 INT64 ENCODING=RLE, s2 FLOAT ENCODING=RLE, s3 TEXT ENCODING=PLAIN, s4 BOOLEAN ENCODING=PLAIN)", }; @@ -57,8 +64,9 @@ public class IoTDBAlignedDataDeletionIT { + ") ALIGNED VALUES(%d,%d,%d,%f,%s,%b)"; private String deleteAllTemplate = "DELETE FROM root.vehicle.d0.* WHERE time <= 10000"; - @Before - public void setUp() throws Exception { + // Reuse the shared test cluster; recreate the per-method schema and remove each method's paths. + @BeforeClass + public static void setUpClass() throws Exception { Locale.setDefault(Locale.ENGLISH); EnvFactory.getEnv() .getConfig() @@ -67,11 +75,39 @@ public void setUp() throws Exception { // Adjust memstable threshold size to make it flush automatically .setMemtableSizeThreshold(10000); EnvFactory.getEnv().initClusterEnvironment(); + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.vehicle"); + } + } + + @Before + public void setUp() { + nestedDatabaseCreated = false; + lnSeriesCreated = false; + lzSeriesCreated = false; prepareSeries(); } @After public void tearDown() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + if (nestedDatabaseCreated) { + statement.execute("DROP DATABASE root.ln.wf01.wt01"); + } + statement.execute("DELETE TIMESERIES root.vehicle.**"); + if (lnSeriesCreated) { + statement.execute("DELETE TIMESERIES root.ln.d1.**"); + } + if (lzSeriesCreated) { + statement.execute("DELETE TIMESERIES root.lz.dev.GPS.**"); + } + } + } + + @AfterClass + public static void tearDownClass() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } @@ -80,6 +116,7 @@ public void tearDown() throws Exception { * * @throws SQLException */ + // Reject value-filter predicates for three data types and preserve the matching rows. @Test public void testUnsupportedValueFilter() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -141,6 +178,8 @@ public void testUnsupportedValueFilter() throws SQLException { } } + // Delete inclusive and overlapping time ranges across aligned columns; verify 250, 100, and 50 + // rows remain. @Test public void test() throws SQLException { prepareData(); @@ -180,11 +219,13 @@ public void test() throws SQLException { cleanData(); } + // Delete through NOW before and after flush; the query must return no remaining rows. @Test public void testDelAfterFlush() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute("CREATE DATABASE root.ln.wf01.wt01"); + nestedDatabaseCreated = true; statement.execute( "CREATE TIMESERIES root.ln.wf01.wt01.status WITH DATATYPE=BOOLEAN," + " ENCODING=PLAIN"); statement.execute( @@ -201,6 +242,8 @@ public void testDelAfterFlush() throws SQLException { } } + // Combine single-column and aligned-range deletes with overlapping bounds; verify each resulting + // row count. @Test public void testRangeDelete() throws SQLException { prepareData(); @@ -237,6 +280,7 @@ public void testRangeDelete() throws SQLException { cleanData(); } + // Delete an entire extended column from sparse aligned rows without returning an error. @Test public void testDeleteAfterColumnExtended() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -254,6 +298,7 @@ public void testDeleteAfterColumnExtended() throws SQLException { } } + // Delete all values from one aligned column without a time predicate; its query must be empty. @Test public void testFullDeleteWithoutWhereClause() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -270,6 +315,8 @@ public void testFullDeleteWithoutWhereClause() throws SQLException { } } + // Delete through partial measurement and device paths; verify the remaining counts are 250 and + // 150. @Test public void testPartialPathRangeDelete() throws SQLException { prepareData(); @@ -297,6 +344,8 @@ public void testPartialPathRangeDelete() throws SQLException { cleanData(); } + // Insert 10,000 aligned rows and delete (1500,9000]; 2,500 values must remain in the target + // column. @Test public void testDelFlushingMemtable() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -319,6 +368,8 @@ public void testDelFlushingMemtable() throws SQLException { } } + // Delete several ranges across two batches and a flush; 1,100 values must remain in the target + // column. @Test public void testDelMultipleFlushingMemtable() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -351,12 +402,15 @@ public void testDelMultipleFlushingMemtable() throws SQLException { } } + // Delete a quoted measurement containing a comma by range and then fully; counts must go from two + // to zero. @Test public void testDelSeriesWithSpecialSymbol() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute( "CREATE TIMESERIES root.ln.d1.`status,01` WITH DATATYPE=BOOLEAN, ENCODING=PLAIN"); + lnSeriesCreated = true; statement.execute("INSERT INTO root.ln.d1(timestamp,`status,01`) VALUES(300, true)"); statement.execute("INSERT INTO root.ln.d1(timestamp,`status,01`) VALUES(500, false)"); @@ -390,12 +444,15 @@ public void testDelSeriesWithSpecialSymbol() throws SQLException { } } + // Insert partial aligned rows at a duplicate timestamp, then delete latitude; both timestamps + // must remain. @Test public void testInsertDuplicatedTimeThenDel() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute( "CREATE ALIGNED TIMESERIES root.lz.dev.GPS(latitude FLOAT encoding=PLAIN compressor=SNAPPY, longitude FLOAT encoding=PLAIN compressor=SNAPPY)"); + lzSeriesCreated = true; statement.execute( "insert into root.lz.dev.GPS(time, latitude, longitude) aligned values(9,3.2,9.8)"); statement.execute("insert into root.lz.dev.GPS(time, latitude) aligned values(11,4.5)"); @@ -421,12 +478,15 @@ public void testInsertDuplicatedTimeThenDel() throws SQLException { } } + // Delete every series under root.lz after aligned inserts; the subsequent query must return no + // rows. @Test public void testDeleteAll() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute( "CREATE ALIGNED TIMESERIES root.lz.dev.GPS(latitude FLOAT encoding=PLAIN compressor=SNAPPY, longitude FLOAT encoding=PLAIN compressor=SNAPPY)"); + lzSeriesCreated = true; statement.execute( "insert into root.lz.dev.GPS(time, latitude, longitude) aligned values(9,3.2,9.8)"); statement.execute("insert into root.lz.dev.GPS(time, latitude) aligned values(11,4.5)"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java index 1d451edf9730b..a8c4f743de074 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java @@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableList; import org.junit.*; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.sql.BatchUpdateException; @@ -62,16 +63,167 @@ @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBAuthIT { + private static final List ROOT_PASSWORD_CANDIDATES = + Arrays.asList("root", "newPassword666888", "rootHasANewPassword123"); + + private static boolean clusterStarted; + + @Rule public final TestName testName = new TestName(); @Before public void setUp() throws Exception { - EnvFactory.getEnv().getConfig().getCommonConfig().setEnforceStrongPassword(false); - EnvFactory.getEnv().initClusterEnvironment(); + if (clusterStarted && requiresFreshAuthState(testName.getMethodName())) { + EnvFactory.getEnv().cleanClusterEnvironment(); + clusterStarted = false; + } + if (!clusterStarted) { + EnvFactory.getEnv().getConfig().getCommonConfig().setEnforceStrongPassword(false); + EnvFactory.getEnv().initClusterEnvironment(); + clusterStarted = true; + } + } + + private static boolean requiresFreshAuthState(String methodName) { + return "allPrivilegesTest".equals(methodName) + || "testListUser".equals(methodName) + || "testListRoleUsers".equals(methodName) + || "testListUserPrivilege".equals(methodName); } @After public void tearDown() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); + if (testName.getMethodName().startsWith("testClusterManagementSql")) { + try { + EnvFactory.getEnv().cleanClusterEnvironment(); + } finally { + clusterStarted = false; + } + return; + } + + try { + cleanupAfterTest(); + } catch (Exception e) { + try { + EnvFactory.getEnv().cleanClusterEnvironment(); + } catch (Exception cleanupException) { + e.addSuppressed(cleanupException); + } finally { + clusterStarted = false; + } + throw e; + } + } + + @AfterClass + public static void tearDownClass() throws Exception { + if (clusterStarted) { + try { + EnvFactory.getEnv().cleanClusterEnvironment(); + } finally { + clusterStarted = false; + } + } + } + + private static void cleanupAfterTest() throws SQLException { + try (Connection connection = getRootConnection(); + Statement statement = connection.createStatement()) { + // Restore credentials before the next test; testChangeBackPassword intentionally changes + // root. + statement.execute("ALTER USER root SET PASSWORD 'root'"); + + List users = new ArrayList<>(); + try (ResultSet resultSet = statement.executeQuery("LIST USER")) { + while (resultSet.next()) { + String user = resultSet.getString(2); + if (!"root".equals(user)) { + users.add(user); + } + } + } + for (String user : users) { + statement.execute("DROP USER " + user); + } + + List roles = new ArrayList<>(); + try (ResultSet resultSet = statement.executeQuery("LIST ROLE")) { + while (resultSet.next()) { + roles.add(resultSet.getString(1)); + } + } + for (String role : roles) { + statement.execute("DROP ROLE " + role); + } + + List databases = new ArrayList<>(); + try (ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) { + while (resultSet.next()) { + String database = resultSet.getString(1); + if (database != null + && !database.startsWith("root.__") + && !database.startsWith("information_schema")) { + databases.add(database); + } + } + } + for (String database : databases) { + statement.execute("DELETE DATABASE " + database); + } + + List timeseries = new ArrayList<>(); + try (ResultSet resultSet = statement.executeQuery("SHOW TIMESERIES root.**")) { + while (resultSet.next()) { + String path = resultSet.getString(1); + if (path != null && !path.startsWith("root.__")) { + timeseries.add(path); + } + } + } + for (String path : timeseries) { + statement.execute("DELETE TIMESERIES " + path); + } + + unsetDeviceTemplatePaths(statement, "t1"); + executeQuietly(statement, "DROP DEVICE TEMPLATE t1"); + executeQuietly(statement, "DROP SCHEMA TEMPLATE t1"); + } + } + + private static Connection getRootConnection() throws SQLException { + SQLException lastException = null; + for (String password : ROOT_PASSWORD_CANDIDATES) { + try { + return EnvFactory.getEnv().getConnection("root", password); + } catch (SQLException e) { + lastException = e; + } + } + throw lastException; + } + + private static void unsetDeviceTemplatePaths(Statement statement, String template) + throws SQLException { + List paths = new ArrayList<>(); + try (ResultSet resultSet = + statement.executeQuery("SHOW PATHS SET DEVICE TEMPLATE " + template)) { + while (resultSet.next()) { + paths.add(resultSet.getString(1)); + } + } catch (SQLException ignored) { + return; + } + for (String path : paths) { + statement.execute("UNSET DEVICE TEMPLATE " + template + " FROM " + path); + } + } + + private static void executeQuietly(Statement statement, String sql) { + try { + statement.execute(sql); + } catch (SQLException ignored) { + // The template may not have been created by this test. + } } @Test @@ -1517,12 +1669,15 @@ public void testStrongPassword() throws SQLException { } } + // With password policy disabled, changing the administrator password twice permits restoring + // root. @Test public void testChangeBackPassword() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute("ALTER USER root SET PASSWORD 'newPassword666888'"); statement.execute("ALTER USER root SET PASSWORD 'rootHasANewPassword123'"); + statement.execute("ALTER USER root SET PASSWORD 'root'"); } catch (SQLException e) { e.printStackTrace(); fail(e.getMessage()); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java index db713a9aa6f69..651706162388f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java @@ -28,9 +28,11 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.itbase.runtime.ClusterTestResultSet; import org.apache.iotdb.mpp.rpc.thrift.TExecuteCQ; import org.apache.iotdb.rpc.TSStatusCode; +import org.awaitility.Awaitility; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -55,6 +57,9 @@ @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBCQExecIT { + private static final long CQ_RESULT_POLL_INTERVAL_IN_MS = 100; + private static final long CQ_RESULT_TIMEOUT_IN_SECONDS = 30; + @BeforeClass public static void setUp() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); @@ -65,21 +70,25 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + // A wildcard CQ copies only readable fields, observes revoke/grant, and rejects a deleted owner. @Test public void testCQExecutionWithReadDataPermission() throws Exception { testCQExecutionWithReadDataPermission(false, false); } + // An aligned wildcard CQ filters unreadable fields and rechecks owner permissions per window. @Test public void testCQExecutionWithReadDataPermissionAlignByDevice() throws Exception { testCQExecutionWithReadDataPermission(true, false); } + // Explicit CQ targets reject missing source access and succeed when that access is restored. @Test public void testCQExecutionWithExplicitTargetReadDataPermission() throws Exception { testCQExecutionWithReadDataPermission(false, true); } + // Aligned explicit CQ targets retain positional mapping across permission revoke/grant retries. @Test public void testCQExecutionWithExplicitTargetReadDataPermissionAlignByDevice() throws Exception { testCQExecutionWithReadDataPermission(true, true); @@ -264,6 +273,8 @@ public void close() throws SQLException { } } + // A 2s resampling CQ must write the expected maximum for every 1s window after its future + // boundary. @Test public void testCQExecution1() { String insertTemplate = @@ -322,25 +333,22 @@ public void testCQExecution1() { return; } - long targetTime = firstExecutionTime + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = { startTime + 1_000, startTime + 2_000, startTime + 3_000, startTime + 4_000 }; long[] expectedValue = {4, 6, 8, 10}; - try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.d1")) { - int cnt = 0; - while (resultSet.next()) { - assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); - assertEquals(expectedValue[cnt], resultSet.getLong("root.sg.d1.s1_max")); - cnt++; + try { + awaitCQResultCount(statement, "select s1_max from root.sg.d1", expectedTime.length); + try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.d1")) { + int cnt = 0; + while (resultSet.next()) { + assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); + assertEquals(expectedValue[cnt], resultSet.getLong("root.sg.d1.s1_max")); + cnt++; + } + assertEquals(expectedTime.length, cnt); } - assertEquals(expectedTime.length, cnt); } finally { statement.execute("DROP CQ cq1"); } @@ -351,6 +359,7 @@ public void testCQExecution1() { } } + // A CQ without EVERY and with a 4s range must produce the five expected 1s-window maxima. @Test public void testCQExecution2() { String insertTemplate = @@ -410,25 +419,22 @@ public void testCQExecution2() { return; } - long targetTime = firstExecutionTime + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = { startTime, startTime + 1_000, startTime + 2_000, startTime + 3_000, startTime + 4_000 }; long[] expectedValue = {2, 4, 6, 8, 10}; - try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.d2")) { - int cnt = 0; - while (resultSet.next()) { - assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); - assertEquals(expectedValue[cnt], resultSet.getLong("root.sg.d2.s1_max")); - cnt++; + try { + awaitCQResultCount(statement, "select s1_max from root.sg.d2", expectedTime.length); + try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.d2")) { + int cnt = 0; + while (resultSet.next()) { + assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); + assertEquals(expectedValue[cnt], resultSet.getLong("root.sg.d2.s1_max")); + cnt++; + } + assertEquals(expectedTime.length, cnt); } - assertEquals(expectedTime.length, cnt); } finally { statement.execute("DROP CQ cq2"); } @@ -439,6 +445,7 @@ public void testCQExecution2() { } } + // A 4s range with FILL must materialize the missing leading 1s window and retain later maxima. @Test public void testCQExecution3() { String insertTemplate = @@ -499,12 +506,6 @@ public void testCQExecution3() { return; } - long targetTime = firstExecutionTime + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = { startTime - 1_000, startTime, @@ -515,19 +516,22 @@ public void testCQExecution3() { }; long[] expectedValue = {100, 2, 4, 6, 8, 10}; - try (ResultSet resultSet = - statement.executeQuery( - "select s1_max from root.sg.d3 where time between " - + (startTime - 1_000) - + " and " - + (startTime + 4_000))) { - int cnt = 0; - while (resultSet.next()) { - assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); - assertEquals(expectedValue[cnt], resultSet.getLong("root.sg.d3.s1_max")); - cnt++; + String query = + "select s1_max from root.sg.d3 where time between " + + (startTime - 1_000) + + " and " + + (startTime + 4_000); + try { + awaitCQResultCount(statement, query, expectedTime.length); + try (ResultSet resultSet = statement.executeQuery(query)) { + int cnt = 0; + while (resultSet.next()) { + assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); + assertEquals(expectedValue[cnt], resultSet.getLong("root.sg.d3.s1_max")); + cnt++; + } + assertEquals(expectedTime.length, cnt); } - assertEquals(expectedTime.length, cnt); } finally { statement.execute("DROP CQ cq3"); } @@ -538,6 +542,7 @@ public void testCQExecution3() { } } + // RANGE 2s, 1s must select two separated 1s windows as resampling advances. @Test public void testCQExecution4() { String insertTemplate = @@ -597,23 +602,20 @@ public void testCQExecution4() { return; } - long targetTime = firstExecutionTime + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = {startTime + 1_000, startTime + 3_000}; long[] expectedValue = {4, 8}; - try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.d4")) { - int cnt = 0; - while (resultSet.next()) { - assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); - assertEquals(expectedValue[cnt], resultSet.getLong("root.sg.d4.s1_max")); - cnt++; + try { + awaitCQResultCount(statement, "select s1_max from root.sg.d4", expectedTime.length); + try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.d4")) { + int cnt = 0; + while (resultSet.next()) { + assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); + assertEquals(expectedValue[cnt], resultSet.getLong("root.sg.d4.s1_max")); + cnt++; + } + assertEquals(expectedTime.length, cnt); } - assertEquals(expectedTime.length, cnt); } finally { statement.execute("DROP CQ cq4"); } @@ -624,6 +626,7 @@ public void testCQExecution4() { } } + // An aligned-by-device CQ must write the expression result for each of the ten input timestamps. @Test public void testCQExecution5() { String insertTemplate = @@ -683,12 +686,6 @@ public void testCQExecution5() { return; } - long targetTime = firstExecutionTime + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = { startTime, startTime + 500, @@ -703,20 +700,23 @@ public void testCQExecution5() { }; double[] expectedValue = {2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0}; - try (ResultSet resultSet = - statement.executeQuery( - "select precalculated_s1 from root.sg.d5 where time between " - + startTime - + " and " - + (startTime + 4_500))) { - int cnt = 0; - while (resultSet.next()) { - assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); - assertEquals( - expectedValue[cnt], resultSet.getDouble("root.sg.d5.precalculated_s1"), 0.00001); - cnt++; + String query = + "select precalculated_s1 from root.sg.d5 where time between " + + startTime + + " and " + + (startTime + 4_500); + try { + awaitCQResultCount(statement, query, expectedTime.length); + try (ResultSet resultSet = statement.executeQuery(query)) { + int cnt = 0; + while (resultSet.next()) { + assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); + assertEquals( + expectedValue[cnt], resultSet.getDouble("root.sg.d5.precalculated_s1"), 0.00001); + cnt++; + } + assertEquals(expectedTime.length, cnt); } - assertEquals(expectedTime.length, cnt); } finally { statement.execute("DROP CQ cq5"); } @@ -726,4 +726,30 @@ public void testCQExecution5() { fail(e.getMessage()); } } + + private static void awaitCQResultCount(Statement statement, String query, int expectedRowCount) + throws Exception { + // CQ writes can fall between DataNode query snapshots. Count each snapshot independently + // while waiting; the caller then compares all timestamps and values through the cluster + // wrapper. Query and transport errors still propagate immediately. + Awaitility.await("continuous query output row count") + .atMost(CQ_RESULT_TIMEOUT_IN_SECONDS, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(CQ_RESULT_POLL_INTERVAL_IN_MS, TimeUnit.MILLISECONDS) + .until( + () -> { + try (ResultSet resultSet = statement.executeQuery(query)) { + for (ResultSet nodeResultSet : ClusterTestResultSet.getNodeResultSets(resultSet)) { + int rowCount = 0; + while (nodeResultSet.next()) { + rowCount++; + } + if (rowCount < expectedRowCount) { + return false; + } + } + return true; + } + }); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecInNsIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecInNsIT.java index 80acdb1408728..129983a83a229 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecInNsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecInNsIT.java @@ -23,7 +23,9 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.itbase.runtime.ClusterTestResultSet; +import org.awaitility.Awaitility; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -43,6 +45,9 @@ @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBCQExecInNsIT { + private static final long CQ_RESULT_POLL_INTERVAL_IN_MS = 100; + private static final long CQ_RESULT_TIMEOUT_IN_SECONDS = 30; + @BeforeClass public static void setUp() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecision("ns"); @@ -54,6 +59,8 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + // With nanosecond timestamps, a 2s resampling CQ must write the expected maximum for every 1s + // window after its future boundary. @Test public void testCQExecution1() { String insertTemplate = @@ -112,12 +119,6 @@ public void testCQExecution1() { return; } - long targetTime = firstExecutionTime / 1_000_000L + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = { startTime + 1_000_000_000L, startTime + 2_000_000_000L, @@ -126,7 +127,8 @@ public void testCQExecution1() { }; long[] expectedValue = {4, 6, 8, 10}; - try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.d1")) { + try (ResultSet resultSet = + awaitCQResult(statement, "select s1_max from root.sg.d1", expectedTime.length)) { int cnt = 0; while (resultSet.next()) { assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); @@ -144,6 +146,8 @@ public void testCQExecution1() { } } + // With nanosecond timestamps, a CQ without EVERY and with a 4s range must produce the five + // expected 1s-window maxima. @Test public void testCQExecution2() { String insertTemplate = @@ -203,12 +207,6 @@ public void testCQExecution2() { return; } - long targetTime = firstExecutionTime / 1_000_000L + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = { startTime, startTime + 1_000_000_000L, @@ -218,7 +216,8 @@ public void testCQExecution2() { }; long[] expectedValue = {2, 4, 6, 8, 10}; - try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.d2")) { + try (ResultSet resultSet = + awaitCQResult(statement, "select s1_max from root.sg.d2", expectedTime.length)) { int cnt = 0; while (resultSet.next()) { assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); @@ -236,6 +235,8 @@ public void testCQExecution2() { } } + // With nanosecond timestamps, a 4s-range CQ must fill the empty 1s window with 100 and write + // the expected maxima for the populated windows. @Test public void testCQExecution3() { String insertTemplate = @@ -296,12 +297,6 @@ public void testCQExecution3() { return; } - long targetTime = firstExecutionTime / 1_000_000L + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = { startTime - 1_000_000_000L, startTime, @@ -313,11 +308,13 @@ public void testCQExecution3() { long[] expectedValue = {100, 2, 4, 6, 8, 10}; try (ResultSet resultSet = - statement.executeQuery( + awaitCQResult( + statement, "select s1_max from root.sg.d3 where time between " + (startTime - 1_000_000_000L) + " and " - + (startTime + 4_000_000_000L))) { + + (startTime + 4_000_000_000L), + expectedTime.length)) { int cnt = 0; while (resultSet.next()) { assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); @@ -335,6 +332,8 @@ public void testCQExecution3() { } } + // With nanosecond timestamps, a CQ with RANGE 2s, 1s must write only the two expected 1s-window + // maxima at its 2s resampling boundaries. @Test public void testCQExecution4() { String insertTemplate = @@ -394,16 +393,11 @@ public void testCQExecution4() { return; } - long targetTime = firstExecutionTime / 1_000_000L + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = {startTime + 1_000_000_000L, startTime + 3_000_000_000L}; long[] expectedValue = {4, 8}; - try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.d4")) { + try (ResultSet resultSet = + awaitCQResult(statement, "select s1_max from root.sg.d4", expectedTime.length)) { int cnt = 0; while (resultSet.next()) { assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); @@ -421,6 +415,8 @@ public void testCQExecution4() { } } + // With nanosecond timestamps, an align-by-device CQ must add one to all ten input values + // while preserving their half-second timestamps across overlapping 4s ranges. @Test public void testCQExecution5() { String insertTemplate = @@ -480,12 +476,6 @@ public void testCQExecution5() { return; } - long targetTime = firstExecutionTime / 1_000_000L + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = { startTime, startTime + 500_000_000L, @@ -501,11 +491,13 @@ public void testCQExecution5() { double[] expectedValue = {2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0}; try (ResultSet resultSet = - statement.executeQuery( + awaitCQResult( + statement, "select precalculated_s1 from root.sg.d5 where time between " + startTime + " and " - + (startTime + 4_500_000_000L))) { + + (startTime + 4_500_000_000L), + expectedTime.length)) { int cnt = 0; while (resultSet.next()) { assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); @@ -523,4 +515,33 @@ public void testCQExecution5() { fail(e.getMessage()); } } + + // The row count signals that asynchronous CQ writes are ready without a fixed post-boundary + // delay. The caller closes the returned result set and validates every timestamp and value. + private static ResultSet awaitCQResult(Statement statement, String query, int expectedRowCount) + throws Exception { + // CQ writes can fall between DataNode query snapshots. Count each snapshot independently + // while waiting; the caller then compares all timestamps and values through the cluster + // wrapper. Query and transport errors still propagate immediately. + Awaitility.await("continuous query output row count") + .atMost(CQ_RESULT_TIMEOUT_IN_SECONDS, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(CQ_RESULT_POLL_INTERVAL_IN_MS, TimeUnit.MILLISECONDS) + .until( + () -> { + try (ResultSet resultSet = statement.executeQuery(query)) { + for (ResultSet nodeResultSet : ClusterTestResultSet.getNodeResultSets(resultSet)) { + int rowCount = 0; + while (nodeResultSet.next()) { + rowCount++; + } + if (rowCount < expectedRowCount) { + return false; + } + } + return true; + } + }); + return statement.executeQuery(query); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecInUsIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecInUsIT.java index c9971af3afb27..7e2ca7d9d6855 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecInUsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecInUsIT.java @@ -23,7 +23,9 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.itbase.runtime.ClusterTestResultSet; +import org.awaitility.Awaitility; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -43,6 +45,9 @@ @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBCQExecInUsIT { + private static final long CQ_RESULT_POLL_INTERVAL_IN_MS = 100; + private static final long CQ_RESULT_TIMEOUT_IN_SECONDS = 30; + @BeforeClass public static void setUp() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecision("us"); @@ -54,6 +59,8 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + // With microsecond timestamps, a 2s resampling CQ must write the expected maximum for every 1s + // window after its future boundary. @Test public void testCQExecution1() { String insertTemplate = @@ -112,12 +119,6 @@ public void testCQExecution1() { return; } - long targetTime = firstExecutionTime / 1_000L + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = { startTime + 1_000_000L, startTime + 2_000_000L, @@ -126,7 +127,8 @@ public void testCQExecution1() { }; long[] expectedValue = {4, 6, 8, 10}; - try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.d1")) { + try (ResultSet resultSet = + awaitCQResult(statement, "select s1_max from root.sg.d1", expectedTime.length)) { int cnt = 0; while (resultSet.next()) { assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); @@ -144,6 +146,8 @@ public void testCQExecution1() { } } + // With microsecond timestamps, a CQ without EVERY and with a 4s range must produce the five + // expected 1s-window maxima. @Test public void testCQExecution2() { String insertTemplate = @@ -203,12 +207,6 @@ public void testCQExecution2() { return; } - long targetTime = firstExecutionTime / 1_000L + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = { startTime, startTime + 1_000_000L, @@ -218,7 +216,8 @@ public void testCQExecution2() { }; long[] expectedValue = {2, 4, 6, 8, 10}; - try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.d2")) { + try (ResultSet resultSet = + awaitCQResult(statement, "select s1_max from root.sg.d2", expectedTime.length)) { int cnt = 0; while (resultSet.next()) { assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); @@ -236,6 +235,8 @@ public void testCQExecution2() { } } + // With microsecond timestamps, a 4s-range CQ must fill the empty 1s window with 100 and write + // the expected maxima for the populated windows. @Test public void testCQExecution3() { String insertTemplate = @@ -296,12 +297,6 @@ public void testCQExecution3() { return; } - long targetTime = firstExecutionTime / 1_000L + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = { startTime - 1_000_000L, startTime, @@ -313,11 +308,13 @@ public void testCQExecution3() { long[] expectedValue = {100, 2, 4, 6, 8, 10}; try (ResultSet resultSet = - statement.executeQuery( + awaitCQResult( + statement, "select s1_max from root.sg.d3 where time between " + (startTime - 1_000_000L) + " and " - + (startTime + 4_000_000L))) { + + (startTime + 4_000_000L), + expectedTime.length)) { int cnt = 0; while (resultSet.next()) { assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); @@ -335,6 +332,8 @@ public void testCQExecution3() { } } + // With microsecond timestamps, a CQ with RANGE 2s, 1s must write only the two expected 1s-window + // maxima at its 2s resampling boundaries. @Test public void testCQExecution4() { String insertTemplate = @@ -394,16 +393,11 @@ public void testCQExecution4() { return; } - long targetTime = firstExecutionTime / 1_000L + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = {startTime + 1_000_000L, startTime + 3_000_000L}; long[] expectedValue = {4, 8}; - try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.d4")) { + try (ResultSet resultSet = + awaitCQResult(statement, "select s1_max from root.sg.d4", expectedTime.length)) { int cnt = 0; while (resultSet.next()) { assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); @@ -421,6 +415,8 @@ public void testCQExecution4() { } } + // With microsecond timestamps, an align-by-device CQ must add one to all ten input values + // while preserving their half-second timestamps across overlapping 4s ranges. @Test public void testCQExecution5() { String insertTemplate = @@ -480,12 +476,6 @@ public void testCQExecution5() { return; } - long targetTime = firstExecutionTime / 1_000L + 10_000; - - while (System.currentTimeMillis() - targetTime < 0) { - TimeUnit.SECONDS.sleep(1); - } - long[] expectedTime = { startTime, startTime + 500_000L, @@ -501,11 +491,13 @@ public void testCQExecution5() { double[] expectedValue = {2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0}; try (ResultSet resultSet = - statement.executeQuery( + awaitCQResult( + statement, "select precalculated_s1 from root.sg.d5 where time between " + startTime + " and " - + (startTime + 4_500_000L))) { + + (startTime + 4_500_000L), + expectedTime.length)) { int cnt = 0; while (resultSet.next()) { assertEquals(expectedTime[cnt], resultSet.getLong(TIMESTAMP_STR)); @@ -523,4 +515,33 @@ public void testCQExecution5() { fail(e.getMessage()); } } + + // The row count signals that asynchronous CQ writes are ready without a fixed post-boundary + // delay. The caller closes the returned result set and validates every timestamp and value. + private static ResultSet awaitCQResult(Statement statement, String query, int expectedRowCount) + throws Exception { + // CQ writes can fall between DataNode query snapshots. Count each snapshot independently + // while waiting; the caller then compares all timestamps and values through the cluster + // wrapper. Query and transport errors still propagate immediately. + Awaitility.await("continuous query output row count") + .atMost(CQ_RESULT_TIMEOUT_IN_SECONDS, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(CQ_RESULT_POLL_INTERVAL_IN_MS, TimeUnit.MILLISECONDS) + .until( + () -> { + try (ResultSet resultSet = statement.executeQuery(query)) { + for (ResultSet nodeResultSet : ClusterTestResultSet.getNodeResultSets(resultSet)) { + int rowCount = 0; + while (nodeResultSet.next()) { + rowCount++; + } + if (rowCount < expectedRowCount) { + return false; + } + } + return true; + } + }); + return statement.executeQuery(query); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/mqtt/IoTDBMQTTServiceJsonIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/mqtt/IoTDBMQTTServiceJsonIT.java index 8d372c1d81009..93527178cd6cd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/mqtt/IoTDBMQTTServiceJsonIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/mqtt/IoTDBMQTTServiceJsonIT.java @@ -43,29 +43,26 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; -import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; import java.util.List; +import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * Integration tests for MQTT service with JSON payload formatter. JSON formatter supports tree * model data insertion. + * + *

Each test owns distinct device paths and only queries those paths. Retain data until class + * cleanup to avoid racing asynchronous MQTT writes with database deletion between methods. */ @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBMQTTServiceJsonIT { private BlockingConnection connection; - private static int mqttPort; - private static final String IP = System.getProperty("RemoteIp", "127.0.0.1"); private static final String USER = System.getProperty("RemoteUser", SessionConfig.DEFAULT_USER); private static final String PASSWORD = System.getProperty("RemotePassword", SessionConfig.DEFAULT_PASSWORD); @@ -76,9 +73,10 @@ public static void beforeClass() throws Exception { BaseEnv baseEnv = EnvFactory.getEnv(); baseEnv.getConfig().getDataNodeConfig().setEnableMQTTService(true); baseEnv.getConfig().getDataNodeConfig().setMqttPayloadFormatter(FORMATTER); + // Serialize CONNECT and PUBLISH callbacks so the session is registered before each + // payload's single publish is processed. + baseEnv.getConfig().getDataNodeConfig().setMqttHandlerPoolSize(1); baseEnv.initClusterEnvironment(); - DataNodeWrapper portConflictDataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); - mqttPort = portConflictDataNodeWrapper.getMqttPort(); } @AfterClass @@ -88,13 +86,17 @@ public static void afterClass() { @Before public void setUp() throws Exception { + DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); MQTT mqtt = new MQTT(); - mqtt.setHost(IP, mqttPort); + mqtt.setHost( + System.getProperty("RemoteIp", dataNodeWrapper.getIp()), dataNodeWrapper.getMqttPort()); mqtt.setUserName(USER); mqtt.setPassword(PASSWORD); mqtt.setConnectAttemptsMax(3); mqtt.setReconnectDelay(1000); - mqtt.setClientId("jsonClientId1"); + // A delayed disconnect must not remove the next test's broker session. Keep IDs within the + // 23-character MQTT 3.1 limit. + mqtt.setClientId("client-" + UUID.randomUUID().toString().substring(0, 16)); connection = mqtt.blockingConnection(); connection.connect(); @@ -102,36 +104,15 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - try { - if (connection != null && connection.isConnected()) { - connection.disconnect(); - } - } catch (IOException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - cleanupSgData(); - } - - private static void executeQuietly(Statement statement, String sql) { - try { - statement.execute(sql); - } catch (SQLException ignored) { - // ignore - } - } - - private static void cleanupSgData() { - try (Connection conn = EnvFactory.getEnv().getConnection(); - Statement statement = conn.createStatement()) { - executeQuietly(statement, "DELETE DATABASE root.sg"); - executeQuietly(statement, "DELETE DATABASE root.sg.**"); - } catch (SQLException ignored) { - // ignore + if (connection != null && connection.isConnected()) { + connection.disconnect(); } } - /** Test single JSON message with multiple measurements */ + /** + * Publish one JSON message for d1 at time 1 with s1=1.5 and s2=2.5; wait for both measurements + * and verify the returned row contains both values. + */ @Test public void testSingleJsonMessage() throws Exception { try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { @@ -143,12 +124,13 @@ public void testSingleJsonMessage() throws Exception { + "\"values\":[1.5,2.5]" + "}"; + connection.publish("root.sg.d1", payload.getBytes(), QoS.AT_LEAST_ONCE, false); Awaitility.await() .atMost(3, TimeUnit.MINUTES) - .pollInterval(1, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .until( () -> { - connection.publish("root.sg.d1", payload.getBytes(), QoS.AT_LEAST_ONCE, false); try (final SessionDataSet dataSet = session.executeQueryStatement("select s1, s2 from root.sg.d1 where time = 1")) { if (!dataSet.hasNext()) { @@ -183,7 +165,10 @@ public void testSingleJsonMessage() throws Exception { } } - /** Test batch JSON message with timestamps array */ + /** + * Publish one batch JSON message for d2 with timestamps 1, 2, and 3 and two measurements; verify + * all three s1 points become visible without republishing the batch during polling. + */ @Test public void testBatchJsonMessage() throws Exception { try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { @@ -195,12 +180,13 @@ public void testBatchJsonMessage() throws Exception { + "\"values\":[[1.0,2.0],[3.0,4.0],[5.0,6.0]]" + "}"; + connection.publish("root.sg.d2", payload.getBytes(), QoS.AT_LEAST_ONCE, false); Awaitility.await() .atMost(3, TimeUnit.MINUTES) - .pollInterval(1, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .until( () -> { - connection.publish("root.sg.d2", payload.getBytes(), QoS.AT_LEAST_ONCE, false); try (final SessionDataSet dataSet = session.executeQueryStatement("select count(s1) from root.sg.d2")) { if (!dataSet.hasNext()) { @@ -230,7 +216,10 @@ public void testBatchJsonMessage() throws Exception { } } - /** Test JSON array with multiple messages */ + /** + * Publish one JSON array containing d3 values 10, 20, and 30 at three timestamps; wait for all + * three rows and verify their sum is 60. + */ @Test public void testJsonArray() throws Exception { try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { @@ -241,12 +230,13 @@ public void testJsonArray() throws Exception { + "{\"device\":\"root.sg.d3\",\"timestamp\":3,\"measurements\":[\"s1\"],\"values\":[30.0]}" + "]"; + connection.publish("root.sg.d3", payload.getBytes(), QoS.AT_LEAST_ONCE, false); Awaitility.await() .atMost(3, TimeUnit.MINUTES) - .pollInterval(1, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .until( () -> { - connection.publish("root.sg.d3", payload.getBytes(), QoS.AT_LEAST_ONCE, false); try (final SessionDataSet dataSet = session.executeQueryStatement("select s1 from root.sg.d3")) { int count = 0; @@ -286,7 +276,10 @@ public void testJsonArray() throws Exception { } } - /** Test JSON with explicit data types */ + /** + * Publish one explicitly typed JSON row for d4 at time 1; verify INT32 100, FLOAT 3.14, BOOLEAN + * true, and TEXT hello are readable with the corresponding typed field accessors. + */ @Test public void testJsonWithDataTypes() throws Exception { try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { @@ -299,12 +292,13 @@ public void testJsonWithDataTypes() throws Exception { + "\"datatypes\":[\"INT32\",\"FLOAT\",\"BOOLEAN\",\"TEXT\"]" + "}"; + connection.publish("root.sg.d4", payload.getBytes(), QoS.AT_LEAST_ONCE, false); Awaitility.await() .atMost(3, TimeUnit.MINUTES) - .pollInterval(1, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .until( () -> { - connection.publish("root.sg.d4", payload.getBytes(), QoS.AT_LEAST_ONCE, false); try (final SessionDataSet dataSet = session.executeQueryStatement( "select intVal, floatVal, boolVal, textVal from root.sg.d4 where time = 1")) { @@ -344,7 +338,10 @@ public void testJsonWithDataTypes() throws Exception { } } - /** Test multiple devices in single JSON array */ + /** + * Publish one JSON array for device1, device2, and device3 at time 1; wait for each device and + * verify their respective temperatures are 25.5, 26.5, and 27.5. + */ @Test public void testMultipleDevicesJsonArray() throws Exception { try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { @@ -355,12 +352,13 @@ public void testMultipleDevicesJsonArray() throws Exception { + "{\"device\":\"root.sg.device3\",\"timestamp\":1,\"measurements\":[\"temp\"],\"values\":[27.5]}" + "]"; + connection.publish("root.sg", payload.getBytes(), QoS.AT_LEAST_ONCE, false); Awaitility.await() .atMost(3, TimeUnit.MINUTES) - .pollInterval(1, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .until( () -> { - connection.publish("root.sg", payload.getBytes(), QoS.AT_LEAST_ONCE, false); try { // Check device1 try (final SessionDataSet dataSet1 = @@ -422,7 +420,10 @@ public void testMultipleDevicesJsonArray() throws Exception { } } - /** Test batch JSON with different values per timestamp */ + /** + * Publish one d5 batch with five timestamps from 100 to 500 and varying temperature/humidity + * pairs; verify five rows, two fields per row, and the original temperature and humidity ranges. + */ @Test public void testBatchJsonWithVariousValues() throws Exception { try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { @@ -434,12 +435,13 @@ public void testBatchJsonWithVariousValues() throws Exception { + "\"values\":[[20.1,60.0],[21.2,61.5],[22.3,62.0],[23.4,63.5],[24.5,64.0]]" + "}"; + connection.publish("root.sg.d5", payload.getBytes(), QoS.AT_LEAST_ONCE, false); Awaitility.await() .atMost(3, TimeUnit.MINUTES) - .pollInterval(1, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .until( () -> { - connection.publish("root.sg.d5", payload.getBytes(), QoS.AT_LEAST_ONCE, false); try (final SessionDataSet dataSet = session.executeQueryStatement("select temperature, humidity from root.sg.d5")) { int count = 0; @@ -483,6 +485,10 @@ public void testBatchJsonWithVariousValues() throws Exception { } } + /** + * Publish one array containing illegal measurement a.b at time 1 followed by valid at time 2; + * after valid becomes visible, verify only one timeseries exists and the illegal path is absent. + */ @Test public void testIllegalMeasurementIsRejected() throws Exception { try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { @@ -492,12 +498,13 @@ public void testIllegalMeasurementIsRejected() throws Exception { + "{\"device\":\"root.sg.invalid\",\"timestamp\":2,\"measurements\":[\"valid\"],\"values\":[2]}" + "]"; + connection.publish("root.sg.invalid", payload.getBytes(), QoS.AT_LEAST_ONCE, false); Awaitility.await() .atMost(3, TimeUnit.MINUTES) - .pollInterval(1, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .until( () -> { - connection.publish("root.sg.invalid", payload.getBytes(), QoS.AT_LEAST_ONCE, false); try (final SessionDataSet dataSet = session.executeQueryStatement( "select valid from root.sg.invalid where time = 2")) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java index cf953aa9e7c4c..8563d64c55381 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.it.quotas; -import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -63,6 +62,7 @@ public void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + // Reject a fourth timeseries after quota usage reaches the configured limit of three. @Test public void timeseriesNumExceedTest() { try (Connection adminCon = EnvFactory.getEnv().getConnection(); @@ -74,15 +74,16 @@ public void timeseriesNumExceedTest() { "create timeseries root.sg0.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN;"); adminStmt.execute( "create timeseries root.sg0.wf01.wt01.status2 with datatype=BOOLEAN,encoding=PLAIN;"); - Thread.sleep(ConfigNodeDescriptor.getInstance().getConf().getHeartbeatIntervalInMs() * 2); + validateQuotaUsageEventually(adminStmt, "root.sg0", "timeSeriesNum", "3"); adminStmt.execute( "create timeseries root.sg0.wf01.wt01.status3 with datatype=BOOLEAN,encoding=PLAIN;"); - } catch (SQLException | InterruptedException throwables) { + } catch (SQLException throwables) { Assert.assertEquals( "1700: The number of timeseries has reached the upper limit", throwables.getMessage()); } } + // Reject a fourth device after quota usage reaches the configured limit of three. @Test public void devicesNumExceedTest() { try (Connection adminCon = EnvFactory.getEnv().getConnection(); @@ -94,15 +95,16 @@ public void devicesNumExceedTest() { "create timeseries root.sg0.wf02.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN;"); adminStmt.execute( "create timeseries root.sg0.wf03.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN;"); - Thread.sleep(ConfigNodeDescriptor.getInstance().getConf().getHeartbeatIntervalInMs() * 2); + validateQuotaUsageEventually(adminStmt, "root.sg0", "deviceNum", "3"); adminStmt.execute( "create timeseries root.sg0.wf04.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN;"); - } catch (SQLException | InterruptedException throwables) { + } catch (SQLException throwables) { Assert.assertEquals( "1700: The number of devices has reached the upper limit", throwables.getMessage()); } } + // Raise timeseries and device limits only after each original limit is fully used. @Test public void setSpaceQuotaTest0() { try (Connection adminCon = EnvFactory.getEnv().getConnection(); @@ -114,11 +116,11 @@ public void setSpaceQuotaTest0() { "create timeseries root.sg0.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN;"); adminStmt.execute( "create timeseries root.sg0.wf01.wt01.status2 with datatype=BOOLEAN,encoding=PLAIN;"); - Thread.sleep(ConfigNodeDescriptor.getInstance().getConf().getHeartbeatIntervalInMs() * 2); + validateQuotaUsageEventually(adminStmt, "root.sg0", "timeSeriesNum", "3"); adminStmt.execute("set space quota timeseries=4 on root.sg0"); adminStmt.execute( "create timeseries root.sg0.wf01.wt01.status3 with datatype=BOOLEAN,encoding=PLAIN;"); - } catch (SQLException | InterruptedException throwables) { + } catch (SQLException throwables) { Assert.fail(throwables.getMessage()); } @@ -131,11 +133,11 @@ public void setSpaceQuotaTest0() { "create timeseries root.sg1.wf02.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN;"); adminStmt.execute( "create timeseries root.sg1.wf03.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN;"); - Thread.sleep(ConfigNodeDescriptor.getInstance().getConf().getHeartbeatIntervalInMs() * 2); + validateQuotaUsageEventually(adminStmt, "root.sg1", "deviceNum", "3"); adminStmt.execute("set space quota devices=4 on root.sg1"); adminStmt.execute( "create timeseries root.sg1.wf04.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN;"); - } catch (SQLException | InterruptedException throwables) { + } catch (SQLException throwables) { Assert.fail(throwables.getMessage()); } @@ -148,6 +150,7 @@ public void setSpaceQuotaTest0() { } } + // Reject reducing timeseries and device limits below their synchronized usage of three. @Test public void setSpaceQuotaTest1() { try (Connection adminCon = EnvFactory.getEnv().getConnection(); @@ -159,9 +162,9 @@ public void setSpaceQuotaTest1() { "create timeseries root.sg0.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN;"); adminStmt.execute( "create timeseries root.sg0.wf01.wt01.status2 with datatype=BOOLEAN,encoding=PLAIN;"); - Thread.sleep(ConfigNodeDescriptor.getInstance().getConf().getHeartbeatIntervalInMs() * 2); + validateQuotaUsageEventually(adminStmt, "root.sg0", "timeSeriesNum", "3"); adminStmt.execute("set space quota timeseries=2 on root.sg0"); - } catch (SQLException | InterruptedException throwables) { + } catch (SQLException throwables) { Assert.assertEquals( "301: The used quota exceeds the preset quota. Please set a larger value.", throwables.getMessage()); @@ -176,9 +179,9 @@ public void setSpaceQuotaTest1() { "create timeseries root.sg1.wf02.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN;"); adminStmt.execute( "create timeseries root.sg1.wf03.wt01.status0 with datatype=BOOLEAN,encoding=PLAIN;"); - Thread.sleep(ConfigNodeDescriptor.getInstance().getConf().getHeartbeatIntervalInMs() * 2); + validateQuotaUsageEventually(adminStmt, "root.sg1", "deviceNum", "3"); adminStmt.execute("set space quota devices=2 on root.sg1"); - } catch (SQLException | InterruptedException throwables) { + } catch (SQLException throwables) { Assert.assertEquals( "301: The used quota exceeds the preset quota. Please set a larger value.", throwables.getMessage()); @@ -514,7 +517,7 @@ private void validateResultSetEventually(Statement statement, String sql, String Awaitility.await() .pollInSameThread() .pollDelay(0, TimeUnit.MILLISECONDS) - .pollInterval(1, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .atMost(30, TimeUnit.SECONDS) // Cluster queries compare results from all DataNodes. Space quota usage is refreshed // asynchronously by heartbeats, so the comparison can observe different snapshots while @@ -530,6 +533,33 @@ private void validateResultSetEventually(Statement statement, String sql, String }); } + private void validateQuotaUsageEventually( + Statement statement, String database, String quotaType, String usage) { + Awaitility.await() + .pollInSameThread() + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .atMost(30, TimeUnit.SECONDS) + .ignoreExceptionsMatching(e -> e instanceof InconsistentDataException) + .untilAsserted( + () -> { + try (ResultSet resultSet = + statement.executeQuery("show space quota " + database + ";")) { + while (resultSet.next()) { + if (database.equals(resultSet.getString(1)) + && quotaType.equals(resultSet.getString(2)) + && "3".equals(resultSet.getString(3)) + && usage.equals(resultSet.getString(4))) { + return; + } + } + Assert.fail("Expected quota usage " + database + "," + quotaType + ",3," + usage); + } catch (SQLException e) { + throw new AssertionError(e); + } + }); + } + private void validateResultSet(ResultSet set, String ans) throws SQLException { try { StringBuilder builder = new StringBuilder(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFNormalQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFNormalQueryIT.java index 82f61879dd774..85adc7fbc7dd3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFNormalQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFNormalQueryIT.java @@ -19,11 +19,17 @@ package org.apache.iotdb.db.it.udaf; +import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -33,9 +39,8 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.sql.SQLException; import java.sql.Statement; -import java.util.Locale; +import java.util.List; import static org.apache.iotdb.itbase.constant.TestConstant.*; import static org.junit.Assert.assertArrayEquals; @@ -92,87 +97,81 @@ public class IoTDBUDAFNormalQueryIT { private final String d0s1 = "root.vehicle.d0.s1"; private final String d0s2 = "root.vehicle.d0.s2"; private final String d0s3 = "root.vehicle.d0.s3"; - private static final String insertTemplate = - "INSERT INTO root.vehicle.d0(timestamp,s0,s1,s2,s3,s4)" + " VALUES(%d,%d,%d,%f,%s,%s)"; @BeforeClass public static void setUp() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setUdfMemoryBudgetInMB(5); EnvFactory.getEnv().initClusterEnvironment(); - prepareData(); - registerUDAF(); + // Reuse one session for schema creation, fixture writes, flushes and UDAF registration. + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + prepareData(session); + registerUDAF(session); + } } - private static void prepareData() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + private static void prepareData(ISession session) + throws IoTDBConnectionException, StatementExecutionException { + for (String sql : creationSqls) { + session.executeNonQueryStatement(sql); + } - for (String sql : creationSqls) { - statement.execute(sql); - } + // Batch the fixture without SQL parsing per row; UDAF queries retain all original input points. + Tablet tablet = + new Tablet( + "root.vehicle.d0", + List.of( + new MeasurementSchema("s0", TSDataType.INT32), + new MeasurementSchema("s1", TSDataType.INT64), + new MeasurementSchema("s2", TSDataType.FLOAT), + new MeasurementSchema("s3", TSDataType.TEXT), + new MeasurementSchema("s4", TSDataType.BOOLEAN)), + 500); - // prepare BufferWrite file - for (int i = 5000; i < 7000; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); - } - statement.executeBatch(); - statement.clearBatch(); - statement.execute("flush"); + // prepare BufferWrite file + insertRange(session, tablet, 5000, 7000, true); + session.executeNonQueryStatement("flush"); - for (int i = 7500; i < 8500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); - } - statement.executeBatch(); - statement.clearBatch(); - statement.execute("flush"); - // prepare Unseq-File - for (int i = 500; i < 1500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); - } - statement.executeBatch(); - statement.clearBatch(); - statement.execute("flush"); - for (int i = 3000; i < 6500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); - } - statement.executeBatch(); - statement.clearBatch(); + insertRange(session, tablet, 7500, 8500, false); + session.executeNonQueryStatement("flush"); + // prepare Unseq-File + insertRange(session, tablet, 500, 1500, true); + session.executeNonQueryStatement("flush"); + insertRange(session, tablet, 3000, 6500, false); - // prepare BufferWrite cache - for (int i = 9000; i < 10000; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "true")); - } - statement.executeBatch(); - statement.clearBatch(); - // prepare Overflow cache - for (int i = 2000; i < 2500; i++) { - statement.addBatch( - String.format( - Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "'" + i + "'", "false")); - } - statement.executeBatch(); - statement.clearBatch(); + // prepare BufferWrite cache + insertRange(session, tablet, 9000, 10000, true); + // prepare Overflow cache + insertRange(session, tablet, 2000, 2500, false); - for (String sql : dataSet3) { - statement.execute(sql); - } + for (String sql : dataSet3) { + session.executeNonQueryStatement(sql); + } + + for (String sql : dataSet2) { + session.executeNonQueryStatement(sql); + } + } - for (String sql : dataSet2) { - statement.execute(sql); + private static void insertRange( + ISession session, Tablet tablet, int startTime, int endTime, boolean booleanValue) + throws IoTDBConnectionException, StatementExecutionException { + // Reuse the bounded tablet while preserving write order and the callers' flush boundaries. + for (int time = startTime; time < endTime; time++) { + int row = tablet.getRowSize(); + tablet.addTimestamp(row, time); + tablet.addValue("s0", row, time); + tablet.addValue("s1", row, (long) time); + tablet.addValue("s2", row, (float) time); + tablet.addValue("s3", row, Integer.toString(time)); + tablet.addValue("s4", row, booleanValue); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insertTablet(tablet); + tablet.reset(); } - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); + } + if (tablet.getRowSize() > 0) { + session.insertTablet(tablet); + tablet.reset(); } } @@ -181,18 +180,14 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } - private static void registerUDAF() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - statement.execute( - "CREATE FUNCTION avg_udaf AS 'org.apache.iotdb.db.query.udf.example.UDAFAvg'"); - statement.execute( - "CREATE FUNCTION count_udaf AS 'org.apache.iotdb.db.query.udf.example.UDAFCount'"); - statement.execute( - "CREATE FUNCTION sum_udaf AS 'org.apache.iotdb.db.query.udf.example.UDAFSum'"); - } catch (SQLException throwable) { - fail(throwable.getMessage()); - } + private static void registerUDAF(ISession session) + throws IoTDBConnectionException, StatementExecutionException { + session.executeNonQueryStatement( + "CREATE FUNCTION avg_udaf AS 'org.apache.iotdb.db.query.udf.example.UDAFAvg'"); + session.executeNonQueryStatement( + "CREATE FUNCTION count_udaf AS 'org.apache.iotdb.db.query.udf.example.UDAFCount'"); + session.executeNonQueryStatement( + "CREATE FUNCTION sum_udaf AS 'org.apache.iotdb.db.query.udf.example.UDAFSum'"); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java index 71555bce0e3cd..0da7fe8fd6794 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java @@ -24,8 +24,8 @@ import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; -import org.junit.After; -import org.junit.Before; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -45,8 +45,10 @@ public class IoTDBUDFBlockQueryIT { private static final double E = 0.0001; - @Before - public void setUp() throws Exception { + // Queries share read-only input, and the untrusted URI registration must leave the UDFs + // unchanged. + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setUdfMemoryBudgetInMB(5); EnvFactory.getEnv().initClusterEnvironment(); createTimeSeries(); @@ -54,8 +56,8 @@ public void setUp() throws Exception { registerUDF(); } - @After - public void tearDown() { + @AfterClass + public static void tearDownClass() { EnvFactory.getEnv().cleanClusterEnvironment(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFOtherWindowQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFOtherWindowQueryIT.java index c98ea802d28fb..cb25d1dd4d08e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFOtherWindowQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFOtherWindowQueryIT.java @@ -36,6 +36,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.function.IntFunction; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -46,6 +47,7 @@ public class IoTDBUDFOtherWindowQueryIT { protected static final int ITERATION_TIMES = 1000; + private static final int INSERT_BATCH_SIZE = 100; @BeforeClass public static void setUp() throws Exception { @@ -82,67 +84,89 @@ private static void createTimeSeries() { } private static void generateData() { - // SessionWindow try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i == 5 || i == 6 || i == 7 || i == 51 || i == 52 || i == 53 || i == 54 || i == 996 - || i == 997 || i == 998) { - continue; - } - statement.execute( - (String.format("insert into root.vehicle.d1(timestamp,s3) values(%d,%d)", i, i))); - } + // SessionWindow: retain the missing points that separate sessions. + insertMeasurementInBatches( + statement, + "s3", + i -> { + if (i == 5 || i == 6 || i == 7 || i == 51 || i == 52 || i == 53 || i == 54 || i == 996 + || i == 997 || i == 998) { + return null; + } + return Integer.toString(i); + }); // StateWindow INT32 - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i == 1 || i == 2 || i == 3 || i == 15 || i == 17 || i == 53 || i == 54 || i == 996 - || i == 997 || i == 998) { - statement.execute( - (String.format( - "insert into root.vehicle.d1(timestamp,s4) values(%d,%d)", i, i + 100))); - } else if (i == 500 || i == 501) { - statement.execute( - (String.format("insert into root.vehicle.d1(timestamp,s4) values(%d,%d)", i, 12))); - } else if (i % 2 == 0) { - statement.execute( - (String.format("insert into root.vehicle.d1(timestamp,s4) values(%d,%d)", i, 0))); - } else { - statement.execute( - (String.format("insert into root.vehicle.d1(timestamp,s4) values(%d,%d)", i, 3))); - } - } + insertMeasurementInBatches( + statement, + "s4", + i -> { + if (i == 1 || i == 2 || i == 3 || i == 15 || i == 17 || i == 53 || i == 54 || i == 996 + || i == 997 || i == 998) { + return Integer.toString(i + 100); + } else if (i == 500 || i == 501) { + return "12"; + } + return i % 2 == 0 ? "0" : "3"; + }); // StateWindow BOOLEAN - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i == 1 || i == 2 || i == 3 || i == 15 || i == 17 || i == 53 || i == 54 || i == 996 - || i == 997 || i == 998) { - statement.execute( - (String.format("insert into root.vehicle.d1(timestamp,s5) values(%d, true)", i))); - } else { - statement.execute( - (String.format("insert into root.vehicle.d1(timestamp,s5) values(%d, false)", i))); - } - } + insertMeasurementInBatches( + statement, + "s5", + i -> + i == 1 || i == 2 || i == 3 || i == 15 || i == 17 || i == 53 || i == 54 || i == 996 + || i == 997 || i == 998 + ? "true" + : "false"); // StateWindow TEXT - for (int i = 0; i < ITERATION_TIMES; ++i) { - if (i < 500) { - statement.execute( - (String.format("insert into root.vehicle.d1(timestamp,s6) values(%d, '<500')", i))); - } else if (i < 993) { - statement.execute( - (String.format("insert into root.vehicle.d1(timestamp,s6) values(%d, '<993')", i))); - } else { - statement.execute( - (String.format("insert into root.vehicle.d1(timestamp,s6) values(%d, '>=994')", i))); - } - } + insertMeasurementInBatches( + statement, + "s6", + i -> { + if (i < 500) { + return "'<500'"; + } else if (i < 993) { + return "'<993'"; + } + return "'>=994'"; + }); } catch (SQLException throwable) { fail(throwable.getMessage()); } } + private static void insertMeasurementInBatches( + Statement statement, String measurement, IntFunction valueSupplier) + throws SQLException { + // Batch each measurement separately to preserve the original point order and sparse inputs. + String prefix = "insert into root.vehicle.d1(timestamp," + measurement + ") values"; + StringBuilder insertSql = new StringBuilder(prefix); + int rows = 0; + for (int i = 0; i < ITERATION_TIMES; i++) { + String value = valueSupplier.apply(i); + // Returning null omits the point entirely, retaining the session-window gaps. + if (value == null) { + continue; + } + if (rows > 0) { + insertSql.append(','); + } + insertSql.append('(').append(i).append(',').append(value).append(')'); + if (++rows == INSERT_BATCH_SIZE) { + statement.execute(insertSql.toString()); + insertSql.setLength(prefix.length()); + rows = 0; + } + } + if (rows > 0) { + statement.execute(insertSql.toString()); + } + } + private static void registerUDF() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java index c543d4e6bd10a..42dc123ce0708 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java @@ -46,6 +46,7 @@ public class IoTDBUDFWindowQueryIT { protected static final int ITERATION_TIMES = 10000; + private static final int INSERT_BATCH_SIZE = 100; @BeforeClass public static void setUp() throws Exception { @@ -82,10 +83,18 @@ private static void createTimeSeries() { private static void generateData() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - for (int i = 0; i < ITERATION_TIMES; ++i) { - statement.execute( - (String.format( - "insert into root.vehicle.d1(timestamp,s1,s2) values(%d,%d,%d)", i, i, i))); + // Preserve every point and its order for the window queries while reducing fixture RPCs. + for (int start = 0; start < ITERATION_TIMES; start += INSERT_BATCH_SIZE) { + StringBuilder insertSql = + new StringBuilder("insert into root.vehicle.d1(timestamp,s1,s2) values"); + int end = Math.min(start + INSERT_BATCH_SIZE, ITERATION_TIMES); + for (int i = start; i < end; i++) { + if (i > start) { + insertSql.append(','); + } + insertSql.append('(').append(i).append(',').append(i).append(',').append(i).append(')'); + } + statement.execute(insertSql.toString()); } // test empty window, details could be found at https://github.com/apache/iotdb/issues/7738 statement.execute("insert into root.vehicle1.d1(timestamp, s1) values (1,2),(2,3),(7,8)"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java index 613e858394e31..e0e3a4eb3907a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java @@ -47,6 +47,7 @@ import java.sql.Statement; import java.sql.Types; import java.text.DateFormat; +import java.time.Duration; import java.time.ZoneId; import java.time.ZoneOffset; import java.util.ArrayList; @@ -61,8 +62,11 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; import java.util.function.Consumer; +import java.util.stream.Stream; import static org.apache.iotdb.itbase.constant.TestConstant.DELTA; import static org.apache.iotdb.itbase.constant.TestConstant.NULL; @@ -81,6 +85,8 @@ public class TestUtils { private static final Logger LOGGER = LoggerFactory.getLogger(TestUtils.class); private static final int MAX_RESULT_SET_DIFF_ROWS = 20; + // Retry promptly while asynchronous data becomes visible, without continuously querying. + private static final long DATA_ASSERTION_POLL_INTERVAL_MILLIS = 100; public static final ZoneId DEFAULT_ZONE_ID = ZoneId.ofOffset("UTC", ZoneOffset.of("Z")); @@ -1740,12 +1746,26 @@ public static void revokeUserSeriesPrivilege( } public static void restartCluster(BaseEnv env) { - env.shutdownAllDataNodes(); - env.shutdownAllConfigNodes(); - env.startAllConfigNodes(); - env.startAllDataNodes(); + restartCluster(env, (phase, duration) -> {}); + } + + public static void restartCluster(BaseEnv env, final BiConsumer phaseProbe) { + final long restartStartedNanos = System.nanoTime(); + long phaseStartedNanos = restartStartedNanos; + // Node wrappers wait up to 30 seconds per process. Stop and start nodes in each phase together + // so a cluster restart is bounded by the slowest node instead of the sum of all node waits. + runConcurrently(env.getDataNodeWrapperList().stream().map(node -> (Runnable) node::stop)); + phaseStartedNanos = logRestartPhase("stop DataNodes", phaseStartedNanos, phaseProbe); + runConcurrently(env.getConfigNodeWrapperList().stream().map(node -> (Runnable) node::stop)); + phaseStartedNanos = logRestartPhase("stop ConfigNodes", phaseStartedNanos, phaseProbe); + runConcurrently(env.getConfigNodeWrapperList().stream().map(node -> (Runnable) node::start)); + phaseStartedNanos = logRestartPhase("start ConfigNodes", phaseStartedNanos, phaseProbe); + runConcurrently(env.getDataNodeWrapperList().stream().map(node -> (Runnable) node::start)); + phaseStartedNanos = logRestartPhase("start DataNodes", phaseStartedNanos, phaseProbe); ((AbstractEnv) env).checkClusterStatusWithoutUnknown(); awaitAllRegionsRunning(env); + logRestartPhase("await cluster readiness", phaseStartedNanos, phaseProbe); + logRestartPhase("total", restartStartedNanos, phaseProbe); } /** @@ -1786,6 +1806,21 @@ private static void assertAllRegionsRunning(final BaseEnv env, final boolean tab } } + private static long logRestartPhase( + final String phase, + final long phaseStartedNanos, + final BiConsumer phaseProbe) { + final long phaseEndedNanos = System.nanoTime(); + phaseProbe.accept(phase, Duration.ofNanos(phaseEndedNanos - phaseStartedNanos)); + return phaseEndedNanos; + } + + private static void runConcurrently(final Stream actions) { + CompletableFuture.allOf( + actions.map(CompletableFuture::runAsync).toArray(CompletableFuture[]::new)) + .join(); + } + public static void assertDataEventuallyOnEnv( BaseEnv env, String sql, String expectedHeader, Set expectedResSet) { assertDataEventuallyOnEnv(env, sql, expectedHeader, expectedResSet, 600); @@ -1814,7 +1849,7 @@ public static void assertDataEventuallyOnEnv( await() .pollInSameThread() .pollDelay(0L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollInterval(DATA_ASSERTION_POLL_INTERVAL_MILLIS, TimeUnit.MILLISECONDS) .atMost(timeoutSeconds, TimeUnit.SECONDS) .untilAsserted( () -> { @@ -1858,7 +1893,7 @@ public static void assertDataEventuallyOnEnv( await() .pollInSameThread() .pollDelay(0L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollInterval(DATA_ASSERTION_POLL_INTERVAL_MILLIS, TimeUnit.MILLISECONDS) .atMost(timeoutSeconds, TimeUnit.SECONDS) .untilAsserted( () -> { @@ -1894,7 +1929,7 @@ public static void assertDataSizeEventuallyOnEnv( await() .pollInSameThread() .pollDelay(0L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollInterval(DATA_ASSERTION_POLL_INTERVAL_MILLIS, TimeUnit.MILLISECONDS) .atMost(timeoutSeconds, TimeUnit.SECONDS) .untilAsserted( () -> { @@ -1938,7 +1973,7 @@ public static void assertDataEventuallyOnEnv( await() .pollInSameThread() .pollDelay(0L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollInterval(DATA_ASSERTION_POLL_INTERVAL_MILLIS, TimeUnit.MILLISECONDS) .atMost(timeoutSeconds, TimeUnit.SECONDS) .untilAsserted( () -> { @@ -2019,7 +2054,7 @@ public static void assertDataEventuallyOnEnv( await() .pollInSameThread() .pollDelay(0L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollInterval(DATA_ASSERTION_POLL_INTERVAL_MILLIS, TimeUnit.MILLISECONDS) .atMost(timeoutSeconds, TimeUnit.SECONDS) .untilAsserted( () -> { @@ -2094,7 +2129,7 @@ public static void assertDataEventuallyOnEnv( await() .pollInSameThread() .pollDelay(0L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollInterval(DATA_ASSERTION_POLL_INTERVAL_MILLIS, TimeUnit.MILLISECONDS) .atMost(timeoutSeconds, TimeUnit.SECONDS) .untilAsserted( () -> { @@ -2138,7 +2173,7 @@ public static void assertDataEventuallyOnEnv( await() .pollInSameThread() .pollDelay(0L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollInterval(DATA_ASSERTION_POLL_INTERVAL_MILLIS, TimeUnit.MILLISECONDS) .atMost(timeoutSeconds, TimeUnit.SECONDS) .untilAsserted( () -> { diff --git a/integration-test/src/test/java/org/apache/iotdb/libudf/it/dprofile/DProfileIT.java b/integration-test/src/test/java/org/apache/iotdb/libudf/it/dprofile/DProfileIT.java index 72d23ec5e4279..a30c4f0d12557 100644 --- a/integration-test/src/test/java/org/apache/iotdb/libudf/it/dprofile/DProfileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/libudf/it/dprofile/DProfileIT.java @@ -44,6 +44,7 @@ @Category({LocalStandaloneIT.class}) public class DProfileIT { protected static final int ITERATION_TIMES = 10_000; + private static final int INSERT_BATCH_SIZE = 100; @BeforeClass public static void setUp() throws Exception { @@ -88,20 +89,35 @@ private static void generateData() { double x = -100d, y = 100d; // borders of random value try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - for (int i = 1; i <= ITERATION_TIMES; ++i) { - statement.execute( - String.format( - "insert into root.vehicle.d1(timestamp,s1,s2) values(%d,%d,%d)", - i * 1000, - (int) Math.floor(x + Math.random() * y % (y - x + 1)), - (int) Math.floor(x + Math.random() * y % (y - x + 1)))); - statement.execute( - (String.format( - Locale.ENGLISH, - "insert into root.vehicle.d2(timestamp,s1,s2) values(%d,%f,%f)", - i * 1000, - x + Math.random() * y % (y - x + 1), - x + Math.random() * y % (y - x + 1)))); + // Queries run after setup, so batch each device's rows to reduce SQL requests. + // Generate both rows together to retain random-call order and floating-point rounding. + for (int start = 1; start <= ITERATION_TIMES; start += INSERT_BATCH_SIZE) { + StringBuilder d1InsertSql = + new StringBuilder("insert into root.vehicle.d1(timestamp,s1,s2) values"); + StringBuilder d2InsertSql = + new StringBuilder("insert into root.vehicle.d2(timestamp,s1,s2) values"); + int end = Math.min(start + INSERT_BATCH_SIZE - 1, ITERATION_TIMES); + for (int i = start; i <= end; i++) { + if (i > start) { + d1InsertSql.append(','); + d2InsertSql.append(','); + } + d1InsertSql.append( + String.format( + "(%d,%d,%d)", + i * 1000, + (int) Math.floor(x + Math.random() * y % (y - x + 1)), + (int) Math.floor(x + Math.random() * y % (y - x + 1)))); + d2InsertSql.append( + String.format( + Locale.ENGLISH, + "(%d,%f,%f)", + i * 1000, + x + Math.random() * y % (y - x + 1), + x + Math.random() * y % (y - x + 1))); + } + statement.execute(d1InsertSql.toString()); + statement.execute(d2InsertSql.toString()); } statement.execute("insert into root.test.db(time,s1) values (1,1)"); diff --git a/integration-test/src/test/java/org/apache/iotdb/libudf/it/dquality/DQualityIT.java b/integration-test/src/test/java/org/apache/iotdb/libudf/it/dquality/DQualityIT.java index 116f46e150ced..fe82006e81daa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/libudf/it/dquality/DQualityIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/libudf/it/dquality/DQualityIT.java @@ -42,6 +42,7 @@ @Category({LocalStandaloneIT.class}) public class DQualityIT { protected static final int ITERATION_TIMES = 10_000; + private static final int INSERT_BATCH_SIZE = 100; @BeforeClass public static void setUp() throws Exception { @@ -87,20 +88,33 @@ private static void generateData() { long a = 0, b = 1000000000; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - for (int i = 1; i <= ITERATION_TIMES; ++i) { - statement.execute( - String.format( - "insert into root.vehicle.d1(timestamp,s1,s2) values(%d,%d,%d)", - (int) Math.floor(a + Math.random() * b % (b - a + 1)), - (int) Math.floor(x + Math.random() * y % (y - x + 1)), - (int) Math.floor(x + Math.random() * y % (y - x + 1)))); - statement.execute( - (String.format( - Locale.ENGLISH, - "insert into root.vehicle.d2(timestamp,s1,s2) values(%d,%f,%f)", - (int) Math.floor(a + Math.random() * b % (b - a + 1)), - x + Math.random() * y % (y - x + 1), - x + Math.random() * y % (y - x + 1)))); + for (int start = 1; start <= ITERATION_TIMES; start += INSERT_BATCH_SIZE) { + StringBuilder d1InsertSql = + new StringBuilder("insert into root.vehicle.d1(timestamp,s1,s2) values"); + StringBuilder d2InsertSql = + new StringBuilder("insert into root.vehicle.d2(timestamp,s1,s2) values"); + int end = Math.min(start + INSERT_BATCH_SIZE - 1, ITERATION_TIMES); + for (int i = start; i <= end; i++) { + if (i > start) { + d1InsertSql.append(','); + d2InsertSql.append(','); + } + d1InsertSql.append( + String.format( + "(%d,%d,%d)", + (int) Math.floor(a + Math.random() * b % (b - a + 1)), + (int) Math.floor(x + Math.random() * y % (y - x + 1)), + (int) Math.floor(x + Math.random() * y % (y - x + 1)))); + d2InsertSql.append( + String.format( + Locale.ENGLISH, + "(%d,%f,%f)", + (int) Math.floor(a + Math.random() * b % (b - a + 1)), + x + Math.random() * y % (y - x + 1), + x + Math.random() * y % (y - x + 1))); + } + statement.execute(d1InsertSql.toString()); + statement.execute(d2InsertSql.toString()); } } catch (SQLException throwable) { fail(throwable.getMessage()); diff --git a/integration-test/src/test/java/org/apache/iotdb/libudf/it/frequency/FrequencyIT.java b/integration-test/src/test/java/org/apache/iotdb/libudf/it/frequency/FrequencyIT.java index 1ce563e561827..dc56d89255d7f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/libudf/it/frequency/FrequencyIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/libudf/it/frequency/FrequencyIT.java @@ -43,6 +43,7 @@ @Category({LocalStandaloneIT.class}) public class FrequencyIT { protected static final int ITERATION_TIMES = 16384; + private static final int INSERT_BATCH_SIZE = 100; protected static final int DELTA_T = 100; protected static final int PERIOD_1 = 10; protected static final int PEROID_2 = 200; @@ -97,15 +98,25 @@ private static void generateData() { long a = 0, b = 1000000000; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - for (int i = 0; i < ITERATION_TIMES; ++i) { - statement.execute( - String.format( - Locale.ENGLISH, - "insert into root.vehicle.d1(timestamp,s1,s2,s3) values(%d,%f,%f,%f)", - (long) (i + 1) * DELTA_T, - Math.sin(i % PERIOD_1 * 2 * pi / (double) PERIOD_1), - Math.sin(i % PEROID_2 * 2 * pi / (double) PEROID_2), - 0.5 * Math.sin(i % PERIOD_1 * pi) + Math.sin(i % PEROID_2 * pi))); + // Batch the same samples to reduce SQL requests, retaining the six-decimal input rounding. + for (int start = 0; start < ITERATION_TIMES; start += INSERT_BATCH_SIZE) { + StringBuilder insertSql = + new StringBuilder("insert into root.vehicle.d1(timestamp,s1,s2,s3) values"); + int end = Math.min(start + INSERT_BATCH_SIZE, ITERATION_TIMES); + for (int i = start; i < end; i++) { + if (i > start) { + insertSql.append(','); + } + insertSql.append( + String.format( + Locale.ENGLISH, + "(%d,%f,%f,%f)", + (long) (i + 1) * DELTA_T, + Math.sin(i % PERIOD_1 * 2 * pi / (double) PERIOD_1), + Math.sin(i % PEROID_2 * 2 * pi / (double) PEROID_2), + 0.5 * Math.sin(i % PERIOD_1 * pi) + Math.sin(i % PEROID_2 * pi))); + } + statement.execute(insertSql.toString()); } statement.execute( String.format("insert into root.vehicle.d2(timestamp,s1) values(%d, 2)", DELTA_T)); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/PipeTestUtils.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/PipeTestUtils.java new file mode 100644 index 0000000000000..c5be3e3132a7d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/PipeTestUtils.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; +import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.db.pipe.source.dataregion.DataRegionListeningFilter; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.metrics.core.utils.IoTDBMetricObjNameFactory; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.UrlUtils; + +import org.junit.Assert; + +import javax.management.InstanceNotFoundException; +import javax.management.ObjectName; +import javax.management.remote.JMXConnector; +import javax.management.remote.JMXConnectorFactory; +import javax.management.remote.JMXServiceURL; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import static org.awaitility.Awaitility.await; + +public class PipeTestUtils { + + private static final long COMPLETION_REFLUSH_INTERVAL_MILLIS = 1000; + + private PipeTestUtils() {} + + /** + * Flush both environments after failed assertion polls, lazily retaining one Session per side. + * Close this handler before restarting either cluster and create a new one after recovery. + */ + public static final class RetryFlushHandler implements Consumer, AutoCloseable { + private final BaseEnv[] environments; + private final ISession[] sessions = new ISession[2]; + + @TestOnly + public RetryFlushHandler(final BaseEnv firstEnv, final BaseEnv secondEnv) { + environments = new BaseEnv[] {firstEnv, secondEnv}; + } + + @Override + public void accept(final String ignored) { + // A successful first poll opens no clients; failed polls reuse Sessions instead of + // repeatedly creating multi-node JDBC connections at the assertion's 100 ms interval. + for (int i = 0; i < environments.length; i++) { + try { + if (sessions[i] == null) { + sessions[i] = environments[i].getSessionConnection(); + } + sessions[i].executeNonQueryStatement("flush"); + } catch (final Exception e) { + throw new AssertionError(e); + } + } + } + + @Override + public void close() throws Exception { + // Always close the second side even if closing the first Session fails. + try { + if (sessions[0] != null) { + sessions[0].close(); + } + } finally { + if (sessions[1] != null) { + sessions[1].close(); + } + } + } + } + + /** Check every supplied IoTConsensus replica using live JMX gauges; enable its JMX reporter. */ + public static void assertDataRegionReplicationComplete( + final BaseEnv env, final List replicas) throws Exception { + Assert.assertFalse("No DataRegion replicas to check", replicas.isEmpty()); + // Node -> region ids: read all of a node's replicas through the same connection per sample. + final Map> regionsByNode = new HashMap<>(); + for (final TRegionInfo replica : replicas) { + final DataNodeWrapper node = + env.getDataNodeWrapperList().stream() + .filter(wrapper -> wrapper.getPort() == replica.getClientRpcPort()) + .findFirst() + .orElseThrow(() -> new AssertionError("Unknown DataRegion endpoint: " + replica)); + regionsByNode + .computeIfAbsent(node, ignored -> new ArrayList<>()) + .add(replica.getConsensusGroupId().getId()); + } + for (final Map.Entry> entry : regionsByNode.entrySet()) { + final DataNodeWrapper node = entry.getKey(); + final JMXServiceURL url = + new JMXServiceURL( + "service:jmx:rmi:///jndi/rmi://" + + UrlUtils.formatTEndPointIpv4AndIpv6Url(node.getIp(), node.getJmxPort()) + + "/jmxrmi"); + // Prometheus snapshots can retain either an old lag or an old zero after new writes. + // Read the AutoGauge directly before allowing a flush or restart of these replicas. + try (final JMXConnector connector = JMXConnectorFactory.connect(url)) { + for (final int regionId : entry.getValue()) { + final ObjectName metric = + IoTDBMetricObjNameFactory.getInstance() + .createName( + "IoTDBAutoGauge", + "org.apache.iotdb.metrics", + "iot_consensus", + Map.of( + "name", "ioTConsensusServerImpl", + "region", "DataRegion[" + regionId + "]", + "type", "syncLag")); + final Object value = connector.getMBeanServerConnection().getAttribute(metric, "Value"); + if (!(value instanceof Number lag)) { + throw new AssertionError("Non-numeric replication lag for " + metric + " at " + url); + } + Assert.assertEquals( + "Unreplicated data for " + metric + " at " + url, 0.0, lag.doubleValue(), 0.001); + } + } catch (final InstanceNotFoundException | IOException e) { + // A restarted replica may expose JMX before rebinding its region metrics. Retry within + // the caller's deadline, retaining the endpoint and underlying failure for diagnosis. + throw new AssertionError("Replication metrics unavailable at " + url, e); + } + } + } + + /** + * Wait for committed full-flush barriers, then let the caller verify the receiver's exact result. + * The caller must stop writing and use a running, completion-capable pipe with at least one + * captured DataRegion. Enable the sender's JMX reporter before cluster startup. This proves data + * completion only; it cannot replace observation of paused pipes, rejected events, or schema/auth + * synchronization. + */ + public static void awaitDataPipeCompletion(final BaseEnv senderEnv, final String pipeName) + throws Exception { + final CompletionSnapshot initial = readCompletionSnapshot(senderEnv, pipeName); + // RemainingEventCount can briefly be zero before capture. A full flush publishes an ordered + // barrier behind all preceding data, and completion_ready requires its successful commit. + TestUtils.executeNonQuery(senderEnv, "flush", null); + final long[] lastFlushTimeNanos = {System.nanoTime()}; + await() + .alias("Committed data completion of pipe " + pipeName) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .atMost(120, TimeUnit.SECONDS) + .untilAsserted( + () -> { + final CompletionSnapshot before = readCompletionSnapshot(senderEnv, pipeName); + + Assert.assertEquals( + "Pipe changed while awaiting completion", initial.staticMeta, before.staticMeta); + AssertionError completionFailure = null; + for (final DataNodeWrapper node : before.taskOwners.values()) { + try { + assertCompletionMetric(node, pipeName, initial.staticMeta.getCreationTime()); + } catch (final AssertionError e) { + completionFailure = e; + break; + } + } + + final CompletionSnapshot after = readCompletionSnapshot(senderEnv, pipeName); + Assert.assertEquals( + "Pipe changed while reading metrics", before.staticMeta, after.staticMeta); + Assert.assertEquals( + "DataRegion leaders changed while reading metrics", + before.regionLeaders, + after.regionLeaders); + if (completionFailure != null) { + reflushIfNeeded(senderEnv, lastFlushTimeNanos); + throw completionFailure; + } + }); + } + + private static void reflushIfNeeded(final BaseEnv senderEnv, final long[] lastFlushTimeNanos) + throws Exception { + final long now = System.nanoTime(); + if (now - lastFlushTimeNanos[0] + >= TimeUnit.MILLISECONDS.toNanos(COMPLETION_REFLUSH_INTERVAL_MILLIS)) { + // A new DataRegion can be assigned after the preceding flush. Re-publish the barrier only + // after a short quiet period so asynchronous task startup can be covered without flooding + // the sender with flush requests. + TestUtils.executeNonQuery(senderEnv, "flush", null); + lastFlushTimeNanos[0] = System.nanoTime(); + } + } + + private static CompletionSnapshot readCompletionSnapshot(final BaseEnv env, final String pipeName) + throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { + final TGetAllPipeInfoResp response = client.getAllPipeInfo(); + Assert.assertEquals( + response.toString(), + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + response.getStatus().getCode()); + PipeMeta pipe = null; + for (final ByteBuffer binary : response.getAllPipeInfo()) { + final PipeMeta candidate = PipeMeta.deserialize4Coordinator(binary); + if (pipeName.equals(candidate.getStaticMeta().getPipeName())) { + pipe = candidate; + break; + } + } + Assert.assertNotNull("Missing pipe " + pipeName, pipe); + Assert.assertTrue( + "Pipe does not support completion barriers: " + pipeName, + pipe.getStaticMeta().supportsCompletionMark()); + Assert.assertEquals( + "Pipe is not running: " + pipeName, + PipeStatus.RUNNING, + pipe.getRuntimeMeta().getStatus().get()); + Assert.assertTrue( + "Pipe node errors: " + pipe.getRuntimeMeta().getNodeId2PipeRuntimeExceptionMap(), + pipe.getRuntimeMeta().getNodeId2PipeRuntimeExceptionMap().isEmpty()); + + // Captured region -> pipe task leader, and leader node id -> endpoint. Derive the expected + // nodes from metadata rather than gauges, so a missing/failed source cannot count as + // completed. + final PipeStaticMeta staticMeta = pipe.getStaticMeta(); + final Map regionLeaders = new HashMap<>(); + final Map taskOwners = new HashMap<>(); + for (final boolean tableModel : new boolean[] {false, true}) { + final TShowRegionResp regions = + client.showRegion( + new TShowRegionReq() + .setConsensusGroupType(TConsensusGroupType.DataRegion) + .setIsTableModel(tableModel)); + Assert.assertEquals( + regions.toString(), + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + regions.getStatus().getCode()); + for (final TRegionInfo region : regions.getRegionInfoList()) { + // An excluded database creates no local source or completion metric. Match both models + // as the DataNode task builder does, so an excluded-only node is not mistaken for a + // missing source. Selected regions still require their task and completion metric. + if (!DataRegionListeningFilter.shouldDatabaseBeListened( + staticMeta.getSourceParameters(), + false, + region.getDatabase(), + staticMeta.getPipeType()) + && !DataRegionListeningFilter.shouldDatabaseBeListened( + staticMeta.getSourceParameters(), + true, + region.getDatabase(), + staticMeta.getPipeType())) { + continue; + } + final int regionId = region.getConsensusGroupId().getId(); + final PipeTaskMeta task = + pipe.getRuntimeMeta().getConsensusGroupId2TaskMetaMap().get(regionId); + Assert.assertNotNull("Missing pipe task for DataRegion " + regionId, task); + Assert.assertFalse( + "Pipe task errors in DataRegion " + + regionId + + ": " + + task.getExceptionMessagesString(), + task.hasExceptionMessages()); + regionLeaders.put(regionId, task.getLeaderNodeId()); + if (region.getDataNodeId() == task.getLeaderNodeId()) { + final DataNodeWrapper node = + env.getDataNodeWrapperList().stream() + .filter(wrapper -> wrapper.getPort() == region.getClientRpcPort()) + .findFirst() + .orElseThrow( + () -> new AssertionError("Unknown DataRegion endpoint: " + region)); + taskOwners.put(region.getDataNodeId(), node); + } + } + } + Assert.assertFalse( + "No captured DataRegions to await for " + pipeName, regionLeaders.isEmpty()); + Assert.assertTrue( + "Missing task owner endpoints: " + regionLeaders, + taskOwners.keySet().containsAll(regionLeaders.values())); + return new CompletionSnapshot(staticMeta, regionLeaders, taskOwners); + } + } + + private static void assertCompletionMetric( + final DataNodeWrapper node, final String pipeName, final long creationTime) throws Exception { + final JMXServiceURL url = + new JMXServiceURL( + "service:jmx:rmi:///jndi/rmi://" + + UrlUtils.formatTEndPointIpv4AndIpv6Url(node.getIp(), node.getJmxPort()) + + "/jmxrmi"); + // Prometheus serves a 15-second cached snapshot. Read the AutoGauge through JMX instead, so a + // previous successful wait cannot mask newly published data on the same pipe incarnation. + final ObjectName metric = + IoTDBMetricObjNameFactory.getInstance() + .createName( + "IoTDBAutoGauge", + "org.apache.iotdb.metrics", + "pipe_datanode_completion_ready", + Map.of("name", pipeName, "creation_time", String.valueOf(creationTime))); + try (final JMXConnector connector = JMXConnectorFactory.connect(url)) { + final Object value = connector.getMBeanServerConnection().getAttribute(metric, "Value"); + if (!(value instanceof Number completion)) { + throw new AssertionError("Non-numeric completion metric at " + url + ": " + value); + } + Assert.assertEquals( + "Uncommitted data for " + metric + " at " + url, 1.0, completion.doubleValue(), 0.0); + } catch (final InstanceNotFoundException e) { + throw new AssertionError("Missing completion metric " + metric + " at " + url, e); + } + } + + private static class CompletionSnapshot { + private final PipeStaticMeta staticMeta; + private final Map regionLeaders; + private final Map taskOwners; + + private CompletionSnapshot( + final PipeStaticMeta staticMeta, + final Map regionLeaders, + final Map taskOwners) { + this.staticMeta = staticMeta; + this.regionLeaders = regionLeaders; + this.taskOwners = taskOwners; + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java index 2aa5b49f331f6..0bf5130f1f9bf 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java @@ -24,7 +24,9 @@ import org.apache.iotdb.isession.pool.ITableSessionPool; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; @@ -52,6 +54,7 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Consumer; import static org.junit.Assert.fail; @@ -67,6 +70,50 @@ */ public class TableModelUtils { + // These helpers prepare data for replication assertions, rather than testing individual inserts. + // Keep every row and the caller's flush boundaries while reducing write RPCs to about one tenth. + private static final int ROWS_PER_INSERT = 10; + + private static final class SqlInsertBatch { + private final String prefix; + private final int rowsPerInsert; + private final List statements = new ArrayList<>(); + private final List values; + + private SqlInsertBatch(final String tableName) { + this(tableName, ROWS_PER_INSERT); + } + + private SqlInsertBatch(final String tableName, final int rowsPerInsert) { + this.rowsPerInsert = rowsPerInsert; + values = new ArrayList<>(rowsPerInsert); + prefix = + "insert into " + + tableName + + " (s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, time) values "; + } + + private void addValues(final String row) { + values.add(row); + if (values.size() == rowsPerInsert) { + appendStatement(); + } + } + + private void appendStatement() { + if (!values.isEmpty()) { + statements.add(prefix + String.join(",", values)); + values.clear(); + } + } + + private List finish() { + // Retain the final partial batch, including ranges smaller than the requested batch size. + appendStatement(); + return statements; + } + } + public static void createDataBaseAndTable( final BaseEnv baseEnv, final String table, final String database) { try (Connection connection = baseEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -104,13 +151,14 @@ public static void insertData( final int startInclusive, final int endExclusive, final BaseEnv baseEnv) { - List list = new ArrayList<>(endExclusive - startInclusive + 1); + final SqlInsertBatch batch = new SqlInsertBatch(tableName); for (int i = startInclusive; i < endExclusive; ++i) { - list.add( + batch.addValues( String.format( - "insert into %s (s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", - tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i)); + "('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", + i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i)); } + final List list = batch.finish(); list.add("flush"); TestUtils.executeNonQueries(dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list, null); } @@ -123,13 +171,57 @@ public static void insertData( final int startInclusive, final int endExclusive, final BaseEnv baseEnv) { - List list = new ArrayList<>(endExclusive - startInclusive + 1); + TestUtils.executeNonQueries( + dataBaseName, + BaseEnv.TABLE_SQL_DIALECT, + baseEnv, + generateInsertStatements( + tableName, + deviceStartIndex, + deviceEndIndex, + startInclusive, + endExclusive, + ROWS_PER_INSERT), + null); + } + + /** Reuses a caller-owned Session for SQL inserts, leaving all flush boundaries to the caller. */ + public static void insertData( + final String dataBaseName, + final String tableName, + final int deviceStartIndex, + final int deviceEndIndex, + final int startInclusive, + final int endExclusive, + final ITableSession session, + final int rowsPerInsert) + throws IoTDBConnectionException, StatementExecutionException { + session.executeNonQueryStatement("use " + dataBaseName); + for (final String statement : + generateInsertStatements( + tableName, + deviceStartIndex, + deviceEndIndex, + startInclusive, + endExclusive, + rowsPerInsert)) { + session.executeNonQueryStatement(statement); + } + } + + private static List generateInsertStatements( + final String tableName, + final int deviceStartIndex, + final int deviceEndIndex, + final int startInclusive, + final int endExclusive, + final int rowsPerInsert) { + final SqlInsertBatch batch = new SqlInsertBatch(tableName, rowsPerInsert); for (int deviceIndex = deviceStartIndex; deviceIndex < deviceEndIndex; ++deviceIndex) { for (int i = startInclusive; i < endExclusive; ++i) { - list.add( + batch.addValues( String.format( - "insert into %s (s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", - tableName, + "('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", deviceIndex, deviceIndex, deviceIndex, @@ -145,7 +237,7 @@ public static void insertData( i)); } } - TestUtils.executeNonQueries(dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list, null); + return batch.finish(); } public static void insertData( @@ -155,7 +247,7 @@ public static void insertData( final int end, final BaseEnv baseEnv, final boolean allowNullValue) { - List list = new ArrayList<>(end - start + 1); + final SqlInsertBatch batch = new SqlInsertBatch(tableName); Object[] values = new Object[12]; Random random = new Random(); // s0 string, s1 string, s2 string, s3 string, s4 int64, s5 float, s6 string s7 timestamp, s8 @@ -177,10 +269,9 @@ public static void insertData( if (allowNullValue) { values[random.nextInt(9)] = "null"; } - list.add( + batch.addValues( String.format( - "insert into %s (s0, s1, s2, s3, s4, s5, s6, s7, s8,s9, s10, s11, time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", - tableName, + "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", values[0], values[1], values[2], @@ -195,7 +286,8 @@ public static void insertData( values[11], i)); } - TestUtils.executeNonQueries(dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list, null); + TestUtils.executeNonQueries( + dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, batch.finish(), null); } public static boolean insertDataNotThrowError( @@ -245,12 +337,9 @@ public static boolean insertTablet( final int end, final BaseEnv baseEnv, final boolean allowNullValue) { - final Tablet tablet = generateTablet(tableName, start, end, allowNullValue, true); - ITableSessionPool tableSessionPool = baseEnv.getTableSessionPool(1); - try (final ITableSession session = tableSessionPool.getSession()) { - session.executeNonQueryStatement("use " + dataBaseName); - session.insert(tablet); - session.executeNonQueryStatement("flush"); + try (final ITableSessionPool tableSessionPool = baseEnv.getTableSessionPool(1); + final ITableSession session = tableSessionPool.getSession()) { + insertTablet(dataBaseName, tableName, start, end, session, allowNullValue); return true; } catch (Exception e) { e.printStackTrace(); @@ -258,18 +347,76 @@ public static boolean insertTablet( } } + /** Reuses a caller-owned Session while retaining one flush for each Tablet. */ + public static void insertTablet( + final String dataBaseName, + final String tableName, + final int start, + final int end, + final ITableSession session, + final boolean allowNullValue) + throws Exception { + final Tablet tablet = generateTablet(tableName, start, end, allowNullValue, true); + session.executeNonQueryStatement("use " + dataBaseName); + session.insert(tablet); + session.executeNonQueryStatement("flush"); + } + public static boolean insertTablet( final String dataBaseName, final Tablet tablet, final BaseEnv baseEnv) { + return insertTablet(dataBaseName, tablet, baseEnv, null); + } + + public enum TabletInsertPhase { + OPEN_SESSION, + USE_DATABASE, + INSERT, + FLUSH, + CLOSE_SESSION + } + + /** + * Records client-side step durations in nanoseconds without changing the per-Tablet connection or + * flush boundaries. If a step fails, its duration includes automatic resource cleanup. + */ + public static boolean insertTablet( + final String dataBaseName, + final Tablet tablet, + final BaseEnv baseEnv, + final BiConsumer recordTiming) { + long stepStart = System.nanoTime(); + TabletInsertPhase phase = TabletInsertPhase.OPEN_SESSION; try (ITableSessionPool tableSessionPool = baseEnv.getTableSessionPool(20); final ITableSession session = tableSessionPool.getSession()) { + stepStart = recordTabletInsertTime(recordTiming, phase, stepStart); + phase = TabletInsertPhase.USE_DATABASE; session.executeNonQueryStatement("use " + dataBaseName); + stepStart = recordTabletInsertTime(recordTiming, phase, stepStart); + phase = TabletInsertPhase.INSERT; session.insert(tablet); + stepStart = recordTabletInsertTime(recordTiming, phase, stepStart); + phase = TabletInsertPhase.FLUSH; session.executeNonQueryStatement("flush"); + stepStart = recordTabletInsertTime(recordTiming, phase, stepStart); + phase = TabletInsertPhase.CLOSE_SESSION; return true; } catch (Exception e) { e.printStackTrace(); return false; + } finally { + recordTabletInsertTime(recordTiming, phase, stepStart); + } + } + + private static long recordTabletInsertTime( + final BiConsumer recordTiming, + final TabletInsertPhase phase, + final long start) { + final long end = System.nanoTime(); + if (recordTiming != null) { + recordTiming.accept(phase, end - start); } + return end; } public static void deleteData( @@ -840,7 +987,16 @@ public static Tablet generateTabletDeviceIDAllIsNull( public static int showPipesCount(final BaseEnv baseEnv, final String sqlDialect) { try (final Connection connection = baseEnv.getConnection(sqlDialect); final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); + return showPipesCount(statement); + } catch (final SQLException e) { + fail(e.getMessage()); + } + return 0; + } + + /** Count visible pipes without closing the caller's dialect-specific Statement or connection. */ + public static int showPipesCount(final Statement statement) throws SQLException { + try (final ResultSet resultSet = statement.executeQuery("show pipes")) { int count = 0; while (resultSet.next()) { if (resultSet.getString("ID").startsWith("__consensus")) { @@ -849,9 +1005,6 @@ public static int showPipesCount(final BaseEnv baseEnv, final String sqlDialect) count++; } return count; - } catch (final SQLException e) { - fail(e.getMessage()); } - return 0; } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java index e304fc81d965e..3cb0fae167334 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java @@ -26,6 +26,9 @@ import org.junit.After; import org.junit.Before; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + public abstract class AbstractPipeTableModelDualManualIT { protected BaseEnv senderEnv; @@ -37,8 +40,16 @@ public void setUp() { senderEnv = MultiEnvFactory.getEnv(0); receiverEnv = MultiEnvFactory.getEnv(1); setupConfig(); - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); + executeOnBothEnvs(BaseEnv::initClusterEnvironment); + } + + protected void executeOnBothEnvs(final Consumer action) { + // The environments own separate node directories and ports. Await both tasks even if one + // fails, so teardown cannot race with a cluster that is still starting or restarting. + CompletableFuture.allOf( + CompletableFuture.runAsync(() -> action.accept(senderEnv)), + CompletableFuture.runAsync(() -> action.accept(receiverEnv))) + .join(); } protected void setupConfig() { @@ -74,6 +85,10 @@ protected void setupConfig() { @After public final void tearDown() { + cleanEnvironment(); + } + + protected void cleanEnvironment() { senderEnv.cleanClusterEnvironment(); receiverEnv.cleanClusterEnvironment(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelSharedClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelSharedClusterIT.java new file mode 100644 index 0000000000000..599626a69f36e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelSharedClusterIT.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.ExternalResource; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.junit.Assert.assertEquals; + +/** For pipe tests whose cluster configuration stays constant across all methods in a class. */ +public abstract class AbstractPipeTableModelSharedClusterIT + extends AbstractPipeTableModelDualManualIT { + + private static BaseEnv sharedSender; + private static BaseEnv sharedReceiver; + private static CompletableFuture startup; + + @Override + protected void setupConfig() { + super.setupConfig(); + // These small functional workloads verify models/patterns, not automatic region expansion. + // CPU-based defaults create many tiny TsFiles and concurrent loads can exhaust the IT heap. + // Keep a region per database; separate databases and the configured replicas remain covered. + for (final BaseEnv env : new BaseEnv[] {senderEnv, receiverEnv}) { + env.getConfig() + .getCommonConfig() + .setDataRegionGroupExtensionPolicy("CUSTOM") + .setDefaultDataRegionGroupNumPerDatabase(1); + } + } + + @ClassRule + public static final ExternalResource CLUSTERS = + new ExternalResource() { + @Override + protected void after() { + // Always release both environments, including when initialization or a test fails. + try { + if (sharedSender != null) { + sharedSender.cleanClusterEnvironment(); + } + } finally { + try { + if (sharedReceiver != null) { + sharedReceiver.cleanClusterEnvironment(); + } + } finally { + sharedSender = null; + sharedReceiver = null; + startup = null; + } + } + } + }; + + @Override + @Before + public void setUp() { + if (sharedSender == null) { + MultiEnvFactory.createEnv(2); + sharedSender = MultiEnvFactory.getEnv(0); + sharedReceiver = MultiEnvFactory.getEnv(1); + senderEnv = sharedSender; + receiverEnv = sharedReceiver; + setupConfig(); + // Join both starts before any test or cleanup runs; retain failures for subsequent methods. + startup = initializeClusters(); + } + senderEnv = sharedSender; + receiverEnv = sharedReceiver; + startup.join(); + } + + protected CompletableFuture initializeClusters() { + return CompletableFuture.allOf( + CompletableFuture.runAsync(senderEnv::initClusterEnvironment), + CompletableFuture.runAsync(receiverEnv::initClusterEnvironment)); + } + + @Override + protected void cleanEnvironment() { + if (startup == null || startup.isCompletedExceptionally()) { + // A failed start is cleaned by the class rule; do not open connections to unavailable nodes. + return; + } + try { + // Drop all user pipes in both SQL dialects before deleting any database. Otherwise an active + // pipe can recreate the receiver's data during cleanup and contaminate the next method. + dropUserPipes(senderEnv); + dropUserPipes(receiverEnv); + resetCatalog(senderEnv); + resetCatalog(receiverEnv); + } catch (Exception e) { + throw new AssertionError("Failed to reset shared pipe test clusters", e); + } + } + + private static void dropUserPipes(final BaseEnv env) throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { + for (final boolean tableModel : new boolean[] {false, true}) { + final TShowPipeResp response = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(tableModel) + .setUserName(SessionConfig.DEFAULT_USER)); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + for (final TShowPipeInfo pipe : response.getPipeInfoList()) { + if (!pipe.getId().startsWith(PipeStaticMeta.SYSTEM_PIPE_PREFIX)) { + final int statusCode = + client + .dropPipeExtended(new TDropPipeReq(pipe.getId()).setIsTableModel(tableModel)) + .getCode(); + // Automatic drop can remove a pipe after the SHOW PIPE snapshot. + if (statusCode != TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode()) { + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), statusCode); + } + } + } + } + } + } + + private static void resetCatalog(final BaseEnv env) throws Exception { + for (final String dialect : + new String[] {BaseEnv.TREE_SQL_DIALECT, BaseEnv.TABLE_SQL_DIALECT}) { + try (final Connection connection = env.getConnection(dialect); + final Statement statement = connection.createStatement()) { + final List databases = new ArrayList<>(); + try (final ResultSet result = statement.executeQuery("show databases")) { + while (result.next()) { + final String database = result.getString(1); + if (!"information_schema".equals(database)) { + databases.add(database); + } + } + } + for (final String database : databases) { + if (BaseEnv.TABLE_SQL_DIALECT.equals(dialect)) { + // A preceding DROP TABLE/VIEW can still be running after the pipe progress marker. + TestUtils.dropDatabaseWithRetry( + statement, "\"" + database.replace("\"", "\"\"") + "\""); + } else { + statement.execute("delete database " + database); + } + } + if (BaseEnv.TREE_SQL_DIALECT.equals(dialect)) { + deleteUsersAndRoles(statement); + } + } + } + } + + private static void deleteUsersAndRoles(final Statement statement) throws SQLException { + // Authorization is shared by both dialects and survives database deletion. Remove the users + // and roles created/replicated by a scenario, including their grants and role memberships. + // Reuse the tree connection and retain the administrator needed by subsequent methods. + for (final String type : new String[] {"user", "role"}) { + final List names = new ArrayList<>(); + try (final ResultSet result = statement.executeQuery("list " + type)) { + while (result.next()) { + final String name = result.getString("user".equals(type) ? "User" : "Role"); + if (!"user".equals(type) || !SessionConfig.DEFAULT_USER.equals(name)) { + names.add(name); + } + } + } + for (final String name : names) { + statement.execute("drop " + type + " `" + name.replace("`", "``") + "`"); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java index b2e1aaf539ab8..f9d07e141df30 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java @@ -29,7 +29,7 @@ import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.junit.Assert; import org.junit.Before; @@ -47,7 +47,7 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeAlterIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeAlterIT extends AbstractPipeTableModelSharedClusterIT { @Override @Before @@ -55,6 +55,8 @@ public void setUp() { super.setUp(); } + // MODIFY/REPLACE must preserve stopped/running state, advance creation time, and apply + // source/processor/sink changes, including empty-attribute resets and lowercase table filters. @Test public void testBasicAlterPipe() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -65,365 +67,331 @@ public void testBasicAlterPipe() throws Exception { String.format( "create pipe a2b with source ('source'='iotdb-source', 'database-name'='Test', 'table-name'='Test1', 'mode.streaming'='true') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", receiverDataNode.getIpAndPortString()); + // This sequence keeps the same cluster and dialect; retain both clients through every + // operation while preserving the metadata check after each change. try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + final Statement statement = connection.createStatement(); + final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { statement.execute(sql); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // show pipe - long lastCreationTime; - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe( - new TShowPipeReq().setIsTableModel(true).setUserName(SessionConfig.DEFAULT_USER)) - .pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("mode.streaming=true")); - Assert.assertTrue( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Record last creation time - lastCreationTime = showPipeResult.get(0).creationTime; - } - // Stop pipe - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + // show pipe + long lastCreationTime; + { + final List showPipeResult = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(true) + .setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("mode.streaming=true")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Record last creation time + lastCreationTime = showPipeResult.get(0).creationTime; + } + + // Stop pipe statement.execute("stop pipe a2b"); - } catch (SQLException e) { - fail(e.getMessage()); - } - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe( - new TShowPipeReq().setIsTableModel(true).setUserName(SessionConfig.DEFAULT_USER)) - .pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("STOPPED", showPipeResult.get(0).state); - } - - // Alter pipe (modify) - // The database & table name will be converted to lower case - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + // Show pipe + { + final List showPipeResult = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(true) + .setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + } + + // Alter pipe (modify) + // The database & table name will be converted to lower case statement.execute( "alter pipe a2b modify source ('table-name'='Test1','database-name'='Test1')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe( - new TShowPipeReq().setIsTableModel(true).setUserName(SessionConfig.DEFAULT_USER)) - .pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("STOPPED", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test1")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test1")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("mode.streaming=true")); - Assert.assertTrue( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - // Alter pipe (replace) - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + // Show pipe + { + final List showPipeResult = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(true) + .setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test1")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test1")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("mode.streaming=true")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (replace) statement.execute( "alter pipe a2b replace source ('capture.table'='true','''source'='iotdb-source', 'table-name'='test','database-name'='test')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe( - new TShowPipeReq().setIsTableModel(true).setUserName(SessionConfig.DEFAULT_USER)) - .pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("STOPPED", showPipeResult.get(0).state); - // check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("mode.streaming=true")); - Assert.assertTrue( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Alter pipe (modify) - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + // Show pipe + { + final List showPipeResult = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(true) + .setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + // check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("mode.streaming=true")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify) statement.execute("alter pipe a2b modify sink ('sink.batch.enable'='false')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe( - new TShowPipeReq().setIsTableModel(true).setUserName(SessionConfig.DEFAULT_USER)) - .pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("STOPPED", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=false")); - Assert.assertTrue( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Start pipe - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + // Show pipe + { + final List showPipeResult = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(true) + .setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=false")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Start pipe statement.execute("start pipe a2b"); - } catch (SQLException e) { - fail(e.getMessage()); - } - // Alter pipe (modify) - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + // Alter pipe (modify) statement.execute("alter pipe a2b modify sink ('connector.batch.enable'='true')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe( - new TShowPipeReq().setIsTableModel(true).setUserName(SessionConfig.DEFAULT_USER)) - .pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - // Alter pipe (modify empty) - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + // Show pipe + { + final List showPipeResult = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(true) + .setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify empty) statement.execute("alter pipe a2b modify source ()"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe( - new TShowPipeReq().setIsTableModel(true).setUserName(SessionConfig.DEFAULT_USER)) - .pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - // Alter pipe (replace empty) - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + // show pipe + { + final List showPipeResult = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(true) + .setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (replace empty) statement.execute("alter pipe a2b replace source ()"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe( - new TShowPipeReq().setIsTableModel(true).setUserName(SessionConfig.DEFAULT_USER)) - .pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // check configurations - - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("__system.sql-dialect=table")); - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - // Alter pipe (replace empty) - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + // show pipe + { + final List showPipeResult = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(true) + .setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + + Assert.assertTrue( + showPipeResult.get(0).pipeExtractor.contains("__system.sql-dialect=table")); + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (replace empty) statement.execute("alter pipe a2b replace processor ()"); - } catch (SQLException e) { - fail(e.getMessage()); - } - // show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe( - new TShowPipeReq().setIsTableModel(true).setUserName(SessionConfig.DEFAULT_USER)) - .pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // check configurations - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertFalse( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Alter pipe (modify empty) - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + // show pipe + { + final List showPipeResult = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(true) + .setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertFalse( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify empty) statement.execute("alter pipe a2b modify sink ()"); + + // Show pipe + { + final List showPipeResult = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(true) + .setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } } catch (SQLException e) { fail(e.getMessage()); } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe( - new TShowPipeReq().setIsTableModel(true).setUserName(SessionConfig.DEFAULT_USER)) - .pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } } + // ALTER must reject a missing table-model pipe, and the same connection must then create it. @Test public void testAlterPipeFailure() { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -435,18 +403,17 @@ public void testAlterPipeFailure() { receiverDataNode.getIpAndPortString()); try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { - statement.execute(sql); - fail(); - } catch (SQLException ignore) { - } - - // Create pipe - sql = - String.format( - "create pipe a2b with source ('source'='iotdb-source', 'database-name'='test', 'table-name'='test1', 'mode.streaming'='true') with sink ('node-urls'='%s', 'batch.enable'='false')", - receiverDataNode.getIpAndPortString()); - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { + statement.execute(sql); + fail(); + } catch (SQLException ignore) { + } + + // Create pipe + sql = + String.format( + "create pipe a2b with source ('source'='iotdb-source', 'database-name'='test', 'table-name'='test1', 'mode.streaming'='true') with sink ('node-urls'='%s', 'batch.enable'='false')", + receiverDataNode.getIpAndPortString()); statement.execute(sql); } catch (SQLException e) { fail(e.getMessage()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java index 2a5c9352e2542..55c4e29779352 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java @@ -24,18 +24,25 @@ import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.pool.ITableSessionPool; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestLogger; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.write.record.Tablet; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.util.ArrayList; @@ -43,16 +50,24 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; +import java.util.LongSummaryStatistics; import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.function.BiConsumer; import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeDataSinkIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeDataSinkIT extends AbstractPipeTableModelSharedClusterIT { + + // Two full rounds cover both databases and five tables. One more Tablet per database keeps + // overlapping writes across flushes, with 12 writes instead of 125 (approximately one tenth). + private static final int[] TABLETS_PER_ROUND = {5, 5, 1, 1}; + + @Rule public final TestName testName = new TestName(); @Override @Before @@ -60,27 +75,25 @@ public void setUp() { super.setUp(); } + // With realtime-first disabled, batched thrift transfer must deliver both history/realtime + // table rows and tree points. @Test public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; + try (final RetryFlushHandler handleFailure = new RetryFlushHandler(null); + final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.insertData("test", "test", 0, 50, senderEnv, true); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", database("test")); + TableModelUtils.insertData(database("test"), "test", 0, 50, senderEnv, true); TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (0, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (0, 1)", "flush")), null); final Map sourceAttributes = new HashMap<>(); @@ -111,39 +124,46 @@ public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - TableModelUtils.insertData("test", "test", 50, 100, senderEnv, true); + TableModelUtils.insertData(database("test"), "test", 50, 100, senderEnv, true); TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush")), null); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.vehicle.**", - "Time,root.vehicle.d0.s1,", + isolateDatabasePaths("select * from root.vehicle.**"), + isolateDatabasePaths("Time,root.vehicle.d0.s1,"), Collections.unmodifiableSet(new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,"))), handleFailure); - TableModelUtils.assertCountData("test", "test", 100, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test", 100, receiverEnv, handleFailure); } } + // Tablet-format transfer must preserve the mixed history/realtime data in both models. @Test public void testSinkTabletFormat() throws Exception { testSinkFormat("tablet", false); } + // TsFile-format transfer must preserve the mixed history/realtime data in both models. @Test public void testSinkTsFileFormat() throws Exception { testSinkFormat("tsfile", false); } + // Asynchronously loading transferred TsFiles must eventually expose every expected table row + // and tree point. @Test public void testTsFileFormatAndAsyncLoad() throws Exception { testSinkFormat("tsfile", true); } + // Hybrid-format transfer must preserve all history/realtime rows across the tested flush + // boundaries. @Test public void testSinkHybridFormat() throws Exception { testSinkFormat("hybrid", false); @@ -154,21 +174,17 @@ private void testSinkFormat(final String format, final boolean isAsyncLoad) thro final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; + try (final RetryFlushHandler handleFailure = new RetryFlushHandler(null); + final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.insertData("test", "test", 0, 50, senderEnv, true); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", database("test")); + TableModelUtils.insertData(database("test"), "test", 0, 50, senderEnv, true); TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush")), null); final Map sourceAttributes = new HashMap<>(); @@ -186,6 +202,11 @@ private void testSinkFormat(final String format, final boolean isAsyncLoad) thro sinkAttributes.put("sink.ip", receiverIp); sinkAttributes.put("sink.port", Integer.toString(receiverPort)); sinkAttributes.put("sink.format", format); + if ("tsfile".equals(format)) { + // Keep TsFile batching, but avoid its 5-second default wait for these small workloads. + // Tablet/hybrid formats already use a shorter default and must not inherit this delay. + sinkAttributes.put("sink.batch.max-delay-ms", "1000"); + } sinkAttributes.put("sink.load-tsfile-strategy", isAsyncLoad ? "async" : "sync"); sinkAttributes.put("sink.realtime-first", "false"); @@ -201,19 +222,20 @@ private void testSinkFormat(final String format, final boolean isAsyncLoad) thro Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - TableModelUtils.insertData("test", "test", 50, 150, senderEnv, true); + TableModelUtils.insertData(database("test"), "test", 50, 150, senderEnv, true); TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush")), null); - TableModelUtils.assertCountData("test", "test", 150, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test", 150, receiverEnv, handleFailure); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.vehicle.**", - "Time,root.vehicle.d0.s1,", + isolateDatabasePaths("select * from root.vehicle.**"), + isolateDatabasePaths("Time,root.vehicle.d0.s1,"), Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,"))), handleFailure); @@ -238,28 +260,30 @@ private void testSinkFormat(final String format, final boolean isAsyncLoad) thro TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.vehicle.d0(time, s1) values (4, 1)", - "insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)", + isolateDatabasePaths("insert into root.vehicle.d0(time, s1) values (4, 1)"), + isolateDatabasePaths("insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)"), "flush"), null); - TableModelUtils.insertData("test", "test", 150, 200, senderEnv, true); - TableModelUtils.insertTablet("test", "test", 200, 250, senderEnv, true); - TableModelUtils.insertTablet("test", "test", 250, 300, senderEnv, true); - TableModelUtils.insertTablet("test", "test", 300, 350, senderEnv, true); + TableModelUtils.insertData(database("test"), "test", 150, 200, senderEnv, true); + TableModelUtils.insertTablet(database("test"), "test", 200, 250, senderEnv, true); + TableModelUtils.insertTablet(database("test"), "test", 250, 300, senderEnv, true); + TableModelUtils.insertTablet(database("test"), "test", 300, 350, senderEnv, true); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.vehicle.**", - "Time,root.vehicle.d0.s1,", + isolateDatabasePaths("select * from root.vehicle.**"), + isolateDatabasePaths("Time,root.vehicle.d0.s1,"), Collections.unmodifiableSet( new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,"))), handleFailure); - TableModelUtils.assertCountData("test", "test", 350, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test", 350, receiverEnv, handleFailure); } } + // Writing back to database Test1 with forwarding disabled must produce all 400 table rows + // without a forwarding cycle. @Test public void testWriteBackSink() throws Exception { try (final SyncConfigNodeIServiceClient client = @@ -273,12 +297,12 @@ public void testWriteBackSink() throws Exception { sourceAttributes.put("capture.tree", "true"); sourceAttributes.put("mode.double-living", "true"); sourceAttributes.put("forwarding-pipe-requests", "false"); - sourceAttributes.put("source.database-name", "test.*"); + sourceAttributes.put("source.database-name", databasePattern("test.*")); sourceAttributes.put("source.table-name", "test.*"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "write-back-sink"); - sinkAttributes.put("sink.database", "Test1"); + sinkAttributes.put("sink.database", database("Test1")); sinkAttributes.put("user", "root"); final TSStatus status = @@ -294,76 +318,103 @@ public void testWriteBackSink() throws Exception { TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush")), null); - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.insertDataNotThrowError("test", "test", 0, 20, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", database("test")); + TableModelUtils.insertDataNotThrowError(database("test"), "test", 0, 20, senderEnv); - TableModelUtils.insertTablet("test", "test", 20, 200, senderEnv, true); + TableModelUtils.insertTablet(database("test"), "test", 20, 200, senderEnv, true); - TableModelUtils.insertTablet("test", "test", 200, 400, senderEnv, true); + TableModelUtils.insertTablet(database("test"), "test", 200, 400, senderEnv, true); - TableModelUtils.assertCountData("test1", "test", 400, senderEnv); + TableModelUtils.assertCountData(database("Test1"), "test", 400, senderEnv); } } + // TsFile transfer of shifting device ranges with fixed timestamps must match all generated + // Tablet values. @Test public void testSinkTsFileFormat2() throws Exception { doTest(this::insertTablet1); } + // TsFile transfer of shifting device ranges with generated timestamps must match all generated + // Tablet values. @Test public void testSinkTsFileFormat3() throws Exception { doTest(this::insertTablet2); } + // TsFile transfer of random device/time ranges must preserve every value from the third Tablet + // generator. @Test public void testSinkTsFileFormat4() throws Exception { doTest(this::insertTablet3); } + // TsFile transfer of random device/time ranges must preserve every value from the fourth Tablet + // generator. @Test public void testSinkTsFileFormat5() throws Exception { doTest(this::insertTablet4); } + // TsFile transfer with fixed device IDs and random time ranges must match all generated values. @Test public void testSinkTsFileFormat6() throws Exception { doTest(this::insertTablet5); } + // TsFile transfer with random device ranges and timestamps [100, 110) must match all generated + // values. @Test public void testSinkTsFileFormat7() throws Exception { doTest(this::insertTablet6); } + // TsFile transfer with shifting device ranges and timestamps [100, 110) must match all + // generated values. @Test public void testSinkTsFileFormat8() throws Exception { doTest(this::insertTablet7); } + // TsFile transfer with device IDs [100, 110) and shifting time ranges must match all generated + // values. @Test public void testSinkTsFileFormat9() throws Exception { doTest(this::insertTablet8); } + // TsFile transfer of Tablets with entirely null device IDs must match all generated rows. @Test public void testSinkTsFileFormat10() throws Exception { doTest(this::insertTablet9); } - private void doTest(BiConsumer>, Map>> consumer) + private void doTest(final TabletWriter consumer) throws Exception { + // This covers doTest only; cluster setup/cleanup is reported by IoTDBTestRunner separately. + // Close the recorder on assertion failures too, so slow failures retain their partial timings. + try (final DoTestTimings timings = new DoTestTimings(testName.getMethodName())) { + try (final RetryFlushHandler handleFailure = new RetryFlushHandler(timings)) { + doTest(consumer, timings, handleFailure); + } + // A Session close failure also fails the test, so mark success only after closing clients. + timings.passed = true; + } + } + + private void doTest( + final TabletWriter consumer, + final DoTestTimings timings, + final RetryFlushHandler handleFailure) throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; Map> testResult = new HashMap<>(); Map> test1Result = new HashMap<>(); @@ -371,16 +422,20 @@ private void doTest(BiConsumer>, Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); final Map sinkAttributes = new HashMap<>(); @@ -389,7 +444,7 @@ private void doTest(BiConsumer>, Map>, Map>, Map(Arrays.asList("1,1.0,", "2,1.0,"))), handleFailure); + timings.startPhase(DoTestPhase.OUT_OF_ORDER_TREE_WRITE_AND_FLUSH); TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.vehicle.d0(time, s1) values (4, 1)", - "insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)", + isolateDatabasePaths("insert into root.vehicle.d0(time, s1) values (4, 1)"), + isolateDatabasePaths("insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)"), "flush"), null); + timings.startPhase(DoTestPhase.FINAL_TREE_CHECK); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.vehicle.**", - "Time,root.vehicle.d0.s1,", + isolateDatabasePaths("select * from root.vehicle.**"), + isolateDatabasePaths("Time,root.vehicle.d0.s1,"), Collections.unmodifiableSet( new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,"))), handleFailure); + timings.startPhase(DoTestPhase.CLOSE_CONFIG_CLIENT); } for (Map.Entry> entry : testResult.entrySet()) { + timings.startPhase(DoTestPhase.EXPECTED_RESULTS); final Set set = new HashSet<>(); entry .getValue() @@ -450,12 +517,16 @@ private void doTest(BiConsumer>, Map { set.addAll(TableModelUtils.generateExpectedResults(tablet)); }); + timings.startPhase(DoTestPhase.COUNT_CHECK_WITH_RECEIVER_FLUSH); TableModelUtils.assertCountData( - "test0", entry.getKey(), set.size(), receiverEnv, handleFailure); - TableModelUtils.assertData("test0", entry.getKey(), set, receiverEnv, handleFailure); + database("test0"), entry.getKey(), set.size(), receiverEnv, handleFailure); + timings.startPhase(DoTestPhase.DATA_CHECK); + TableModelUtils.assertData( + database("test0"), entry.getKey(), set, receiverEnv, handleFailure); } for (Map.Entry> entry : test1Result.entrySet()) { + timings.startPhase(DoTestPhase.EXPECTED_RESULTS); final Set set = new HashSet<>(); entry .getValue() @@ -463,31 +534,241 @@ private void doTest(BiConsumer>, Map { set.addAll(TableModelUtils.generateExpectedResults(tablet)); }); + timings.startPhase(DoTestPhase.COUNT_CHECK_WITH_RECEIVER_FLUSH); TableModelUtils.assertCountData( - "test1", entry.getKey(), set.size(), receiverEnv, handleFailure); - TableModelUtils.assertData("test1", entry.getKey(), set, receiverEnv, handleFailure); + database("test1"), entry.getKey(), set.size(), receiverEnv, handleFailure); + timings.startPhase(DoTestPhase.DATA_CHECK); + TableModelUtils.assertData( + database("test1"), entry.getKey(), set, receiverEnv, handleFailure); + } + } + + @FunctionalInterface + private interface TabletWriter { + void accept( + Map> testResult, + Map> test1Result, + DoTestTimings timings, + ITableSession session) + throws Exception; + } + + private final class RetryFlushHandler implements Consumer, AutoCloseable { + // Index 0 owns the sender Session, index 1 owns the receiver Session. Open lazily because + // assertions that pass immediately need no retry clients. + private final BaseEnv[] environments = {senderEnv, receiverEnv}; + private final ISession[] sessions = new ISession[2]; + private final DoTestTimings timings; + + private RetryFlushHandler(final DoTestTimings timings) { + this.timings = timings; + } + + @Override + public void accept(final String ignored) { + // A failed poll can run every 100 ms. Reuse the flush connections instead of repeatedly + // opening multi-node JDBC connections and exhausting the client's ephemeral ports. + for (int i = 0; i < sessions.length; i++) { + final long started = System.nanoTime(); + try { + if (sessions[i] == null) { + sessions[i] = environments[i].getSessionConnection(); + } + sessions[i].executeNonQueryStatement("flush"); + } catch (Exception e) { + throw new AssertionError(e); + } finally { + if (timings != null) { + timings.recordDetail( + i == 0 ? "RETRY_SENDER_FLUSH" : "RETRY_RECEIVER_FLUSH", + System.nanoTime() - started); + } + } + } + } + + @Override + public void close() throws Exception { + try { + if (sessions[0] != null) { + sessions[0].close(); + } + } finally { + if (sessions[1] != null) { + sessions[1].close(); + } + } + } + } + + private enum DoTestPhase { + OPEN_CONFIG_CLIENT, + CREATE_SCHEMA, + HISTORY_TREE_WRITE_AND_FLUSH, + PIPE_ATTRIBUTES, + CREATE_PIPE, + START_PIPE, + REALTIME_TREE_WRITE_AND_FLUSH, + TABLET_WORKLOAD, + INITIAL_TREE_CHECK, + OUT_OF_ORDER_TREE_WRITE_AND_FLUSH, + FINAL_TREE_CHECK, + CLOSE_CONFIG_CLIENT, + EXPECTED_RESULTS, + COUNT_CHECK_WITH_RECEIVER_FLUSH, + DATA_CHECK + } + + // These diagnostics belong to the test source set, which has no compile-time i18n source sets. + private static final class TimingMessages { + public static final String SUMMARY = + "DataSink doTest timing: test={}, passed={}, total_ms={}, scope=doTest-only; " + + "STAGE rows partition total time; DETAIL rows are included in STAGE rows; " + + "checks include JDBC/query/polling/retry time, not isolated Pipe transfer time"; + public static final String ROW = + "DataSink doTest timing: test={}, kind={}, phase={}, calls={}, total_ms={}, " + + "avg_ms={}, max_ms={}, percent_of_doTest={}"; + public static final String NUMBER = "%.3f"; + } + + private static final class DoTestTimings implements AutoCloseable { + private final String method; + private final long started = System.nanoTime(); + // Keys name client-side stages/details; values accumulate nanoseconds for this doTest only. + private final Map stages = new LinkedHashMap<>(); + private final Map details = new LinkedHashMap<>(); + private DoTestPhase phase = DoTestPhase.OPEN_CONFIG_CLIENT; + private long phaseStarted = started; + private boolean passed; + + private DoTestTimings(final String method) { + this.method = method; + } + + private void startPhase(final DoTestPhase next) { + final long now = System.nanoTime(); + stages + .computeIfAbsent(phase.name(), key -> new LongSummaryStatistics()) + .accept(now - phaseStarted); + phase = next; + phaseStarted = now; + } + + private void recordDetail(final String step, final long nanos) { + details.computeIfAbsent(step, key -> new LongSummaryStatistics()).accept(nanos); + } + + @Override + public void close() { + final long finished = System.nanoTime(); + stages + .computeIfAbsent(phase.name(), key -> new LongSummaryStatistics()) + .accept(finished - phaseStarted); + final long total = finished - started; + IoTDBTestLogger.logger.info( + TimingMessages.SUMMARY, method, passed, format(total / 1_000_000.0)); + logRows("STAGE", stages, total); + logRows("DETAIL", details, total); + } + + private void logRows( + final String kind, final Map rows, final long total) { + rows.forEach( + (step, statistics) -> + IoTDBTestLogger.logger.info( + TimingMessages.ROW, + method, + kind, + step, + statistics.getCount(), + format(statistics.getSum() / 1_000_000.0), + format(statistics.getAverage() / 1_000_000.0), + format(statistics.getMax() / 1_000_000.0), + format(100.0 * statistics.getSum() / total))); + } + + private static String format(final double value) { + return String.format(Locale.ROOT, TimingMessages.NUMBER, value); } } + private void writeTablets( + final TabletWriter consumer, + final Map> testResult, + final Map> test1Result, + final DoTestTimings timings) + throws Exception { + long stepStarted = System.nanoTime(); + boolean opened = false; + // Own one pool/session for the whole workload, including when writing or an assertion fails. + try (final ITableSessionPool pool = senderEnv.getTableSessionPool(1); + final ITableSession session = pool.getSession()) { + recordTabletTiming(timings, "TABLET_OPEN_SESSION", stepStarted); + opened = true; + try { + consumer.accept(testResult, test1Result, timings, session); + } finally { + stepStarted = System.nanoTime(); + } + } finally { + recordTabletTiming( + timings, opened ? "TABLET_CLOSE_SESSION" : "TABLET_OPEN_SESSION", stepStarted); + } + } + + private void insertTabletWithTiming( + final String database, + final Tablet tablet, + final DoTestTimings timings, + final ITableSession session, + final long generationStarted) + throws Exception { + long stepStarted = recordTabletTiming(timings, "TABLET_GENERATION", generationStarted); + String phase = "TABLET_USE_DATABASE"; + try { + session.executeNonQueryStatement("use " + this.database(database)); + stepStarted = recordTabletTiming(timings, phase, stepStarted); + phase = "TABLET_INSERT"; + session.insert(tablet); + stepStarted = recordTabletTiming(timings, phase, stepStarted); + phase = "TABLET_FLUSH"; + // Keep a flush after each retained Tablet so the workload still transfers multiple TsFiles. + session.executeNonQueryStatement("flush"); + } finally { + recordTabletTiming(timings, phase, stepStarted); + } + } + + private static long recordTabletTiming( + final DoTestTimings timings, final String phase, final long started) { + final long finished = System.nanoTime(); + if (timings != null) { + timings.recordDetail(phase, finished - started); + } + return finished; + } + + // Completion assertions wait for asynchronous transfer; per-Tablet sleeps add no coverage. + // Validate every retained Tablet while reusing the workload's Session for insert and flush. private void insertTablet1( - final Map> testResult, final Map> test1Result) { + final Map> testResult, + final Map> test1Result, + final DoTestTimings timings, + final ITableSession session) + throws Exception { int deviceIDStartIndex = 0; int deviceIDEndIndex = 10; - for (int j = 0; j < 25; j++) { + for (int j = 0; j < TABLETS_PER_ROUND.length; j++) { final String dataBaseName = "test" + j % 2; - for (int i = 0; i < 5; i++) { + for (int i = 0; i < TABLETS_PER_ROUND[j]; i++) { final String tableName = "test" + i; + final long generationStarted = System.nanoTime(); Tablet tablet = TableModelUtils.generateTablet( tableName, deviceIDStartIndex, deviceIDEndIndex, 0, 10, false, false); - TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + insertTabletWithTiming(dataBaseName, tablet, timings, session, generationStarted); Map> map = j % 2 == 0 ? testResult : test1Result; map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); } @@ -497,23 +778,23 @@ private void insertTablet1( } private void insertTablet2( - final Map> testResult, final Map> test1Result) { + final Map> testResult, + final Map> test1Result, + final DoTestTimings timings, + final ITableSession session) + throws Exception { int deviceIDStartIndex = 0; int deviceIDEndIndex = 10; - for (int j = 0; j < 25; j++) { + for (int j = 0; j < TABLETS_PER_ROUND.length; j++) { final String dataBaseName = "test" + j % 2; - for (int i = 0; i < 5; i++) { + for (int i = 0; i < TABLETS_PER_ROUND[j]; i++) { final String tableName = "test" + i; + final long generationStarted = System.nanoTime(); Tablet tablet = TableModelUtils.generateTablet( tableName, deviceIDStartIndex, deviceIDEndIndex, 10, false, false); - TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + insertTabletWithTiming(dataBaseName, tablet, timings, session, generationStarted); Map> map = j % 2 == 0 ? testResult : test1Result; map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); } @@ -523,17 +804,22 @@ private void insertTablet2( } private void insertTablet3( - final Map> testResult, final Map> test1Result) { + final Map> testResult, + final Map> test1Result, + final DoTestTimings timings, + final ITableSession session) + throws Exception { final Random random = new Random(); int deviceIDStartIndex = 0; int deviceIDEndIndex = 100; - for (int j = 0; j < 25; j++) { - for (int i = 0; i < 5; i++) { + for (int j = 0; j < TABLETS_PER_ROUND.length; j++) { + for (int i = 0; i < TABLETS_PER_ROUND[j]; i++) { final String tableName = "test" + i; final String dataBaseName = "test" + j % 2; deviceIDStartIndex = random.nextInt(1 << 16) - 10; deviceIDEndIndex = deviceIDStartIndex + 10; + final long generationStarted = System.nanoTime(); Tablet tablet = TableModelUtils.generateTablet( tableName, @@ -543,12 +829,7 @@ private void insertTablet3( deviceIDEndIndex, false, true); - TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + insertTabletWithTiming(dataBaseName, tablet, timings, session, generationStarted); Map> map = j % 2 == 0 ? testResult : test1Result; map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); } @@ -556,17 +837,22 @@ private void insertTablet3( } private void insertTablet4( - final Map> testResult, final Map> test1Result) { + final Map> testResult, + final Map> test1Result, + final DoTestTimings timings, + final ITableSession session) + throws Exception { final Random random = new Random(); int deviceIDStartIndex = 0; int deviceIDEndIndex = 100; - for (int j = 0; j < 25; j++) { + for (int j = 0; j < TABLETS_PER_ROUND.length; j++) { final String dataBaseName = "test" + j % 2; - for (int i = 0; i < 5; i++) { + for (int i = 0; i < TABLETS_PER_ROUND[j]; i++) { final String tableName = "test" + i; deviceIDStartIndex = random.nextInt(1 << 16) - 10; deviceIDEndIndex = deviceIDStartIndex + 10; + final long generationStarted = System.nanoTime(); Tablet tablet = TableModelUtils.generateTablet( tableName, @@ -576,12 +862,7 @@ private void insertTablet4( deviceIDEndIndex, false, false); - TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + insertTabletWithTiming(dataBaseName, tablet, timings, session, generationStarted); Map> map = j % 2 == 0 ? testResult : test1Result; map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); } @@ -589,25 +870,25 @@ private void insertTablet4( } private void insertTablet5( - final Map> testResult, final Map> test1Result) { + final Map> testResult, + final Map> test1Result, + final DoTestTimings timings, + final ITableSession session) + throws Exception { final Random random = new Random(); int deviceIDStartIndex = 0; int deviceIDEndIndex = 100; - for (int j = 0; j < 25; j++) { - for (int i = 0; i < 5; i++) { + for (int j = 0; j < TABLETS_PER_ROUND.length; j++) { + for (int i = 0; i < TABLETS_PER_ROUND[j]; i++) { final String tableName = "test" + i; final String dataBaseName = "test" + j % 2; deviceIDStartIndex = random.nextInt(1 << 16) - 10; deviceIDEndIndex = deviceIDStartIndex + 10; + final long generationStarted = System.nanoTime(); Tablet tablet = TableModelUtils.generateTablet( tableName, 0, 10, deviceIDStartIndex, deviceIDEndIndex, false, true); - TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + insertTabletWithTiming(dataBaseName, tablet, timings, session, generationStarted); Map> map = j % 2 == 0 ? testResult : test1Result; map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); } @@ -615,26 +896,26 @@ private void insertTablet5( } private void insertTablet6( - final Map> testResult, final Map> test1Result) { + final Map> testResult, + final Map> test1Result, + final DoTestTimings timings, + final ITableSession session) + throws Exception { final Random random = new Random(); int deviceIDStartIndex = 0; int deviceIDEndIndex = 100; - for (int j = 0; j < 25; j++) { + for (int j = 0; j < TABLETS_PER_ROUND.length; j++) { final String dataBaseName = "test" + j % 2; - for (int i = 0; i < 5; i++) { + for (int i = 0; i < TABLETS_PER_ROUND[j]; i++) { final String tableName = "test" + i; deviceIDStartIndex = random.nextInt(1 << 16) - 10; deviceIDEndIndex = deviceIDStartIndex + 10; + final long generationStarted = System.nanoTime(); Tablet tablet = TableModelUtils.generateTablet( tableName, deviceIDStartIndex, deviceIDEndIndex, 100, 110, false, true); - TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + insertTabletWithTiming(dataBaseName, tablet, timings, session, generationStarted); Map> map = j % 2 == 0 ? testResult : test1Result; map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); } @@ -642,26 +923,26 @@ private void insertTablet6( } private void insertTablet7( - final Map> testResult, final Map> test1Result) { + final Map> testResult, + final Map> test1Result, + final DoTestTimings timings, + final ITableSession session) + throws Exception { final Random random = new Random(); int deviceIDStartIndex = 0; int deviceIDEndIndex = 100; deviceIDStartIndex = random.nextInt(1 << 16) - 10; deviceIDEndIndex = deviceIDStartIndex + 10; - for (int j = 0; j < 25; j++) { + for (int j = 0; j < TABLETS_PER_ROUND.length; j++) { final String dataBaseName = "test" + j % 2; - for (int i = 0; i < 5; i++) { + for (int i = 0; i < TABLETS_PER_ROUND[j]; i++) { final String tableName = "test" + i; + final long generationStarted = System.nanoTime(); Tablet tablet = TableModelUtils.generateTablet( tableName, deviceIDStartIndex, deviceIDEndIndex, 100, 110, false, true); - TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + insertTabletWithTiming(dataBaseName, tablet, timings, session, generationStarted); Map> map = j % 2 == 0 ? testResult : test1Result; map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); } @@ -671,23 +952,23 @@ private void insertTablet7( } private void insertTablet8( - final Map> testResult, final Map> test1Result) { + final Map> testResult, + final Map> test1Result, + final DoTestTimings timings, + final ITableSession session) + throws Exception { final Random random = new Random(); int deviceIDStartIndex = random.nextInt(1 << 16); int deviceIDEndIndex = deviceIDStartIndex + 10; - for (int j = 0; j < 25; j++) { + for (int j = 0; j < TABLETS_PER_ROUND.length; j++) { final String dataBaseName = "test" + j % 2; - for (int i = 0; i < 5; i++) { + for (int i = 0; i < TABLETS_PER_ROUND[j]; i++) { final String tableName = "test" + i; + final long generationStarted = System.nanoTime(); Tablet tablet = TableModelUtils.generateTablet( tableName, 100, 110, deviceIDStartIndex, deviceIDEndIndex, false, true); - TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + insertTabletWithTiming(dataBaseName, tablet, timings, session, generationStarted); Map> map = j % 2 == 0 ? testResult : test1Result; map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); } @@ -697,26 +978,28 @@ private void insertTablet8( } private void insertTablet9( - final Map> testResult, final Map> test1Result) { + final Map> testResult, + final Map> test1Result, + final DoTestTimings timings, + final ITableSession session) + throws Exception { final Random random = new Random(); - for (int j = 0; j < 25; j++) { + for (int j = 0; j < TABLETS_PER_ROUND.length; j++) { final String dataBaseName = "test" + j % 2; - for (int i = 0; i < 5; i++) { + for (int i = 0; i < TABLETS_PER_ROUND[j]; i++) { final String tableName = "test" + i; + final long generationStarted = System.nanoTime(); Tablet tablet = TableModelUtils.generateTabletDeviceIDAllIsNull(tableName, 100, 110, 10, false); - TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + insertTabletWithTiming(dataBaseName, tablet, timings, session, generationStarted); Map> map = j % 2 == 0 ? testResult : test1Result; map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); } } } + // Disabling receiver TsFile validation must still preserve the tree points and all generated + // table values. @Test public void testLoadTsFileWithoutVerify() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -731,12 +1014,13 @@ public void testLoadTsFileWithoutVerify() throws Exception { // Because the failures will randomly generate due to resource limitation TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush")), null); for (int i = 0; i < 5; i++) { - TableModelUtils.createDataBaseAndTable(senderEnv, "test" + i, "test0"); - TableModelUtils.createDataBaseAndTable(senderEnv, "test" + i, "test1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test" + i, database("test0")); + TableModelUtils.createDataBaseAndTable(senderEnv, "test" + i, database("test1")); } final Map sourceAttributes = new HashMap<>(); @@ -772,18 +1056,19 @@ public void testLoadTsFileWithoutVerify() throws Exception { // Because the failures will randomly generate due to resource limitation TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush")), null); Map> testResult = new HashMap<>(); Map> test1Result = new HashMap<>(); - insertTablet1(testResult, test1Result); + writeTablets(this::insertTablet1, testResult, test1Result, null); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.vehicle.**", - "Time,root.vehicle.d0.s1,", + isolateDatabasePaths("select * from root.vehicle.**"), + isolateDatabasePaths("Time,root.vehicle.d0.s1,"), Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,")))); for (Map.Entry> entry : testResult.entrySet()) { @@ -791,8 +1076,9 @@ public void testLoadTsFileWithoutVerify() throws Exception { entry .getValue() .forEach(tablet -> set.addAll(TableModelUtils.generateExpectedResults(tablet))); - TableModelUtils.assertCountData("test0", entry.getKey(), set.size(), receiverEnv, s -> {}); - TableModelUtils.assertData("test0", entry.getKey(), set, receiverEnv, s -> {}); + TableModelUtils.assertCountData( + database("test0"), entry.getKey(), set.size(), receiverEnv, s -> {}); + TableModelUtils.assertData(database("test0"), entry.getKey(), set, receiverEnv, s -> {}); } for (Map.Entry> entry : test1Result.entrySet()) { @@ -800,9 +1086,31 @@ public void testLoadTsFileWithoutVerify() throws Exception { entry .getValue() .forEach(tablet -> set.addAll(TableModelUtils.generateExpectedResults(tablet))); - TableModelUtils.assertCountData("test1", entry.getKey(), set.size(), receiverEnv, s -> {}); - TableModelUtils.assertData("test1", entry.getKey(), set, receiverEnv, s -> {}); + TableModelUtils.assertCountData( + database("test1"), entry.getKey(), set.size(), receiverEnv, s -> {}); + TableModelUtils.assertData(database("test1"), entry.getKey(), set, receiverEnv, s -> {}); } } } + + // Shared-cluster cleanup is asynchronous, so suffix each workload's databases with its method. + private String database(final String name) { + return name + "_" + testName.getMethodName().toLowerCase(Locale.ENGLISH); + } + + private String databasePattern(final String pattern) { + return pattern + "_" + testName.getMethodName().toLowerCase(Locale.ENGLISH); + } + + private String isolateDatabasePaths(final String value) { + return value.replace("root.vehicle", "root.vehicle_" + testName.getMethodName()); + } + + private List isolateDatabasePaths(final List values) { + final List isolatedValues = new ArrayList<>(values.size()); + for (final String value : values) { + isolatedValues.add(isolateDatabasePaths(value)); + } + return isolatedValues; + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java index d0ce0bdad46d4..78c2910ef5eb5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java @@ -34,7 +34,7 @@ import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -55,7 +55,7 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeIsolationIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeIsolationIT extends AbstractPipeTableModelSharedClusterIT { @Override @Before diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java index be9f569e1729b..905856aa47f77 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java @@ -27,7 +27,7 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -49,7 +49,15 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeLifeCycleIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeLifeCycleIT extends AbstractPipeTableModelSharedClusterIT { + + @Override + protected void setupConfig() { + super.setupConfig(); + // Acknowledged writes must survive fast process stops in the restart scenarios. + senderEnv.getConfig().getCommonConfig().setWalMode("SYNC"); + receiverEnv.getConfig().getCommonConfig().setWalMode("SYNC"); + } @Override @Before @@ -345,6 +353,7 @@ public void testLifeCycleHybridMode() throws Exception { } } + // Transfer 200 rows, restart both clusters, then require all 300 rows after another write. @Test public void testLifeCycleWithClusterRestart() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -647,9 +656,11 @@ public void testDoubleLiving() throws Exception { TableModelUtils.assertData("test", "test", 0, 500, receiverEnv, handleFailure); } + // An unprivileged user may list pipes but cannot manage pipes/plugins; after granting SYSTEM, + // lifecycle operations succeed and plugin installation still validates its URI. @Test public void testPermission() { - createUser(senderEnv, "test", "test123123456"); + createUser(senderEnv, "test", "IoTDB!Test2024"); assertTableNonQueryTestFail( senderEnv, @@ -661,28 +672,28 @@ public void testPermission() { + ")", "803: Access Denied: No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456", + "IoTDB!Test2024", null); assertTableNonQueryTestFail( senderEnv, "drop pipe testPipe", "803: Access Denied: No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456", + "IoTDB!Test2024", null); assertTableNonQueryTestFail( senderEnv, "start pipe testPipe", "803: Access Denied: No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456", + "IoTDB!Test2024", null); assertTableNonQueryTestFail( senderEnv, "stop pipe testPipe", "803: Access Denied: No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456", + "IoTDB!Test2024", null); assertTableNonQueryTestFail( @@ -690,14 +701,14 @@ public void testPermission() { "create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'", "803: Access Denied: No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456", + "IoTDB!Test2024", null); assertTableNonQueryTestFail( senderEnv, "drop pipePlugin TestProcessor", "803: Access Denied: No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456", + "IoTDB!Test2024", null); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java index 0cda218010a64..d8ce0e74db052 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java @@ -26,17 +26,16 @@ import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -48,22 +47,27 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipePermissionIT extends AbstractPipeTableModelDualManualIT { - @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); +public class IoTDBPipePermissionIT extends AbstractPipeTableModelSharedClusterIT { + + private static final int PIPE_META_HEARTBEAT_INTERVAL_SECONDS = 3; + // Observe a configured metadata heartbeat plus a 2 s margin, and flush once midway. Configure + // both clusters explicitly so changes to server/template defaults cannot shorten this coverage. + private static final long UNAUTHORIZED_TRANSFER_OBSERVATION_SECONDS = + PIPE_META_HEARTBEAT_INTERVAL_SECONDS + 2L; + @Override + protected void setupConfig() { + super.setupConfig(); senderEnv .getConfig() .getCommonConfig() + .setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(PIPE_META_HEARTBEAT_INTERVAL_SECONDS) .setAutoCreateSchemaEnabled(false) .setDatanodeMemoryProportion("3:3:1:1:1:0") .setDefaultSchemaRegionGroupNumPerDatabase(1) @@ -77,6 +81,7 @@ public void setUp() { receiverEnv .getConfig() .getCommonConfig() + .setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(PIPE_META_HEARTBEAT_INTERVAL_SECONDS) .setAutoCreateSchemaEnabled(false) .setDatanodeMemoryProportion("3:3:1:1:1:0") .setTimestampPrecision("ms") @@ -91,13 +96,20 @@ public void setUp() { .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false) .setPipeAutoSplitFullEnabled(false); + } - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(3, 3); + @Override + protected CompletableFuture initializeClusters() { + return CompletableFuture.allOf( + CompletableFuture.runAsync(senderEnv::initClusterEnvironment), + CompletableFuture.runAsync(() -> receiverEnv.initClusterEnvironment(3, 3))); } + // A source user sees only its own pipe and transfers schema/rows only after the required grants. @Test public void testSourcePermission() { + final String databaseName = "permission_source"; + final String grantedDatabaseName = "permission_source_grant"; TestUtils.executeNonQuery(senderEnv, "create user `thulab` 'passwD@123456'", null); // Shall fail if username is specified without password @@ -174,7 +186,7 @@ public void testSourcePermission() { fail("Alter pipe shall not fail if user and password are specified"); } - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", databaseName); // Shall not be transferred TestUtils.assertDataAlwaysOnEnv( @@ -182,18 +194,19 @@ public void testSourcePermission() { "count databases", "count,", Collections.singleton("1,"), + UNAUTHORIZED_TRANSFER_OBSERVATION_SECONDS, "information_schema"); // Grant some privilege TestUtils.executeNonQuery( - "test", BaseEnv.TABLE_SQL_DIALECT, senderEnv, "grant INSERT on any to user thulab"); + databaseName, BaseEnv.TABLE_SQL_DIALECT, senderEnv, "grant INSERT on any to user thulab"); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", grantedDatabaseName); // Shall be transferred TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "show tables from test1", + "show tables from " + grantedDatabaseName, "TableName,TTL(ms),", Collections.singleton("test1,INF,"), "information_schema"); @@ -208,20 +221,26 @@ public void testSourcePermission() { } // Write some data - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + TableModelUtils.insertData(databaseName, "test", 0, 100, senderEnv); try { - TableModelUtils.createDataBaseAndTable(receiverEnv, "test", "test"); + TableModelUtils.createDataBaseAndTable(receiverEnv, "test", databaseName); } catch (final Exception | Error ignore) { // Ignore because the db/table may be transferred because sender user may see these } // Exception, block here - TableModelUtils.assertCountDataAlwaysOnEnv("test", "test", 0, receiverEnv); + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + TableModelUtils.getQueryCountSql("test"), + "_col0,", + Collections.singleton("0,"), + UNAUTHORIZED_TRANSFER_OBSERVATION_SECONDS, + databaseName); // Grant SELECT privilege TestUtils.executeNonQueries( - "test", + databaseName, BaseEnv.TABLE_SQL_DIALECT, senderEnv, Arrays.asList("grant SELECT on any to user thulab", "start pipe a2b"), @@ -229,7 +248,7 @@ public void testSourcePermission() { // Will finally pass TableModelUtils.assertCountData( - "test", + databaseName, "test", 100, receiverEnv, @@ -271,6 +290,7 @@ public void testSourcePermission() { } } + // Receiver grants unblock queued rows; skipif excludes a second database until it is authorized. @Test public void testReceiverPermission() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -286,14 +306,14 @@ public void testReceiverPermission() throws Exception { final Map processorAttributes = new HashMap<>(); final Map sinkAttributes = new HashMap<>(); - final String dbName = "test"; + final String dbName = "permission_receiver"; final String tbName = "test"; sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.capture.tree", "false"); sourceAttributes.put("source.capture.table", "true"); sourceAttributes.put("__system.sql-dialect", "table"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.ip", receiverIp); @@ -330,7 +350,7 @@ public void testReceiverPermission() throws Exception { "information_schema", BaseEnv.TABLE_SQL_DIALECT, receiverEnv, - "grant insert,create on database test to user testUser", + "grant insert,create on database " + dbName + " to user testUser", null); // Will finally pass @@ -353,7 +373,7 @@ public void testReceiverPermission() throws Exception { fail(e.getMessage()); } - final String dbName2 = "test2"; + final String dbName2 = "permission_receiver_skip"; TableModelUtils.createDataBaseAndTable(senderEnv, tbName, dbName2); // Write some data @@ -384,7 +404,7 @@ public void testReceiverPermission() throws Exception { "information_schema", BaseEnv.TABLE_SQL_DIALECT, receiverEnv, - "grant insert,create on database test2 to user testUser", + "grant insert,create on database " + dbName2 + " to user testUser", null); TableModelUtils.createDataBaseAndTable(receiverEnv, tbName, dbName2); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java index 809278f998a60..2ed1a7de65416 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java @@ -29,6 +29,7 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; @@ -54,6 +55,12 @@ public void setUp() { MultiEnvFactory.createEnv(2); senderEnv = MultiEnvFactory.getEnv(0); receiverEnv = MultiEnvFactory.getEnv(1); + // These tests cover consensus/topology combinations. Many CPU-scaled tiny + // regions exhaust the IT direct memory, and concurrent loads need only a small schema budget. + for (final BaseEnv env : new BaseEnv[] {senderEnv, receiverEnv}) { + env.getConfig().getCommonConfig().setDataRegionPerDataNode(2); + env.getConfig().getDataNodeConfig().setLoadTsFileAnalyzeSchemaMemorySizeInBytes(256 * 1024L); + } } private void innerSetUp( @@ -94,10 +101,11 @@ private void innerSetUp( .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); - senderEnv.initClusterEnvironment(configNodesNum, dataNodesNum); - receiverEnv.initClusterEnvironment(configNodesNum, dataNodesNum); + executeOnBothEnvs(env -> env.initClusterEnvironment(configNodesNum, dataNodesNum)); } + // Replicate historical and realtime rows between two 1C1D clusters using Ratis/Ratis/IoT; + // stopping the pipe must leave the receiver at 20 rows after the next 10 sender inserts. @Test public void test1C1DWithRatisRatisIot() throws Exception { innerSetUp( @@ -111,6 +119,8 @@ public void test1C1DWithRatisRatisIot() throws Exception { doTest(); } + // Use Simple consensus for the 1C1D receiver's ConfigNode and SchemaRegion and IoT for data; + // history/realtime replication and the stopped-pipe boundary must match the Ratis sender. @Test public void test1C1DWithSimpleSimpleIot() throws Exception { innerSetUp( @@ -124,6 +134,8 @@ public void test1C1DWithSimpleSimpleIot() throws Exception { doTest(); } + // Use Simple data consensus on both 1C1D clusters while retaining Ratis for schema/config; + // replicate 20 rows and keep the next 10 rows on the sender after stopping the pipe. @Test public void test1C1DWithRatisRatisSimple() throws Exception { innerSetUp( @@ -137,6 +149,8 @@ public void test1C1DWithRatisRatisSimple() throws Exception { doTest(); } + // Replicate historical and realtime rows between 3C3D clusters with three schema/data replicas; + // stopping the pipe must prevent the final 10 rows from reaching the receiver. @Test public void test3C3DWith3SchemaRegionFactor3DataRegionFactor() throws Exception { innerSetUp( @@ -150,6 +164,8 @@ public void test3C3DWith3SchemaRegionFactor3DataRegionFactor() throws Exception doTest(); } + // Exercise 3C3D clusters with three schema replicas and two data replicas; history/realtime + // rows must replicate, and the receiver must remain at 20 rows after the pipe is stopped. @Test public void test3C3DWith3SchemaRegionFactor2DataRegionFactor() throws Exception { innerSetUp( @@ -163,6 +179,8 @@ public void test3C3DWith3SchemaRegionFactor2DataRegionFactor() throws Exception doTest(); } + // Send 10 rows from a 3C3D cluster to a 1C1D cluster, then reverse direction with another + // database; each destination must receive all rows through its own pipe. @Test public void testPipeOnBothSenderAndReceiver() throws Exception { senderEnv @@ -192,8 +210,11 @@ public void testPipeOnBothSenderAndReceiver() throws Exception { .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); - senderEnv.initClusterEnvironment(3, 3); - receiverEnv.initClusterEnvironment(1, 1); + executeOnBothEnvs( + env -> { + final int nodeCount = env == senderEnv ? 3 : 1; + env.initClusterEnvironment(nodeCount, nodeCount); + }); final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); @@ -208,7 +229,7 @@ public void testPipeOnBothSenderAndReceiver() throws Exception { (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + TableModelUtils.insertData("test", "test", 0, 10, senderEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -239,7 +260,7 @@ public void testPipeOnBothSenderAndReceiver() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - TableModelUtils.assertCountData("test", "test", 100, receiverEnv, handleFailure); + TableModelUtils.assertCountData("test", "test", 10, receiverEnv, handleFailure); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); @@ -252,7 +273,7 @@ public void testPipeOnBothSenderAndReceiver() throws Exception { (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { TableModelUtils.createDataBaseAndTable(receiverEnv, "test1", "test1"); - TableModelUtils.insertData("test1", "test1", 0, 100, receiverEnv); + TableModelUtils.insertData("test1", "test1", 0, 10, receiverEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -283,7 +304,7 @@ public void testPipeOnBothSenderAndReceiver() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - TableModelUtils.assertCountData("test1", "test1", 100, senderEnv, handleFailure); + TableModelUtils.assertCountData("test1", "test1", 10, senderEnv, handleFailure); } } @@ -302,7 +323,8 @@ private void doTest() throws Exception { (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + // Ten distinct devices cover protocol behavior without a bulk metadata/ingestion workload. + TableModelUtils.insertData("test", "test", 0, 10, senderEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -333,37 +355,43 @@ private void doTest() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - TableModelUtils.assertData("test", "test", 0, 100, receiverEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 10, receiverEnv, handleFailure); - TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + TableModelUtils.insertData("test", "test", 10, 20, senderEnv); - TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 20, receiverEnv, handleFailure); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - TableModelUtils.insertData("test", "test", 200, 300, senderEnv); + TableModelUtils.insertData("test", "test", 20, 30, senderEnv); // A stopped pipe must keep the receiver data unchanged. TestUtils.assertDataAlwaysOnEnv( receiverEnv, TableModelUtils.getQuerySql("test"), TableModelUtils.generateHeaderResults(), - TableModelUtils.generateExpectedResults(0, 200), + TableModelUtils.generateExpectedResults(0, 20), "test"); } } + // A synchronous sink using all three receiver node URLs must replicate rows in forced-log + // and file modes, including historical rows and later inserts (40 rows in total). @Test public void testSyncConnectorUseNodeUrls() throws Exception { doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_SYNC_CONNECTOR.getPipePluginName()); } + // An asynchronous sink using all three receiver node URLs must replicate historical and + // realtime rows in forced-log and file modes, ending with all 40 rows on the receiver. @Test public void testAsyncConnectorUseNodeUrls() throws Exception { doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName()); } + // Air-gap receiver ports supplied through node URLs must transfer historical and realtime + // rows in forced-log and file modes, ending with all 40 rows on the receiver. @Test public void testAirGapConnectorUseNodeUrls() throws Exception { doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()); @@ -402,8 +430,7 @@ private void doTestUseNodeUrls(String connectorName) throws Exception { .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); - senderEnv.initClusterEnvironment(1, 1); - receiverEnv.initClusterEnvironment(1, 3); + executeOnBothEnvs(env -> env.initClusterEnvironment(1, env == senderEnv ? 1 : 3)); final StringBuilder nodeUrlsBuilder = new StringBuilder(); final Consumer handleFailure = @@ -429,7 +456,8 @@ private void doTestUseNodeUrls(String connectorName) throws Exception { (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + // Keep separate history/realtime/file-mode ranges while limiting each to ten devices. + TableModelUtils.insertData("test", "test", 0, 10, senderEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -463,13 +491,13 @@ private void doTestUseNodeUrls(String connectorName) throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + TableModelUtils.insertData("test", "test", 10, 20, senderEnv); - TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 20, receiverEnv, handleFailure); - TableModelUtils.insertData("test", "test", 200, 300, senderEnv); + TableModelUtils.insertData("test", "test", 20, 30, senderEnv); - TableModelUtils.assertData("test", "test", 0, 300, receiverEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 30, receiverEnv, handleFailure); extractorAttributes.replace("realtime.mode", "file"); @@ -483,9 +511,9 @@ private void doTestUseNodeUrls(String connectorName) throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - TableModelUtils.insertData("test", "test", 300, 400, senderEnv); + TableModelUtils.insertData("test", "test", 30, 40, senderEnv); - TableModelUtils.assertData("test", "test", 0, 400, receiverEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 40, receiverEnv, handleFailure); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeRelationalSecurityIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeRelationalSecurityIT.java index 27985c75ef2af..af002fb70b7a3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeRelationalSecurityIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeRelationalSecurityIT.java @@ -54,27 +54,39 @@ public void setUp() { senderEnv = MultiEnvFactory.getEnv(0); receiverEnv = MultiEnvFactory.getEnv(1); setupConfig(); - senderEnv.initClusterEnvironment(1, 1); - receiverEnv.initClusterEnvironment(1, 1); + executeOnBothEnvs(env -> env.initClusterEnvironment(1, 1)); } @Override protected void setupConfig() { super.setupConfig(); + // This small auth snapshot needs few Regions and small schema-analysis buffers; the defaults + // can otherwise exhaust query memory while loading several tiny files concurrently. senderEnv .getConfig() .getCommonConfig() .setAutoCreateSchemaEnabled(false) + .setDataRegionPerDataNode(2) .setDataReplicationFactor(1) .setSchemaReplicationFactor(1); receiverEnv .getConfig() .getCommonConfig() .setAutoCreateSchemaEnabled(false) + .setDataRegionPerDataNode(2) .setDataReplicationFactor(1) .setSchemaReplicationFactor(1); + senderEnv + .getConfig() + .getDataNodeConfig() + .setLoadTsFileAnalyzeSchemaMemorySizeInBytes(256 * 1024L); + receiverEnv + .getConfig() + .getDataNodeConfig() + .setLoadTsFileAnalyzeSchemaMemorySizeInBytes(256 * 1024L); } + // A receiver with SECURITY, CREATE and INSERT must load all six snapshot privileges and ten rows. @Test public void testReceiverSecurityPrivilegeForRelationalAuthSnapshot() throws Exception { final String receiverUser = "pipe_receiver"; @@ -104,7 +116,8 @@ public void testReceiverSecurityPrivilegeForRelationalAuthSnapshot() throws Exce replicatedUser)), null); TableModelUtils.createDataBaseAndTable(senderEnv, tableName, databaseName); - TableModelUtils.insertData(databaseName, tableName, 0, 100, senderEnv); + // A small batch still exercises data transfer alongside the relational authorization snapshot. + TableModelUtils.insertData(databaseName, tableName, 0, 10, senderEnv); final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final Map sourceAttributes = new HashMap<>(); @@ -152,7 +165,7 @@ public void testReceiverSecurityPrivilegeForRelationalAuthSnapshot() throws Exce TableModelUtils.assertCountData( databaseName, tableName, - 100, + 10, receiverEnv, o -> { TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java index 559b17722b2b3..825668aac3e78 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java @@ -25,20 +25,21 @@ import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.PipeTestUtils; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; -import org.junit.Before; import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.sql.Connection; @@ -47,6 +48,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.function.Consumer; @@ -54,14 +57,19 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeSourceIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeSourceIT extends AbstractPipeTableModelSharedClusterIT { - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); + @Rule public final TestName testName = new TestName(); + @Override + protected void setupConfig() { + // Retain the shared fixture's single-Region database budget. Extra matching Regions can + // otherwise mask excluded-only owners when completion is checked across multiple DataNodes. + super.setupConfig(); + senderEnv + .getConfig() + .getDataNodeConfig() + .setMetricReporterType(Collections.singletonList("JMX")); senderEnv .getConfig() .getCommonConfig() @@ -89,11 +97,11 @@ public void setUp() { .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false) .setPipeAutoSplitFullEnabled(false); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); } + // Database-name filters and SQL model isolation must capture matching table data and reject + // operations from the wrong dialect. Completion checks must ignore Region owners whose + // databases are excluded by both model filters. @Test public void testMatchingMultipleDatabases() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -116,9 +124,9 @@ public void testMatchingMultipleDatabases() throws Exception { extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.capture.tree", "true"); extractorAttributes.put("mode.double-living", "true"); - extractorAttributes.put("extractor.database-name", "test"); + extractorAttributes.put("extractor.database-name", database("test")); extractorAttributes.put("extractor.table-name", "test"); - extractorAttributes.put("extractor.pattern", "root.db1"); + extractorAttributes.put("extractor.pattern", isolateDatabasePaths("root.db1")); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("user", "root"); @@ -140,18 +148,18 @@ public void testMatchingMultipleDatabases() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db1.d1 (time, at1) values (1, 10)", - "insert into root.db2.d1 (time, at1) values (1, 20)", + isolateDatabasePaths("insert into root.db1.d1 (time, at1) values (1, 10)"), + isolateDatabasePaths("insert into root.db2.d1 (time, at1) values (1, 20)"), "flush"), null); - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", database("test")); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test1")); + TableModelUtils.insertData(database("test"), "test", 0, 100, senderEnv); - TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + TableModelUtils.insertData(database("test1"), "test1", 0, 100, senderEnv); - extractorAttributes.replace("extractor.pattern", "root.db2"); + extractorAttributes.replace("extractor.pattern", isolateDatabasePaths("root.db2")); extractorAttributes.replace("extractor.table-name", "test1"); status = client.createPipe( @@ -161,9 +169,18 @@ public void testMatchingMultipleDatabases() throws Exception { Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); + // Wait for each filtered pipe to commit its source-side flush barrier before checking the + // matching tree rows. The schema count alone can become visible before the rows arrive. + PipeTestUtils.awaitDataPipeCompletion(senderEnv, "p1"); + PipeTestUtils.awaitDataPipeCompletion(senderEnv, "p2"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + isolateDatabasePaths("select count(*) from root.db*.**"), + Map.of( + isolateDatabasePaths("count(root.db1.d1.at1)"), "1", + isolateDatabasePaths("count(root.db2.d1.at1)"), "1")); assertTimeseriesCountOnReceiver(receiverEnv, 2); - Thread.sleep(10000); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipeExtended(new TDropPipeReq("p1").setIsTableModel(false)).getCode()); @@ -180,14 +197,14 @@ public void testMatchingMultipleDatabases() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db1.d1 (time, at1) values (2, 11)", - "insert into root.db2.d1 (time, at1) values (2, 21)", + isolateDatabasePaths("insert into root.db1.d1 (time, at1) values (2, 11)"), + isolateDatabasePaths("insert into root.db2.d1 (time, at1) values (2, 21)"), "flush"), null); - TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + TableModelUtils.insertData(database("test"), "test", 100, 200, senderEnv); - TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + TableModelUtils.insertData(database("test1"), "test1", 100, 200, senderEnv); extractorAttributes.remove("extractor.pattern"); // no pattern, will match all databases extractorAttributes.remove("extractor.table-name"); @@ -201,17 +218,20 @@ public void testMatchingMultipleDatabases() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p3").getCode()); + // Renaming databases can reorder wildcard columns; compare each labelled count instead. TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db*.**", - "count(root.db1.d1.at1),count(root.db2.d1.at1),", - Collections.singleton("2,2,"), - handleFailure); + isolateDatabasePaths("select count(*) from root.db*.**"), + Map.of( + isolateDatabasePaths("count(root.db1.d1.at1)"), "2", + isolateDatabasePaths("count(root.db2.d1.at1)"), "2")); - TableModelUtils.assertCountData("test", "test", 200, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test", 200, receiverEnv, handleFailure); } } + // Source filters must replicate matching history and realtime events in both the tree and table + // models. @Ignore @Test public void testHistoryAndRealtime() throws Exception { @@ -230,24 +250,24 @@ public void testHistoryAndRealtime() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db.d1 (time, at1) values (1, 10)", - "insert into root.db.d2 (time, at1) values (1, 20)", - "insert into root.db.d3 (time, at1) values (1, 30)", - "insert into root.db.d4 (time, at1) values (1, 40)", + isolateDatabasePaths("insert into root.db.d1 (time, at1) values (1, 10)"), + isolateDatabasePaths("insert into root.db.d2 (time, at1) values (1, 20)"), + isolateDatabasePaths("insert into root.db.d3 (time, at1) values (1, 30)"), + isolateDatabasePaths("insert into root.db.d4 (time, at1) values (1, 40)"), "flush"), null); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); - TableModelUtils.insertData("test", "test1", 0, 100, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test")); + TableModelUtils.insertData(database("test"), "test1", 0, 100, senderEnv); - TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); - TableModelUtils.insertData("test", "test2", 0, 100, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", database("test")); + TableModelUtils.insertData(database("test"), "test2", 0, 100, senderEnv); - TableModelUtils.createDataBaseAndTable(senderEnv, "test3", "test"); - TableModelUtils.insertData("test", "test3", 0, 100, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test3", database("test")); + TableModelUtils.insertData(database("test"), "test3", 0, 100, senderEnv); - TableModelUtils.createDataBaseAndTable(senderEnv, "test4", "test"); - TableModelUtils.insertData("test", "test4", 0, 100, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test4", database("test")); + TableModelUtils.insertData(database("test"), "test4", 0, 100, senderEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -262,10 +282,10 @@ public void testHistoryAndRealtime() throws Exception { extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.capture.tree", "true"); extractorAttributes.put("mode.double-living", "true"); - extractorAttributes.put("extractor.database-name", "test"); + extractorAttributes.put("extractor.database-name", database("test")); extractorAttributes.put("extractor.table-name", "test2"); extractorAttributes.put("extractor.inclusion", "data.insert"); - extractorAttributes.put("extractor.pattern", "root.db.d2"); + extractorAttributes.put("extractor.pattern", isolateDatabasePaths("root.db.d2")); extractorAttributes.put("extractor.history.enable", "false"); extractorAttributes.put("extractor.realtime.enable", "true"); extractorAttributes.put("user", "root"); @@ -279,7 +299,7 @@ public void testHistoryAndRealtime() throws Exception { TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); extractorAttributes.replace("extractor.table-name", "test3"); - extractorAttributes.replace("extractor.pattern", "root.db.d3"); + extractorAttributes.replace("extractor.pattern", isolateDatabasePaths("root.db.d3")); extractorAttributes.replace("extractor.history.enable", "true"); extractorAttributes.replace("extractor.realtime.enable", "false"); status = @@ -292,7 +312,7 @@ public void testHistoryAndRealtime() throws Exception { TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p3").getCode()); extractorAttributes.replace("extractor.table-name", "test4"); - extractorAttributes.replace("extractor.pattern", "root.db.d4"); + extractorAttributes.replace("extractor.pattern", isolateDatabasePaths("root.db.d4")); extractorAttributes.replace("extractor.history.enable", "true"); extractorAttributes.replace("extractor.realtime.enable", "true"); status = @@ -307,39 +327,43 @@ public void testHistoryAndRealtime() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db.d1 (time, at1) values (2, 11)", - "insert into root.db.d2 (time, at1) values (2, 21)", - "insert into root.db.d3 (time, at1) values (2, 31)", - "insert into root.db.d4 (time, at1) values (2, 41), (3, 51)"), + isolateDatabasePaths("insert into root.db.d1 (time, at1) values (2, 11)"), + isolateDatabasePaths("insert into root.db.d2 (time, at1) values (2, 21)"), + isolateDatabasePaths("insert into root.db.d3 (time, at1) values (2, 31)"), + isolateDatabasePaths("insert into root.db.d4 (time, at1) values (2, 41), (3, 51)")), null); - TableModelUtils.insertData("test", "test2", 100, 200, senderEnv); + TableModelUtils.insertData(database("test"), "test2", 100, 200, senderEnv); - TableModelUtils.insertData("test", "test3", 100, 200, senderEnv); + TableModelUtils.insertData(database("test"), "test3", 100, 200, senderEnv); - TableModelUtils.insertData("test", "test4", 100, 200, senderEnv); + TableModelUtils.insertData(database("test"), "test4", 100, 200, senderEnv); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.** where time <= 1", - "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),", + isolateDatabasePaths("select count(*) from root.db.** where time <= 1"), + isolateDatabasePaths( + "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),"), Collections.singleton("1,0,1,"), handleFailure); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.** where time >= 2", - "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),", + isolateDatabasePaths("select count(*) from root.db.** where time >= 2"), + isolateDatabasePaths( + "count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),"), Collections.singleton("2,1,0,"), handleFailure); - TableModelUtils.assertData("test", "test2", 100, 200, receiverEnv, handleFailure); + TableModelUtils.assertData(database("test"), "test2", 100, 200, receiverEnv, handleFailure); - TableModelUtils.assertData("test", "test3", 0, 100, receiverEnv, handleFailure); + TableModelUtils.assertData(database("test"), "test3", 0, 100, receiverEnv, handleFailure); - TableModelUtils.assertData("test", "test4", 0, 200, receiverEnv, handleFailure); + TableModelUtils.assertData(database("test"), "test4", 0, 200, receiverEnv, handleFailure); } } + // History time bounds, with and without path/table filters, must filter old events while + // allowing subsequent realtime rows. @Ignore @Test public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Exception { @@ -358,18 +382,18 @@ public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws E TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db.d1 (time, at1)" + isolateDatabasePaths("insert into root.db.d1 (time, at1)") + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", - "insert into root.db.d2 (time, at1)" + isolateDatabasePaths("insert into root.db.d2 (time, at1)") + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", "flush"), null); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); - TableModelUtils.insertData("test", "test1", 0, 10, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test")); + TableModelUtils.insertData(database("test"), "test1", 0, 10, senderEnv); - TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); - TableModelUtils.insertData("test", "test2", 0, 10, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", database("test")); + TableModelUtils.insertData(database("test"), "test2", 0, 10, senderEnv); // wait for flush to complete TestUtils.executeNonQueries(senderEnv, Collections.singletonList("flush"), null); @@ -383,9 +407,9 @@ public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws E extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.capture.tree", "true"); extractorAttributes.put("mode.double-living", "true"); - extractorAttributes.put("extractor.database-name", "test"); + extractorAttributes.put("extractor.database-name", database("test")); extractorAttributes.put("extractor.table-name", "test1"); - extractorAttributes.put("extractor.pattern", "root.db.d1"); + extractorAttributes.put("extractor.pattern", isolateDatabasePaths("root.db.d1")); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.history.enable", "true"); // 1970-01-01T08:00:02+08:00 @@ -409,8 +433,8 @@ public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws E TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),", + isolateDatabasePaths("select count(*) from root.db.**"), + isolateDatabasePaths("count(root.db.d1.at1),"), Collections.singleton("2,"), handleFailure); @@ -427,16 +451,18 @@ public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws E TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),count(root.db.d2.at1),", + isolateDatabasePaths("select count(*) from root.db.**"), + isolateDatabasePaths("count(root.db.d1.at1),count(root.db.d2.at1),"), Collections.singleton("2,2,"), handleFailure); - TableModelUtils.assertData("test", "test1", 2, 4, receiverEnv, handleFailure); - TableModelUtils.assertData("test", "test2", 2, 4, receiverEnv, handleFailure); + TableModelUtils.assertData(database("test"), "test1", 2, 4, receiverEnv, handleFailure); + TableModelUtils.assertData(database("test"), "test2", 2, 4, receiverEnv, handleFailure); } } + // History and realtime events in [2, 4] must replicate in both models; out-of-range events must + // remain excluded after all tree and table DataRegions have committed their completion barriers. @Test public void testExtractorTimeRangeMatch() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -455,18 +481,18 @@ public void testExtractorTimeRangeMatch() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db.d1 (time, at1)" + isolateDatabasePaths("insert into root.db.d1 (time, at1)") + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", - "insert into root.db.d2 (time, at1)" + isolateDatabasePaths("insert into root.db.d2 (time, at1)") + " values (6, 6), (7, 7), (8, 8), (9, 9), (10, 10)", "flush"), null); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); - TableModelUtils.insertData("test", "test1", 0, 10, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test")); + TableModelUtils.insertData(database("test"), "test1", 0, 10, senderEnv); - TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); - TableModelUtils.insertData("test", "test2", 0, 10, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", database("test")); + TableModelUtils.insertData(database("test"), "test2", 0, 10, senderEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -495,59 +521,73 @@ public void testExtractorTimeRangeMatch() throws Exception { TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),", + isolateDatabasePaths("select count(*) from root.db.**"), + isolateDatabasePaths("count(root.db.d1.at1),"), Collections.singleton("3,"), handleFailure); - TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test1", 3, receiverEnv, handleFailure); // Insert realtime data that overlapped with time range TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db.d3 (time, at1)" + isolateDatabasePaths("insert into root.db.d3 (time, at1)") + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", "flush"), null); - TableModelUtils.createDataBaseAndTable(senderEnv, "test3", "test"); - TableModelUtils.insertData("test", "test3", 0, 5, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test3", database("test")); + TableModelUtils.insertData(database("test"), "test3", 0, 5, senderEnv); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),count(root.db.d3.at1),", + isolateDatabasePaths("select count(*) from root.db.**"), + isolateDatabasePaths("count(root.db.d1.at1),count(root.db.d3.at1),"), Collections.singleton("3,3,"), handleFailure); - TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); - TableModelUtils.assertCountData("test", "test3", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test1", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test3", 3, receiverEnv, handleFailure); // Insert realtime data that does not overlap with time range TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db.d4 (time, at1)" + isolateDatabasePaths("insert into root.db.d4 (time, at1)") + " values (6, 6), (7, 7), (8, 8), (9, 9), (10, 10)", "flush"), null); - TableModelUtils.createDataBaseAndTable(senderEnv, "test4", "test"); - TableModelUtils.insertData("test", "test4", 6, 10, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test4", database("test")); + TableModelUtils.insertData(database("test"), "test4", 6, 10, senderEnv); - TestUtils.assertDataAlwaysOnEnv( + TestUtils.executeNonQuery( + senderEnv, + isolateDatabasePaths("insert into root.db.d3 (time, at1) values (3, 30)"), + null); + PipeTestUtils.awaitDataPipeCompletion(senderEnv, "p1"); + TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),count(root.db.d3.at1),", - Collections.singleton("3,3,"), - 600, + isolateDatabasePaths("select at1 from root.db.d3 where time = 3"), + isolateDatabasePaths("Time,root.db.d3.at1,"), + Collections.singleton("3,30.0,"), + handleFailure); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + isolateDatabasePaths("select count(at1) from root.db.** align by device"), + "Device,count(at1),", + new HashSet<>( + Arrays.asList( + isolateDatabasePaths("root.db.d1,3,"), isolateDatabasePaths("root.db.d3,3,"))), handleFailure); - TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); - TableModelUtils.assertCountData("test", "test3", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test1", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test3", 3, receiverEnv, handleFailure); } } + // Source time bounds, with and without path/table filters, must restrict both history and + // realtime events to the configured interval. @Test public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -566,18 +606,18 @@ public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Ex TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db.d1 (time, at1)" + isolateDatabasePaths("insert into root.db.d1 (time, at1)") + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", - "insert into root.db.d2 (time, at1)" + isolateDatabasePaths("insert into root.db.d2 (time, at1)") + " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)", "flush"), null); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); - TableModelUtils.insertData("test", "test1", 0, 5, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test")); + TableModelUtils.insertData(database("test"), "test1", 0, 5, senderEnv); - TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); - TableModelUtils.insertData("test", "test2", 0, 5, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", database("test")); + TableModelUtils.insertData(database("test"), "test2", 0, 5, senderEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -587,7 +627,7 @@ public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Ex extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("source.capture.tree", "true"); extractorAttributes.put("mode.double-living", "true"); - extractorAttributes.put("source.pattern", "root.db.d1"); + extractorAttributes.put("source.pattern", isolateDatabasePaths("root.db.d1")); extractorAttributes.put("source.table-name", "test1"); extractorAttributes.put("source.inclusion", "data.insert"); extractorAttributes.put("source.start-time", "2"); @@ -611,12 +651,12 @@ public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Ex TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),", + isolateDatabasePaths("select count(*) from root.db.**"), + isolateDatabasePaths("count(root.db.d1.at1),"), Collections.singleton("3,"), handleFailure); - TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test1", 3, receiverEnv, handleFailure); extractorAttributes.remove("source.pattern"); extractorAttributes.remove("source.table-name"); @@ -631,16 +671,18 @@ public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Ex TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.at1),count(root.db.d2.at1),", + isolateDatabasePaths("select count(*) from root.db.**"), + isolateDatabasePaths("count(root.db.d1.at1),count(root.db.d2.at1),"), Collections.singleton("3,3,"), handleFailure); - TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); - TableModelUtils.assertCountData("test", "test2", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test1", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test2", 3, receiverEnv, handleFailure); } } + // Loose history filtering with and without path/table patterns must preserve the expected + // matching data. @Ignore @Test public void testHistoryLooseRange() throws Exception { @@ -654,11 +696,11 @@ public void testHistoryLooseRange() throws Exception { TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); }; - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); - TableModelUtils.insertData("test", "test1", 0, 2, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test")); + TableModelUtils.insertData(database("test"), "test1", 0, 2, senderEnv); - TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); - TableModelUtils.insertData("test", "test2", 0, 2, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", database("test")); + TableModelUtils.insertData(database("test"), "test2", 0, 2, senderEnv); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { @@ -666,9 +708,11 @@ public void testHistoryLooseRange() throws Exception { senderEnv, Arrays.asList( // TsFile 1, extracted without parse - "insert into root.db.d1 (time, at1, at2)" + " values (1, 1, 2), (2, 3, 4)", + isolateDatabasePaths("insert into root.db.d1 (time, at1, at2)") + + " values (1, 1, 2), (2, 3, 4)", // TsFile 2, not extracted because pattern not overlapped - "insert into root.db1.d1 (time, at1, at2)" + " values (1, 1, 2), (2, 3, 4)", + isolateDatabasePaths("insert into root.db1.d1 (time, at1, at2)") + + " values (1, 1, 2), (2, 3, 4)", "flush"), null); @@ -676,10 +720,12 @@ public void testHistoryLooseRange() throws Exception { senderEnv, Arrays.asList( // TsFile 3, not extracted because time range not overlapped - "insert into root.db.d1 (time, at1, at2)" + " values (3, 1, 2), (4, 3, 4)", "flush"), + isolateDatabasePaths("insert into root.db.d1 (time, at1, at2)") + + " values (3, 1, 2), (4, 3, 4)", + "flush"), null); - TableModelUtils.insertData("test", "test1", 2, 5, senderEnv); + TableModelUtils.insertData(database("test"), "test1", 2, 5, senderEnv); // wait for flush to complete TestUtils.executeNonQueries(senderEnv, Collections.singletonList("flush"), null); @@ -694,7 +740,7 @@ public void testHistoryLooseRange() throws Exception { extractorAttributes.put("source.capture.tree", "true"); extractorAttributes.put("mode.double-living", "true"); extractorAttributes.put("source.table-name", "test1"); - extractorAttributes.put("source.path", "root.db.d1.at1"); + extractorAttributes.put("source.path", isolateDatabasePaths("root.db.d1.at1")); extractorAttributes.put("source.inclusion", "data.insert"); extractorAttributes.put("source.history.start-time", "1"); extractorAttributes.put("source.history.end-time", "2"); @@ -717,15 +763,17 @@ public void testHistoryLooseRange() throws Exception { TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.** group by level=0", + isolateDatabasePaths("select count(*) from root.db.** group by level=0"), "count(root.*.*.*),", Collections.singleton("4,"), handleFailure); - TableModelUtils.assertCountData("test", "test1", 3, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test1", 3, receiverEnv, handleFailure); } } + // Loose realtime filtering with and without path/table patterns must preserve the expected + // matching data. @Ignore @Test public void testRealtimeLooseRange() throws Exception { @@ -738,7 +786,7 @@ public void testRealtimeLooseRange() throws Exception { TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); }; - TableModelUtils.insertData("test", "test2", 0, 20, senderEnv); + TableModelUtils.insertData(database("test"), "test2", 0, 20, senderEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -749,7 +797,7 @@ public void testRealtimeLooseRange() throws Exception { extractorAttributes.put("source.capture.tree", "true"); extractorAttributes.put("mode.double-living", "true"); extractorAttributes.put("source.table-name", "test1"); - extractorAttributes.put("source.path", "root.db.d1.at1"); + extractorAttributes.put("source.path", isolateDatabasePaths("root.db.d1.at1")); extractorAttributes.put("source.inclusion", "data.insert"); extractorAttributes.put("source.realtime.loose-range", "time, path"); extractorAttributes.put("source.start-time", "2"); @@ -776,34 +824,40 @@ public void testRealtimeLooseRange() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db.d1 (time, at1, at2)" + " values (1, 1, 2), (3, 3, 4)", "flush"), + isolateDatabasePaths("insert into root.db.d1 (time, at1, at2)") + + " values (1, 1, 2), (3, 3, 4)", + "flush"), null); TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db.d1 (time, at1)" + " values (5, 1), (16, 3)", - "insert into root.db.d1 (time, at1, at2)" + " values (5, 1, 2), (6, 3, 4)", + isolateDatabasePaths("insert into root.db.d1 (time, at1)") + + " values (5, 1), (16, 3)", + isolateDatabasePaths("insert into root.db.d1 (time, at1, at2)") + + " values (5, 1, 2), (6, 3, 4)", "flush"), null); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(at1) from root.db.d1 where time >= 2 and time <= 10", + isolateDatabasePaths("select count(at1) from root.db.d1 where time >= 2 and time <= 10"), new HashMap() { { - put("count(root.db.d1.at1)", "3"); + put(isolateDatabasePaths("count(root.db.d1.at1)"), "3"); } }); - TableModelUtils.insertData("test", "test1", 10, 20, senderEnv); + TableModelUtils.insertData(database("test"), "test1", 10, 20, senderEnv); - TableModelUtils.insertData("test", "test2", 10, 20, senderEnv); + TableModelUtils.insertData(database("test"), "test2", 10, 20, senderEnv); - TableModelUtils.assertCountData("test", "test1", 20, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database("test"), "test1", 20, receiverEnv, handleFailure); } } + // Table-model CREATE/ALTER PIPE must accept now in prefixed and unprefixed source/history time + // attributes; execute all six variants in order on the same connection. @Test public void testTableModeSQLSupportNowFunc() { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -830,96 +884,71 @@ public void testTableModeSQLSupportNowFunc() { try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute(p1); - } catch (final SQLException e) { - fail(e.getMessage()); - } - final String p2 = - String.format( - "create pipe p2" - + " with extractor (" - + "'capture.table'='true'," - + "'extractor.history.enable'='true'," - + "'start-time'='now'," - + "'end-time'='now'," - + "'history.start-time'='now'," - + "'history.end-time'='now')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + final String p2 = + String.format( + "create pipe p2" + + " with extractor (" + + "'capture.table'='true'," + + "'extractor.history.enable'='true'," + + "'start-time'='now'," + + "'end-time'='now'," + + "'history.start-time'='now'," + + "'history.end-time'='now')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p2); - } catch (final SQLException e) { - fail(e.getMessage()); - } - final String p3 = - String.format( - "create pipe p3" - + " with extractor (" - + "'capture.table'='true'," - + "'extractor.history.enable'='true'," - + "'extractor.start-time'='now'," - + "'extractor.end-time'='now'," - + "'extractor.history.start-time'='now'," - + "'extractor.history.end-time'='now')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + final String p3 = + String.format( + "create pipe p3" + + " with extractor (" + + "'capture.table'='true'," + + "'extractor.history.enable'='true'," + + "'extractor.start-time'='now'," + + "'extractor.end-time'='now'," + + "'extractor.history.start-time'='now'," + + "'extractor.history.end-time'='now')" + + " with connector (" + + "'connector'='iotdb-thrift-connector'," + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - String alterP3 = - "alter pipe p3" - + " modify extractor (" - + "'history.enable'='true'," - + "'start-time'='now'," - + "'end-time'='now'," - + "'history.start-time'='now'," - + "'history.end-time'='now')"; - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + String alterP3 = + "alter pipe p3" + + " modify extractor (" + + "'history.enable'='true'," + + "'start-time'='now'," + + "'end-time'='now'," + + "'history.start-time'='now'," + + "'history.end-time'='now')"; statement.execute(alterP3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - alterP3 = - "alter pipe p3" - + " modify extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.start-time'='now'," - + "'extractor.end-time'='now'," - + "'extractor.history.start-time'='now'," - + "'extractor.history.end-time'='now')"; - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + alterP3 = + "alter pipe p3" + + " modify extractor (" + + "'extractor.history.enable'='true'," + + "'extractor.start-time'='now'," + + "'extractor.end-time'='now'," + + "'extractor.history.start-time'='now'," + + "'extractor.history.end-time'='now')"; statement.execute(alterP3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - alterP3 = - "alter pipe p3" - + " modify source (" - + "'extractor.history.enable'='true'," - + "'source.start-time'='now'," - + "'source.end-time'='now'," - + "'source.history.start-time'='now'," - + "'source.history.end-time'='now')"; - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + alterP3 = + "alter pipe p3" + + " modify source (" + + "'extractor.history.enable'='true'," + + "'source.start-time'='now'," + + "'source.end-time'='now'," + + "'source.history.start-time'='now'," + + "'source.history.end-time'='now')"; statement.execute(alterP3); } catch (final SQLException e) { fail(e.getMessage()); @@ -929,8 +958,27 @@ public void testTableModeSQLSupportNowFunc() { private void assertTimeseriesCountOnReceiver(BaseEnv receiverEnv, int count) { TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "count timeseries root.db*.**", + isolateDatabasePaths("count timeseries root.db*.**"), "count(timeseries),", Collections.singleton(count + ",")); } + + private String isolateDatabasePaths(final String value) { + // Prefix the database name so db* remains a legal wildcard at the end of the path node. + return value.replace("root.db", "root." + databaseSuffix() + "_db"); + } + + private String database(final String name) { + // Database deletion is asynchronous; never reuse another method's database incarnation. + return name + "_" + databaseSuffix(); + } + + private String databaseSuffix() { + // Stay below the receiver's 64-character database path limit, including root and the prefix. + // Retain a hash when shortening names so methods with the same prefix use different databases. + final String method = testName.getMethodName().toLowerCase(Locale.ROOT); + return method.length() <= 40 + ? method + : method.substring(0, 31) + "_" + Integer.toUnsignedString(method.hashCode(), 16); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java index a759a714a7faf..a0b04ef450a1e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java @@ -30,7 +30,7 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -45,7 +45,7 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeSwitchStatusIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeSwitchStatusIT extends AbstractPipeTableModelSharedClusterIT { @Override @Before diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java index ec1afaa43eea6..82405d3322196 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java @@ -29,7 +29,7 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -50,7 +50,7 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeSyntaxIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeSyntaxIT extends AbstractPipeTableModelSharedClusterIT { @Override @Before @@ -58,6 +58,8 @@ public void setUp() { super.setUp(); } + // Legal table-model identifiers, including quoted numbers and reserved words, must create, + // report RUNNING, and drop cleanly while all statements reuse one SQL connection. @Test public void testValidPipeName() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -66,15 +68,16 @@ public void testValidPipeName() throws Exception { final int receiverPort = receiverDataNode.getPort(); try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { final List validPipeNames = Arrays.asList("Pipe_1", "\"`33`\"", "root", "\"`中文`\"", "\"`with`\""); final List expectedPipeNames = Arrays.asList("Pipe_1", "`33`", "root", "`中文`", "`with`"); for (final String pipeName : validPipeNames) { - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe %s" @@ -111,8 +114,7 @@ public void testValidPipeName() throws Exception { } for (final String pipeName : validPipeNames) { - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute(String.format("drop pipe %s", pipeName)); } catch (SQLException e) { e.printStackTrace(); @@ -204,6 +206,8 @@ public void testRevertStageOrder() throws Exception { } } + // Missing extractor, processor, or connector stages must be rejected; the two complete forms + // must remain valid after those failures on the same SQL connection. @Test public void testMissingStage() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -212,23 +216,22 @@ public void testMissingStage() throws Exception { final int receiverPort = receiverDataNode.getPort(); try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + try { statement.execute("create pipe p1"); fail(); } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute("create pipe p2 with extractor ('extractor'='iotdb-extractor')"); fail(); } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( "create pipe p3" + " with extractor ('extractor'='iotdb-extractor')" @@ -237,8 +240,7 @@ public void testMissingStage() throws Exception { } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p4" @@ -253,8 +255,7 @@ public void testMissingStage() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p5" @@ -280,6 +281,8 @@ public void testMissingStage() throws Exception { } } + // Empty attributes and unknown extractor/connector parameters must reject only the invalid + // definitions while the valid definition remains visible; reuse one SQL connection throughout. @Test public void testInvalidParameter() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -288,9 +291,10 @@ public void testInvalidParameter() throws Exception { final int receiverPort = receiverDataNode.getPort(); try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p1" @@ -307,8 +311,7 @@ public void testInvalidParameter() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p3" @@ -324,8 +327,7 @@ public void testInvalidParameter() throws Exception { } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p4" @@ -350,6 +352,8 @@ public void testInvalidParameter() throws Exception { } } + // Quoted and unquoted extractor/processor parameters must accept only the valid forms; all + // syntax probes share one table-model SQL connection to avoid reconnect overhead. @Test public void testBrackets() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -358,9 +362,10 @@ public void testBrackets() throws Exception { final int receiverPort = receiverDataNode.getPort(); try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe extractor1" @@ -376,8 +381,7 @@ public void testBrackets() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe source2" @@ -393,8 +397,7 @@ public void testBrackets() throws Exception { e.printStackTrace(); } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe extractor3" @@ -419,8 +422,7 @@ public void testBrackets() throws Exception { e.printStackTrace(); } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe extractor4" @@ -444,8 +446,7 @@ public void testBrackets() throws Exception { } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe extractor5" @@ -469,8 +470,7 @@ public void testBrackets() throws Exception { } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe processor1" @@ -486,8 +486,7 @@ public void testBrackets() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe processor2" @@ -503,8 +502,7 @@ public void testBrackets() throws Exception { e.printStackTrace(); } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe processor3" @@ -520,8 +518,7 @@ public void testBrackets() throws Exception { e.printStackTrace(); } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe processor4" @@ -536,8 +533,7 @@ public void testBrackets() throws Exception { } catch (Exception ignored) { } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe processor5" @@ -649,6 +645,8 @@ public void testShowPipeWithWrongPipeName() throws Exception { } } + // Inclusion and exclusion syntax must reject the three invalid combinations and keep the + // valid definition; all probes reuse one table-model SQL connection. @Test public void testInclusionPattern() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -657,10 +655,11 @@ public void testInclusionPattern() throws Exception { final int receiverPort = receiverDataNode.getPort(); try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { // Empty inclusion - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p2" @@ -677,8 +676,7 @@ public void testInclusionPattern() throws Exception { } // Invalid inclusion - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p3" @@ -695,8 +693,7 @@ public void testInclusionPattern() throws Exception { } // Invalid exclusion - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p4" @@ -713,8 +710,7 @@ public void testInclusionPattern() throws Exception { } // Valid - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p4" diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeTsFileDecompositionWithModsIT.java index d010513b4818c..2f1e22b441ac8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -20,6 +20,7 @@ package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -32,8 +33,10 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.Set; import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueryWithRetry; @@ -42,96 +45,43 @@ public class IoTDBPipeTsFileDecompositionWithModsIT extends AbstractPipeTableModelDualManualIT { /** - * Test IoTDB pipe handling TsFile decomposition with Mods (modification operations) in table - * model - * - *

Test scenario: 1. Create two storage groups sg1 and sg2, each containing table1 2. Insert - * small amount of data in sg1 (1-6 rows), insert large amount of data in sg2 (110 batches, 100 - * rows per batch) 3. Execute FLUSH operation to persist data to TsFile 4. Execute multiple DELETE - * operations on sg1, deleting data in time ranges 2-4 and 3-5 5. Execute multiple DELETE - * operations on sg2, deleting data matching specific conditions (s0-s3 field values) 6. Execute - * FLUSH operation again 7. Create pipe with mods enabled, synchronize data to receiver 8. Verify - * correctness of receiver data: - sg1 only retains time=1 data, time=2-4 data is correctly - * deleted - sg2 DELETE operation results meet expectations (t10 retains 1000 rows, t11 all - * deleted, t12 retains 5900 rows, etc.) - * - *

Test purpose: Verify that IoTDB pipe can correctly handle Mods (modification operations) in - * TsFile, ensuring DELETE operations can be correctly synchronized to the receiver and data - * consistency is guaranteed. + * Flush five rows in sg1 and 55,000 rows in sg2 (devices t10-t14, timestamps [0, 11000)), then + * apply overlapping time deletions and tag-specific deletions. A mods-enabled tablet pipe must + * retain only time=1 in sg1 and respectively 1000, 0, 5900, 1000 and 10000 rows for t10-t14 in + * sg2, preserving both inclusive and exclusive deletion boundaries. */ @Test - public void testTsFileDecompositionWithMods() { + public void testTsFileDecompositionWithMods() throws Exception { TableModelUtils.createDataBaseAndTable(senderEnv, "table1", "sg1"); TableModelUtils.createDataBaseAndTable(receiverEnv, "table1", "sg1"); TableModelUtils.insertData("sg1", "table1", 1, 6, senderEnv); TableModelUtils.createDataBaseAndTable(senderEnv, "table1", "sg2"); - for (int i = 1; i <= 110; i++) { - TableModelUtils.insertData("sg2", "table1", 10, 15, (i - 1) * 100, i * 100, senderEnv); + try (final ITableSession session = senderEnv.getTableSessionConnection()) { + // The former 110 calls did not flush between batches. Preserve every row and both flush + // boundaries while reusing one Session and increasing SQL batches from 10 to 100 rows. + TableModelUtils.insertData("sg2", "table1", 10, 15, 0, 11000, session, 100); + session.executeNonQueryStatement("FLUSH"); + + session.executeNonQueryStatement("USE sg1"); + session.executeNonQueryStatement("DELETE FROM table1 WHERE time >= 2 AND time <= 4"); + session.executeNonQueryStatement("DELETE FROM table1 WHERE time >= 3 AND time <= 5"); + + session.executeNonQueryStatement("USE sg2"); + for (final String deletion : + Arrays.asList( + "DELETE FROM table1 WHERE time >= 0 AND time < 10000 AND s0 ='t10' AND s1='t10' AND s2='t10' AND s3='t10'", + "DELETE FROM table1 WHERE time >= 0 AND time <= 11000 AND s0 ='t11' AND s1='t11' AND s2='t11' AND s3='t11'", + "DELETE FROM table1 WHERE time >= 5000 AND time < 10100 AND s0 ='t12' AND s1='t12' AND s2='t12' AND s3='t12'", + "DELETE FROM table1 WHERE time >= 0 AND time < 10000 AND s0 ='t13' AND s1='t13' AND s2='t13' AND s3='t13'", + "DELETE FROM table1 WHERE time >= 10000 AND time <= 11000 AND s0 ='t14' AND s1='t14' AND s2='t14' AND s3='t14'")) { + session.executeNonQueryStatement(deletion); + } + session.executeNonQueryStatement("FLUSH"); } - executeNonQueryWithRetry(senderEnv, "FLUSH"); - - executeNonQueryWithRetry( - senderEnv, - "DELETE FROM table1 WHERE time >= 2 AND time <= 4", - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD, - "sg1", - "table"); - - executeNonQueryWithRetry( - senderEnv, - "DELETE FROM table1 WHERE time >= 3 AND time <= 5", - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD, - "sg1", - "table"); - - executeNonQueryWithRetry( - senderEnv, - "DELETE FROM table1 WHERE time >= 0 AND time < 10000 AND s0 ='t10' AND s1='t10' AND s2='t10' AND s3='t10'", - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD, - "sg2", - "table"); - - executeNonQueryWithRetry( - senderEnv, - "DELETE FROM table1 WHERE time >= 0 AND time <= 11000 AND s0 ='t11' AND s1='t11' AND s2='t11' AND s3='t11'", - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD, - "sg2", - "table"); - - executeNonQueryWithRetry( - senderEnv, - "DELETE FROM table1 WHERE time >= 5000 AND time < 10100 AND s0 ='t12' AND s1='t12' AND s2='t12' AND s3='t12'", - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD, - "sg2", - "table"); - - executeNonQueryWithRetry( - senderEnv, - "DELETE FROM table1 WHERE time >= 0 AND time < 10000 AND s0 ='t13' AND s1='t13' AND s2='t13' AND s3='t13'", - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD, - "sg2", - "table"); - - executeNonQueryWithRetry( - senderEnv, - "DELETE FROM table1 WHERE time >= 10000 AND time <= 11000 AND s0 ='t14' AND s1='t14' AND s2='t14' AND s3='t14'", - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD, - "sg2", - "table"); - - executeNonQueryWithRetry(senderEnv, "FLUSH"); - - HashSet expectedResults = new HashSet<>(); + final Set expectedResults = new HashSet<>(); expectedResults.add( "t1,t1,t1,t1,1,1.0,1,1970-01-01T00:00:00.001Z,1,1.0,1970-01-01,1,1970-01-01T00:00:00.001Z,"); @@ -149,56 +99,31 @@ public void testTsFileDecompositionWithMods() { Collections.emptySet(), "sg1"); - // Wait until the deletion mods are visible through every sender endpoint before creating the - // pipe. Historical extraction may otherwise read a source TsFile from a stale region leader. - for (DataNodeWrapper dataNode : senderEnv.getDataNodeWrapperList()) { + final String deviceCountsSql = + "SELECT s0, s1, s2, s3, COUNT(*) AS count FROM table1 GROUP BY s0, s1, s2, s3"; + final String deviceCountsHeader = "s0,s1,s2,s3,count,"; + final Set expectedDeviceCounts = + new HashSet<>( + Arrays.asList( + "t10,t10,t10,t10,1000,", + "t12,t12,t12,t12,5900,", + "t13,t13,t13,t13,1000,", + "t14,t14,t14,t14,10000,")); + + // Check every sender endpoint before extraction to avoid stale mods. Grouping by all four + // tags retains the exact device predicates, including t11's absence, in one query per endpoint. + for (final DataNodeWrapper dataNode : senderEnv.getDataNodeWrapperList()) { TestUtils.assertDataEventuallyOnEnv( - senderEnv, - dataNode, - "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t10' AND s1='t10' AND s2='t10' AND s3='t10'", - "count,", - Collections.singleton("1000,"), - "sg2"); - - TestUtils.assertDataEventuallyOnEnv( - senderEnv, - dataNode, - "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t11' AND s1='t11' AND s2='t11' AND s3='t11'", - "count,", - Collections.singleton("0,"), - "sg2"); - - TestUtils.assertDataEventuallyOnEnv( - senderEnv, - dataNode, - "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t12' AND s1='t12' AND s2='t12' AND s3='t12'", - "count,", - Collections.singleton("5900,"), - "sg2"); - - TestUtils.assertDataEventuallyOnEnv( - senderEnv, - dataNode, - "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t13' AND s1='t13' AND s2='t13' AND s3='t13'", - "count,", - Collections.singleton("1000,"), - "sg2"); - - TestUtils.assertDataEventuallyOnEnv( - senderEnv, - dataNode, - "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t14' AND s1='t14' AND s2='t14' AND s3='t14'", - "count,", - Collections.singleton("10000,"), - "sg2"); + senderEnv, dataNode, deviceCountsSql, deviceCountsHeader, expectedDeviceCounts, "sg2"); } executeNonQueryWithRetry( senderEnv, String.format( - "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true', 'capture.table'='true', 'inclusion'='data.insert,data.delete') WITH CONNECTOR('ip'='%s', 'port'='%s', 'username'='root', 'format'='tablet')", + "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true', 'capture.table'='true', 'inclusion'='data.insert,data.delete') WITH CONNECTOR('ip'='%s', 'port'='%s', 'username'='%s', 'format'='tablet')", receiverEnv.getDataNodeWrapperList().get(0).getIp(), - receiverEnv.getDataNodeWrapperList().get(0).getPort()), + receiverEnv.getDataNodeWrapperList().get(0).getPort(), + SessionConfig.DEFAULT_USER), SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, null, @@ -219,38 +144,6 @@ public void testTsFileDecompositionWithMods() { "sg1"); TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t10' AND s1='t10' AND s2='t10' AND s3='t10'", - "count,", - Collections.singleton("1000,"), - "sg2"); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t11' AND s1='t11' AND s2='t11' AND s3='t11'", - "count,", - Collections.singleton("0,"), - "sg2"); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t12' AND s1='t12' AND s2='t12' AND s3='t12'", - "count,", - Collections.singleton("5900,"), - "sg2"); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t13' AND s1='t13' AND s2='t13' AND s3='t13'", - "count,", - Collections.singleton("1000,"), - "sg2"); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t14' AND s1='t14' AND s2='t14' AND s3='t14'", - "count,", - Collections.singleton("10000,"), - "sg2"); + receiverEnv, deviceCountsSql, deviceCountsHeader, expectedDeviceCounts, "sg2"); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java index fe90aedf647b7..0edc9605c9819 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java @@ -20,45 +20,52 @@ package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.exception.auth.AccessDeniedException; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.auth.AuthorityChecker; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; -import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.nio.file.Files; +import java.nio.file.Path; import java.sql.Connection; +import java.sql.ResultSet; import java.sql.Statement; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.stream.Stream; +import static org.awaitility.Awaitility.await; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeWithLoadIT extends AbstractPipeTableModelDualManualIT { - - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); +public class IoTDBPipeWithLoadIT extends AbstractPipeTableModelSharedClusterIT { + @Override + protected void setupConfig() { + // Retain this suite's original load/compaction settings for the shared clusters. senderEnv .getConfig() .getCommonConfig() @@ -85,11 +92,9 @@ public void setUp() { .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false) .setEnforceStrongPassword(false); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); } + // Verify historical loading applies deletions; disabled until table mods transfer is supported. @Ignore // not support @Test public void testReceiverNotLoadDeletedTimeseries() throws Exception { @@ -110,7 +115,7 @@ public void testReceiverNotLoadDeletedTimeseries() throws Exception { extractorAttributes.put("mods", "true"); extractorAttributes.put("capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); - extractorAttributes.put("user", "root"); + extractorAttributes.put("user", SessionConfig.DEFAULT_USER); connectorAttributes.put("connector.batch.enable", "false"); connectorAttributes.put("connector.ip", receiverIp); @@ -137,9 +142,11 @@ public void testReceiverNotLoadDeletedTimeseries() throws Exception { } } - // Test that receiver will not load data when table exists but TAG columns mismatch + // Historical files whose TAG columns disagree with the receiver must fail schema validation + // without changing the receiver's existing rows. Observe that failure before checking the rows. @Test public void testReceiverNotLoadWhenIdColumnMismatch() throws Exception { + final String database = "load_tag_mismatch"; final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); @@ -156,7 +163,9 @@ public void testReceiverNotLoadWhenIdColumnMismatch() throws Exception { extractorAttributes.put("capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.realtime.mode", "file"); - extractorAttributes.put("user", "root"); + // Restrict capture to this method, including any historical files left in the shared cluster. + extractorAttributes.put("extractor.database-name", database); + extractorAttributes.put("user", SessionConfig.DEFAULT_USER); connectorAttributes.put("connector.batch.enable", "false"); connectorAttributes.put("connector.ip", receiverIp); @@ -166,8 +175,8 @@ public void testReceiverNotLoadWhenIdColumnMismatch() throws Exception { (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { try (Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { - statement.execute("create database if not exists db"); - statement.execute("use db"); + statement.execute("create database if not exists " + database); + statement.execute("use " + database); statement.execute( "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD)"); statement.execute("INSERT INTO t1(time,tag1,tag2,s1,s2) values(1, 'd1', 'd2', 'red', 1)"); @@ -179,8 +188,8 @@ public void testReceiverNotLoadWhenIdColumnMismatch() throws Exception { try (Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { - statement.execute("create database if not exists db"); - statement.execute("use db"); + statement.execute("create database if not exists " + database); + statement.execute("use " + database); statement.execute( "create table if not exists t1(tag3 STRING TAG, tag4 STRING TAG, s3 TEXT FIELD, s4 INT32 FIELD)"); statement.execute("INSERT INTO t1(time,tag3,tag4,s3,s4) values(1, 'd3', 'd4', 'red2', 10)"); @@ -200,14 +209,16 @@ public void testReceiverNotLoadWhenIdColumnMismatch() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - try { - // wait some time - Thread.sleep(10_000); - } catch (InterruptedException e) { - e.printStackTrace(); - } + awaitReceiverLoadFailure( + database, + String.format( + DataNodeQueryMessages + .QUERY_EXCEPTION_CAN_NOT_CREATE_TABLE_BECAUSE_INCOMING_TABLE_HAS_NO_LESS_D3D33555, + "tag3", + 0, + -1)); - Set expectedResSet = new java.util.HashSet<>(); + Set expectedResSet = new HashSet<>(); expectedResSet.add("1970-01-01T00:00:00.002Z,d3,d4,blue2,20,"); expectedResSet.add("1970-01-01T00:00:00.001Z,d3,d4,red2,10,"); // make sure data are not transferred @@ -216,7 +227,7 @@ public void testReceiverNotLoadWhenIdColumnMismatch() throws Exception { "select * from t1", "time,tag3,tag4,s3,s4,", expectedResSet, - "db", + database, handleFailure); } } @@ -225,6 +236,7 @@ public void testReceiverNotLoadWhenIdColumnMismatch() throws Exception { // incoming TAG columns @Test public void testReceiverAutoExtendIdColumn() throws Exception { + final String database = "load_extend_tag"; final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); @@ -241,7 +253,9 @@ public void testReceiverAutoExtendIdColumn() throws Exception { extractorAttributes.put("capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.realtime.mode", "file"); - extractorAttributes.put("user", "root"); + // Restrict capture to this method, including any historical files left in the shared cluster. + extractorAttributes.put("extractor.database-name", database); + extractorAttributes.put("user", SessionConfig.DEFAULT_USER); connectorAttributes.put("connector.batch.enable", "false"); connectorAttributes.put("connector.ip", receiverIp); @@ -251,8 +265,8 @@ public void testReceiverAutoExtendIdColumn() throws Exception { (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { try (Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { - statement.execute("create database if not exists db"); - statement.execute("use db"); + statement.execute("create database if not exists " + database); + statement.execute("use " + database); statement.execute( "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, tag3 STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD)"); statement.execute( @@ -266,8 +280,8 @@ public void testReceiverAutoExtendIdColumn() throws Exception { try (Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { - statement.execute("create database if not exists db"); - statement.execute("use db"); + statement.execute("create database if not exists " + database); + statement.execute("use " + database); statement.execute( "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, s3 TEXT FIELD, s4 INT32 FIELD)"); statement.execute("INSERT INTO t1(time,tag1,tag2,s3,s4) values(1, 'd1', 'd2', 'red2', 10)"); @@ -287,7 +301,7 @@ public void testReceiverAutoExtendIdColumn() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - Set expectedResSet = new java.util.HashSet<>(); + Set expectedResSet = new HashSet<>(); expectedResSet.add("1970-01-01T00:00:00.001Z,d1,d2,null,null,d3,red,1,"); expectedResSet.add("1970-01-01T00:00:00.002Z,d1,d2,null,null,d3,blue,2,"); expectedResSet.add("1970-01-01T00:00:00.001Z,d1,d2,red2,10,null,null,null,"); @@ -298,7 +312,7 @@ public void testReceiverAutoExtendIdColumn() throws Exception { "select * from t1", "time,tag1,tag2,s3,s4,tag3,s1,s2,", expectedResSet, - "db", + database, handleFailure); } } @@ -307,6 +321,7 @@ public void testReceiverAutoExtendIdColumn() throws Exception { // existing TAG columns @Test public void testLoadWhenIncomingIdColumnsArePrefixOfExisting() throws Exception { + final String database = "load_prefix_tag"; final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); @@ -323,7 +338,9 @@ public void testLoadWhenIncomingIdColumnsArePrefixOfExisting() throws Exception extractorAttributes.put("capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.realtime.mode", "file"); - extractorAttributes.put("user", "root"); + // Restrict capture to this method, including any historical files left in the shared cluster. + extractorAttributes.put("extractor.database-name", database); + extractorAttributes.put("user", SessionConfig.DEFAULT_USER); connectorAttributes.put("connector.batch.enable", "false"); connectorAttributes.put("connector.ip", receiverIp); @@ -333,8 +350,8 @@ public void testLoadWhenIncomingIdColumnsArePrefixOfExisting() throws Exception (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { try (Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { - statement.execute("create database if not exists db"); - statement.execute("use db"); + statement.execute("create database if not exists " + database); + statement.execute("use " + database); statement.execute( "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD)"); statement.execute("INSERT INTO t1(time,tag1,tag2,s1,s2) values(1, 'd1', 'd2', 'red', 1)"); @@ -346,8 +363,8 @@ public void testLoadWhenIncomingIdColumnsArePrefixOfExisting() throws Exception try (Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { - statement.execute("create database if not exists db"); - statement.execute("use db"); + statement.execute("create database if not exists " + database); + statement.execute("use " + database); statement.execute( "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, tag3 STRING TAG,s3 TEXT FIELD, s4 INT32 FIELD)"); statement.execute( @@ -368,7 +385,7 @@ public void testLoadWhenIncomingIdColumnsArePrefixOfExisting() throws Exception Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - Set expectedResSet = new java.util.HashSet<>(); + Set expectedResSet = new HashSet<>(); expectedResSet.add("1970-01-01T00:00:00.001Z,d1,d2,d3,red2,10,null,null,"); expectedResSet.add("1970-01-01T00:00:00.002Z,d1,d2,d3,blue2,20,null,null,"); expectedResSet.add("1970-01-01T00:00:00.001Z,d1,d2,null,null,null,red,1,"); @@ -379,13 +396,15 @@ public void testLoadWhenIncomingIdColumnsArePrefixOfExisting() throws Exception "select * from t1", "time,tag1,tag2,tag3,s3,s4,s1,s2,", expectedResSet, - "db", + database, handleFailure); } } + // Verify a table deleted before historical capture is not recreated on the receiver. @Test public void testLoadAutoCreateWithTableDeletion() throws Exception { + final String database = "load_deleted_table"; final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); @@ -397,7 +416,9 @@ public void testLoadAutoCreateWithTableDeletion() throws Exception { extractorAttributes.put("capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.realtime.mode", "file"); - extractorAttributes.put("user", "root"); + // Restrict capture to this method, including any historical files left in the shared cluster. + extractorAttributes.put("extractor.database-name", database); + extractorAttributes.put("user", SessionConfig.DEFAULT_USER); connectorAttributes.put("connector.batch.enable", "false"); connectorAttributes.put("connector.ip", receiverIp); @@ -407,15 +428,15 @@ public void testLoadAutoCreateWithTableDeletion() throws Exception { (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { - statement.execute("create database if not exists db"); - statement.execute("use db"); + statement.execute("create database if not exists " + database); + statement.execute("use " + database); statement.execute( "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD)"); statement.execute("INSERT INTO t1(time,tag1,tag2,s1,s2) values(1, 'd1', 'd2', 'red', 1)"); statement.execute("INSERT INTO t1(time,tag1,tag2,s1,s2) values(2, 'd1', 'd2', 'blue', 2)"); statement.execute("flush"); // Start historical transfer only after the table and its data have been deleted. - TestUtils.dropTable(connection, "db", "t1"); + TestUtils.dropTable(connection, database, "t1"); } catch (Exception e) { fail(e.getMessage()); } @@ -431,12 +452,15 @@ public void testLoadAutoCreateWithTableDeletion() throws Exception { // Ensure the deleted table won't be created // Now the database will also be created at receiver - TestUtils.assertAlwaysFail(receiverEnv, "describe db.t1"); + TestUtils.assertAlwaysFail(receiverEnv, "describe " + database + ".t1"); } } + // A receiver user with CREATE but no INSERT privilege must reject the historical file and leave + // its table absent. Wait for that specific denial before checking the catalog. @Test public void testLoadAutoCreateWithoutInsertPermission() throws Exception { + final String database = "load_no_insert"; final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); @@ -448,19 +472,21 @@ public void testLoadAutoCreateWithoutInsertPermission() throws Exception { extractorAttributes.put("capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.realtime.mode", "file"); - extractorAttributes.put("user", "root"); + // Restrict capture to this method, including any historical files left in the shared cluster. + extractorAttributes.put("extractor.database-name", database); + extractorAttributes.put("user", SessionConfig.DEFAULT_USER); connectorAttributes.put("connector.batch.enable", "false"); connectorAttributes.put("connector.ip", receiverIp); connectorAttributes.put("connector.port", Integer.toString(receiverPort)); connectorAttributes.put("connector.user", "user01"); - connectorAttributes.put("connector.password", "1234123456789"); + connectorAttributes.put("connector.password", "IoTDB!Test2024"); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { try (final Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { - statement.execute("create user user01 '1234123456789'"); + statement.execute("create user user01 'IoTDB!Test2024'"); statement.execute("grant create on any to user user01"); } catch (final Exception e) { fail(e.getMessage()); @@ -468,8 +494,8 @@ public void testLoadAutoCreateWithoutInsertPermission() throws Exception { try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { - statement.execute("create database if not exists db"); - statement.execute("use db"); + statement.execute("create database if not exists " + database); + statement.execute("use " + database); statement.execute( "create table if not exists t1(tag1 STRING TAG, tag2 STRING TAG, s1 TEXT FIELD, s2 INT32 FIELD)"); statement.execute("INSERT INTO t1(time,tag1,tag2,s1,s2) values(1, 'd1', 'd2', 'red', 1)"); @@ -488,9 +514,49 @@ public void testLoadAutoCreateWithoutInsertPermission() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - // Ensure the table without insert privilege won't be created - // Now the database will also be created at receiver - TestUtils.assertAlwaysFail(receiverEnv, "describe db.t1"); + awaitReceiverLoadFailure( + database, + AccessDeniedException.PREFIX + + AuthorityChecker.getTSStatus(false, PrivilegeType.INSERT, database, "t1") + .getMessage()); + try (final Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement(); + final ResultSet tables = + statement.executeQuery( + "select table_name from information_schema.tables where database = '" + + database + + "' and table_name = 't1'")) { + // The load may be rejected before even creating the database. The system catalog checks + // table absence in either case without treating an unrelated query failure as success. + Assert.assertFalse("The rejected load must not create the receiver table", tables.next()); + } } } + + private void awaitReceiverLoadFailure(final String database, final String expectedMessage) { + // Rejected loads keep retrying and suppress runtime error reporting, so neither a successful + // completion barrier nor an empty SHOW PIPES error proves that the receiver tried the file. + // Each scenario has a distinct database in these fresh class-scoped clusters, so a preceding + // scenario's log cannot satisfy the wait. Match the database and failure on the same line. + await() + .alias("Receiver load failure: " + expectedMessage) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .atMost(120, TimeUnit.SECONDS) + .until( + () -> { + for (final DataNodeWrapper node : receiverEnv.getDataNodeWrapperList()) { + final Path log = Path.of(node.getLogDirPath(), node.getId() + ".log"); + if (Files.exists(log)) { + try (final Stream lines = Files.lines(log)) { + if (lines.anyMatch( + line -> line.contains(database) && line.contains(expectedMessage))) { + return true; + } + } + } + } + return false; + }); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java index e7a7cd508afed..1945f8a3bf27f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java @@ -26,25 +26,31 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.pipe.it.dual.PipeTestUtils.RetryFlushHandler; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Locale; import java.util.Map; -import java.util.function.Consumer; +// All pattern cases use the same configuration; reset pipes/databases between methods. @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBTablePatternFormatIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBTablePatternFormatIT extends AbstractPipeTableModelSharedClusterIT { + + @Rule public final TestName testName = new TestName(); @Override @Before @@ -52,32 +58,28 @@ public void setUp() { super.setUp(); } + // A table-name pattern must replicate both matching tables' history and realtime rows [0, 200). @Test public void testTableNamePattern() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final RetryFlushHandler handleFailure = new RetryFlushHandler(senderEnv, receiverEnv)) { - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", database("test")); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test1")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", database("pattern")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", database("pattern1")); + TableModelUtils.insertData(database("test"), "test", 0, 100, senderEnv); - TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + TableModelUtils.insertData(database("test1"), "test1", 0, 100, senderEnv); - TableModelUtils.insertData("pattern", "pattern", 0, 100, senderEnv); + TableModelUtils.insertData(database("pattern"), "pattern", 0, 100, senderEnv); - TableModelUtils.insertData("pattern1", "pattern1", 0, 100, senderEnv); + TableModelUtils.insertData(database("pattern1"), "pattern1", 0, 100, senderEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -104,53 +106,49 @@ public void testTableNamePattern() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + TableModelUtils.insertData(database("test"), "test", 100, 200, senderEnv); - TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + TableModelUtils.insertData(database("test1"), "test1", 100, 200, senderEnv); - TableModelUtils.insertData("pattern", "pattern", 100, 200, senderEnv); + TableModelUtils.insertData(database("pattern"), "pattern", 100, 200, senderEnv); - TableModelUtils.insertData("pattern1", "pattern1", 100, 200, senderEnv); + TableModelUtils.insertData(database("pattern1"), "pattern1", 100, 200, senderEnv); - TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); - TableModelUtils.assertData("test1", "test1", 0, 200, receiverEnv, handleFailure); - TableModelUtils.hasDataBase("test", receiverEnv); - TableModelUtils.hasDataBase("test1", receiverEnv); + TableModelUtils.assertData(database("test"), "test", 0, 200, receiverEnv, handleFailure); + TableModelUtils.assertData(database("test1"), "test1", 0, 200, receiverEnv, handleFailure); + TableModelUtils.hasDataBase(database("test"), receiverEnv); + TableModelUtils.hasDataBase(database("test1"), receiverEnv); } } + // A table-name pattern with time bounds [0, 49] must load only matching historical rows. @Test public void testTableNamePatternByHistoryTSFile() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final RetryFlushHandler handleFailure = new RetryFlushHandler(senderEnv, receiverEnv)) { - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", database("test")); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test1")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", database("pattern")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", database("pattern1")); + TableModelUtils.insertData(database("test"), "test", 0, 100, senderEnv); - TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + TableModelUtils.insertData(database("test1"), "test1", 0, 100, senderEnv); - TableModelUtils.insertData("pattern", "pattern", 0, 100, senderEnv); + TableModelUtils.insertData(database("pattern"), "pattern", 0, 100, senderEnv); - TableModelUtils.insertData("pattern1", "pattern1", 0, 100, senderEnv); + TableModelUtils.insertData(database("pattern1"), "pattern1", 0, 100, senderEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); final Map connectorAttributes = new HashMap<>(); - // extractorAttributes.put("extractor.database-name", "test.*"); + // extractorAttributes.put("extractor.database-name", database("test.*")); extractorAttributes.put("extractor.table-name", "test.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); @@ -175,38 +173,35 @@ public void testTableNamePatternByHistoryTSFile() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - TableModelUtils.assertData("test", "test", 0, 50, receiverEnv, handleFailure); - TableModelUtils.assertData("test1", "test1", 0, 50, receiverEnv, handleFailure); - TableModelUtils.hasDataBase("test", receiverEnv); - TableModelUtils.hasDataBase("test1", receiverEnv); + TableModelUtils.assertData(database("test"), "test", 0, 50, receiverEnv, handleFailure); + TableModelUtils.assertData(database("test1"), "test1", 0, 50, receiverEnv, handleFailure); + TableModelUtils.hasDataBase(database("test"), receiverEnv); + TableModelUtils.hasDataBase(database("test1"), receiverEnv); } } + // A table-name pattern with time bounds [100, 149] must filter the rows written after Pipe + // creation. @Test public void testTableNamePatternByRealTime() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final RetryFlushHandler handleFailure = new RetryFlushHandler(senderEnv, receiverEnv)) { - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", database("test")); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test1")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", database("pattern")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", database("pattern1")); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); final Map connectorAttributes = new HashMap<>(); - // extractorAttributes.put("extractor.database-name", "test.*"); + // extractorAttributes.put("extractor.database-name", database("test.*")); extractorAttributes.put("extractor.table-name", "test.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); @@ -230,54 +225,50 @@ public void testTableNamePatternByRealTime() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + TableModelUtils.insertData(database("test"), "test", 100, 200, senderEnv); - TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + TableModelUtils.insertData(database("test1"), "test1", 100, 200, senderEnv); - TableModelUtils.insertData("pattern", "pattern", 100, 200, senderEnv); + TableModelUtils.insertData(database("pattern"), "pattern", 100, 200, senderEnv); - TableModelUtils.insertData("pattern1", "pattern1", 100, 200, senderEnv); + TableModelUtils.insertData(database("pattern1"), "pattern1", 100, 200, senderEnv); - TableModelUtils.assertData("test", "test", 100, 150, receiverEnv, handleFailure); - TableModelUtils.assertData("test1", "test1", 100, 150, receiverEnv, handleFailure); - TableModelUtils.hasDataBase("test", receiverEnv); - TableModelUtils.hasDataBase("test1", receiverEnv); + TableModelUtils.assertData(database("test"), "test", 100, 150, receiverEnv, handleFailure); + TableModelUtils.assertData(database("test1"), "test1", 100, 150, receiverEnv, handleFailure); + TableModelUtils.hasDataBase(database("test"), receiverEnv); + TableModelUtils.hasDataBase(database("test1"), receiverEnv); } } + // A database-name pattern must replicate all rows [0, 200) from both matching databases. @Test public void testDataBasePattern() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final RetryFlushHandler handleFailure = new RetryFlushHandler(senderEnv, receiverEnv)) { - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", database("test")); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test1")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", database("pattern")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", database("pattern1")); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + TableModelUtils.insertData(database("test"), "test", 0, 100, senderEnv); - TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + TableModelUtils.insertData(database("test1"), "test1", 0, 100, senderEnv); - TableModelUtils.insertData("pattern", "pattern", 0, 100, senderEnv); + TableModelUtils.insertData(database("pattern"), "pattern", 0, 100, senderEnv); - TableModelUtils.insertData("pattern1", "pattern1", 0, 100, senderEnv); + TableModelUtils.insertData(database("pattern1"), "pattern1", 0, 100, senderEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); final Map connectorAttributes = new HashMap<>(); - extractorAttributes.put("extractor.database-name", "pattern.*"); + extractorAttributes.put("extractor.database-name", database("pattern.*")); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); @@ -298,56 +289,54 @@ public void testDataBasePattern() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + TableModelUtils.insertData(database("test"), "test", 100, 200, senderEnv); - TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + TableModelUtils.insertData(database("test1"), "test1", 100, 200, senderEnv); - TableModelUtils.insertData("pattern", "pattern", 100, 200, senderEnv); + TableModelUtils.insertData(database("pattern"), "pattern", 100, 200, senderEnv); - TableModelUtils.insertData("pattern1", "pattern1", 100, 200, senderEnv); + TableModelUtils.insertData(database("pattern1"), "pattern1", 100, 200, senderEnv); - TableModelUtils.assertData("pattern", "pattern", 0, 200, receiverEnv, handleFailure); - TableModelUtils.assertData("pattern1", "pattern1", 0, 200, receiverEnv, handleFailure); + TableModelUtils.assertData( + database("pattern"), "pattern", 0, 200, receiverEnv, handleFailure); + TableModelUtils.assertData( + database("pattern1"), "pattern1", 0, 200, receiverEnv, handleFailure); HashSet expectedResults = new HashSet(); - TableModelUtils.hasDataBase("pattern", receiverEnv); - TableModelUtils.hasDataBase("pattern1", receiverEnv); + TableModelUtils.hasDataBase(database("pattern"), receiverEnv); + TableModelUtils.hasDataBase(database("pattern1"), receiverEnv); } } + // Creating a Pipe after the writes must load historical rows [0, 100) from matching databases. @Test public void testDataBasePatternByHistoryTSFile() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final RetryFlushHandler handleFailure = new RetryFlushHandler(senderEnv, receiverEnv)) { - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", database("test")); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test1")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", database("pattern")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", database("pattern1")); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + TableModelUtils.insertData(database("test"), "test", 0, 100, senderEnv); - TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + TableModelUtils.insertData(database("test1"), "test1", 0, 100, senderEnv); - TableModelUtils.insertData("pattern", "pattern", 0, 100, senderEnv); + TableModelUtils.insertData(database("pattern"), "pattern", 0, 100, senderEnv); - TableModelUtils.insertData("pattern1", "pattern1", 0, 100, senderEnv); + TableModelUtils.insertData(database("pattern1"), "pattern1", 0, 100, senderEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); final Map connectorAttributes = new HashMap<>(); - extractorAttributes.put("extractor.database-name", "pattern.*"); + extractorAttributes.put("extractor.database-name", database("pattern.*")); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); @@ -368,39 +357,37 @@ public void testDataBasePatternByHistoryTSFile() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - TableModelUtils.assertData("pattern", "pattern", 0, 100, receiverEnv, handleFailure); - TableModelUtils.assertData("pattern1", "pattern1", 0, 100, receiverEnv, handleFailure); + TableModelUtils.assertData( + database("pattern"), "pattern", 0, 100, receiverEnv, handleFailure); + TableModelUtils.assertData( + database("pattern1"), "pattern1", 0, 100, receiverEnv, handleFailure); - TableModelUtils.hasDataBase("pattern1", receiverEnv); - TableModelUtils.hasDataBase("pattern", receiverEnv); + TableModelUtils.hasDataBase(database("pattern1"), receiverEnv); + TableModelUtils.hasDataBase(database("pattern"), receiverEnv); } } + // Creating a Pipe before the writes must replicate realtime rows from both matching databases. @Test public void testDataBasePatternByRealtime() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final RetryFlushHandler handleFailure = new RetryFlushHandler(senderEnv, receiverEnv)) { - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", database("test")); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test1")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", database("pattern")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", database("pattern1")); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); final Map connectorAttributes = new HashMap<>(); - extractorAttributes.put("extractor.database-name", "pattern.*"); + extractorAttributes.put("extractor.database-name", database("pattern.*")); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); @@ -422,22 +409,25 @@ public void testDataBasePatternByRealtime() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - TableModelUtils.insertData("pattern", "pattern", 0, 100, senderEnv); + TableModelUtils.insertData(database("pattern"), "pattern", 0, 100, senderEnv); - TableModelUtils.insertData("pattern1", "pattern1", 0, 100, senderEnv); + TableModelUtils.insertData(database("pattern1"), "pattern1", 0, 100, senderEnv); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + TableModelUtils.insertData(database("test"), "test", 0, 100, senderEnv); - TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + TableModelUtils.insertData(database("test1"), "test1", 0, 100, senderEnv); - TableModelUtils.assertData("pattern", "pattern", 0, 100, receiverEnv, handleFailure); - TableModelUtils.assertData("pattern1", "pattern1", 0, 100, receiverEnv, handleFailure); + TableModelUtils.assertData( + database("pattern"), "pattern", 0, 100, receiverEnv, handleFailure); + TableModelUtils.assertData( + database("pattern1"), "pattern1", 0, 100, receiverEnv, handleFailure); - TableModelUtils.hasDataBase("pattern", receiverEnv); - TableModelUtils.hasDataBase("pattern1", receiverEnv); + TableModelUtils.hasDataBase(database("pattern"), receiverEnv); + TableModelUtils.hasDataBase(database("pattern1"), receiverEnv); } } + // When no table matches both database/table patterns, the receiver must keep no user databases. @Test public void testIoTDBPatternWithDataBaseAndTable() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -448,24 +438,24 @@ public void testIoTDBPatternWithDataBaseAndTable() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", "pattern"); - TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", "pattern1"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", database("test")); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test1")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern", database("pattern")); + TableModelUtils.createDataBaseAndTable(senderEnv, "pattern1", database("pattern1")); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + TableModelUtils.insertData(database("test"), "test", 0, 100, senderEnv); - TableModelUtils.insertData("test1", "test1", 0, 100, senderEnv); + TableModelUtils.insertData(database("test1"), "test1", 0, 100, senderEnv); - TableModelUtils.insertData("pattern", "pattern", 0, 100, senderEnv); + TableModelUtils.insertData(database("pattern"), "pattern", 0, 100, senderEnv); - TableModelUtils.insertData("pattern1", "pattern1", 0, 100, senderEnv); + TableModelUtils.insertData(database("pattern1"), "pattern1", 0, 100, senderEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); final Map connectorAttributes = new HashMap<>(); - extractorAttributes.put("extractor.database-name", "pattern.*"); + extractorAttributes.put("extractor.database-name", database("pattern.*")); extractorAttributes.put("extractor.table-name", "test.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); @@ -487,13 +477,13 @@ public void testIoTDBPatternWithDataBaseAndTable() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + TableModelUtils.insertData(database("test"), "test", 100, 200, senderEnv); - TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); + TableModelUtils.insertData(database("test1"), "test1", 100, 200, senderEnv); - TableModelUtils.insertData("pattern", "pattern", 100, 200, senderEnv); + TableModelUtils.insertData(database("pattern"), "pattern", 100, 200, senderEnv); - TableModelUtils.insertData("pattern1", "pattern1", 100, 200, senderEnv); + TableModelUtils.insertData(database("pattern1"), "pattern1", 100, 200, senderEnv); TestUtils.assertDataEventuallyOnEnv( receiverEnv, @@ -503,4 +493,9 @@ public void testIoTDBPatternWithDataBaseAndTable() throws Exception { (String) null); } } + + private String database(final String name) { + // Database deletion is asynchronous; never reuse another method's database incarnation. + return name + "_" + testName.getMethodName().toLowerCase(Locale.ROOT); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java index b86646de0d938..5987157bcaeb0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java @@ -24,42 +24,51 @@ import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.PipeTestUtils.RetryFlushHandler; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; -import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; +import java.time.Duration; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; import java.util.function.Consumer; -import static org.junit.Assert.fail; - @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeAutoConflictIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeAutoConflictIT extends AbstractPipeTableModelSharedClusterIT { + + @Rule public final TestName testName = new TestName(); - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); + private String databaseName() { + return "conflict_" + Integer.toUnsignedString(testName.getMethodName().hashCode(), 16); + } + + @Override + protected void setupConfig() { senderEnv .getConfig() .getCommonConfig() .setAutoCreateSchemaEnabled(true) .setDatanodeMemoryProportion("3:3:1:1:1:0") + // Keep multi-region completion/conflict coverage without CPU-based tiny-file expansion. + .setDataRegionGroupExtensionPolicy("CUSTOM") + .setDefaultDataRegionGroupNumPerDatabase(3) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) @@ -71,38 +80,39 @@ public void setUp() { .getCommonConfig() .setAutoCreateSchemaEnabled(true) .setDatanodeMemoryProportion("3:3:1:1:1:0") + // Keep multi-region completion/conflict coverage without CPU-based tiny-file expansion. + .setDataRegionGroupExtensionPolicy("CUSTOM") + .setDefaultDataRegionGroupNumPerDatabase(3) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) .setDnConnectionTimeoutMs(600000) .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); } + // Two IOT-consensus clusters must merge all 600 rows across bidirectional pipes, including + // the 200 rows written after both clusters restart. Recreating receiver schema must not block + // sync. @Test public void testDoubleLivingAutoConflict() throws Exception { + final long testBodyStartedNanos = System.nanoTime(); + long phaseStartedNanos = testBodyStartedNanos; // Double living is two clusters each with a pipe connecting to the other. final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - createDataBaseAndTable(senderEnv); createDataBaseAndTable(receiverEnv); + phaseStartedNanos = logPhaseDuration("create databases and tables", phaseStartedNanos); final String senderIp = senderDataNode.getIp(); final int senderPort = senderDataNode.getPort(); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - insertData("test", "test1", 0, 100, senderEnv); + insertData(databaseName(), "test1", 0, 100, senderEnv); + phaseStartedNanos = logPhaseDuration("insert initial sender rows [0, 100)", phaseStartedNanos); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { @@ -113,10 +123,11 @@ public void testDoubleLivingAutoConflict() throws Exception { extractorAttributes.put("source.inclusion", "data.insert"); extractorAttributes.put("source.inclusion.exclusion", ""); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", databaseName()); extractorAttributes.put("__system.sql-dialect", "table"); // Add this property to avoid making self cycle. extractorAttributes.put("source.forwarding-pipe-requests", "false"); - extractorAttributes.put("user", "root"); + extractorAttributes.put("user", SessionConfig.DEFAULT_USER); connectorAttributes.put("sink", "iotdb-thrift-sink"); connectorAttributes.put("sink.batch.enable", "false"); @@ -133,9 +144,12 @@ public void testDoubleLivingAutoConflict() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); } + phaseStartedNanos = + logPhaseDuration("create and start sender-to-receiver pipe", phaseStartedNanos); - insertData("test", "test1", 100, 200, senderEnv); - insertData("test", "test1", 200, 300, receiverEnv); + insertData(databaseName(), "test1", 100, 200, senderEnv); + insertData(databaseName(), "test1", 200, 300, receiverEnv); + phaseStartedNanos = logPhaseDuration("insert bidirectional rows [100, 300)", phaseStartedNanos); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { @@ -146,10 +160,11 @@ public void testDoubleLivingAutoConflict() throws Exception { extractorAttributes.put("source.inclusion", "data.insert"); extractorAttributes.put("source.inclusion.exclusion", ""); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", databaseName()); extractorAttributes.put("__system.sql-dialect", "table"); // Add this property to avoid to make self cycle. extractorAttributes.put("source.forwarding-pipe-requests", "false"); - extractorAttributes.put("user", "root"); + extractorAttributes.put("user", SessionConfig.DEFAULT_USER); connectorAttributes.put("connector", "iotdb-thrift-connector"); connectorAttributes.put("connector.batch.enable", "false"); @@ -166,23 +181,92 @@ public void testDoubleLivingAutoConflict() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); } - insertData("test", "test1", 300, 400, receiverEnv); - - TableModelUtils.assertData("test", "test1", 0, 400, receiverEnv, handleFailure); - - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - fail(e.getMessage()); + insertData(databaseName(), "test1", 300, 400, receiverEnv); + phaseStartedNanos = + logPhaseDuration("create reverse pipe and insert rows [300, 400)", phaseStartedNanos); + + // Reuse retry Sessions within this phase and close them before either cluster restarts. + final long[] preRestartRetryCount = {0}; + final long[] preRestartFlushNanos = {0}; + try (final RetryFlushHandler handleFailure = new RetryFlushHandler(senderEnv, receiverEnv)) { + TableModelUtils.assertData( + databaseName(), + "test1", + 0, + 400, + receiverEnv, + failure -> { + preRestartRetryCount[0]++; + final long flushStartedNanos = System.nanoTime(); + handleFailure.accept(failure); + preRestartFlushNanos[0] += System.nanoTime() - flushStartedNanos; + }); + } + phaseStartedNanos = + logPhaseDuration("assert replicated rows [0, 400) before restart", phaseStartedNanos); + logRetryFlushMetrics("before restart", preRestartRetryCount[0], preRestartFlushNanos[0]); + + // Both restarts are part of the scenario; overlap them and propagate any recovery failure. + CompletableFuture.allOf( + CompletableFuture.runAsync( + () -> TestUtils.restartCluster(senderEnv, restartProbe("sender"))), + CompletableFuture.runAsync( + () -> TestUtils.restartCluster(receiverEnv, restartProbe("receiver")))) + .join(); + phaseStartedNanos = + logPhaseDuration("restart both clusters and await readiness", phaseStartedNanos); + + insertData(databaseName(), "test1", 400, 500, senderEnv); + insertData(databaseName(), "test1", 500, 600, receiverEnv); + phaseStartedNanos = logPhaseDuration("insert post-restart rows [400, 600)", phaseStartedNanos); + + // Recovery must use fresh clients; never carry the pre-restart Sessions into this phase. + final long[] postRestartRetryCount = {0}; + final long[] postRestartFlushNanos = {0}; + try (final RetryFlushHandler handleFailure = new RetryFlushHandler(senderEnv, receiverEnv)) { + TableModelUtils.assertData( + databaseName(), + "test1", + 0, + 600, + receiverEnv, + failure -> { + postRestartRetryCount[0]++; + final long flushStartedNanos = System.nanoTime(); + handleFailure.accept(failure); + postRestartFlushNanos[0] += System.nanoTime() - flushStartedNanos; + }); } + phaseStartedNanos = + logPhaseDuration("assert replicated rows [0, 600) after restart", phaseStartedNanos); + logRetryFlushMetrics("after restart", postRestartRetryCount[0], postRestartFlushNanos[0]); + logPhaseDuration("test body total", testBodyStartedNanos); + } + + private static long logPhaseDuration(final String phase, final long phaseStartedNanos) { + final long phaseEndedNanos = System.nanoTime(); + System.out.printf( + "[IT-PHASE] AutoConflict.%s duration=%.3f s%n", + phase, (phaseEndedNanos - phaseStartedNanos) / 1_000_000_000.0); + return System.nanoTime(); + } - insertData("test", "test1", 400, 500, senderEnv); - insertData("test", "test1", 500, 600, receiverEnv); + private static BiConsumer restartProbe(final String cluster) { + return (phase, duration) -> + System.out.printf( + "[IT-PHASE] AutoConflict.restart.%s.%s duration=%.3f s%n", + cluster, phase, duration.toNanos() / 1_000_000_000.0); + } - TableModelUtils.assertData("test", "test1", 0, 600, receiverEnv, handleFailure); + private static void logRetryFlushMetrics( + final String phase, final long retryCount, final long flushNanos) { + System.out.printf( + "[IT-PHASE] AutoConflict.%s retries=%d cumulativeFlush=%.3f s%n", + phase, retryCount, flushNanos / 1_000_000_000.0); } + // With test.* table filtering and opposite sink batching modes, each side must receive the + // other side's disjoint table ranges despite both clusters already containing the table schema. @Test public void testDoubleLivingAutoConflictTemplate() throws Exception { final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); @@ -204,7 +288,7 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { createDataBaseAndTable(senderEnv); createDataBaseAndTable(receiverEnv); - insertData("test", "test", 0, 100, senderEnv); + insertData(databaseName(), "test", 0, 100, senderEnv); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); final Map connectorAttributes = new HashMap<>(); @@ -213,9 +297,10 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { extractorAttributes.put("source.inclusion.exclusion", ""); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", databaseName()); extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("source.forwarding-pipe-requests", "false"); - extractorAttributes.put("user", "root"); + extractorAttributes.put("user", SessionConfig.DEFAULT_USER); connectorAttributes.put("connector", "iotdb-thrift-connector"); connectorAttributes.put("connector.batch.enable", "false"); @@ -233,8 +318,8 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); } - TableModelUtils.insertData("test", "test", 100, 200, senderEnv); - TableModelUtils.insertData("test", "test1", 200, 300, receiverEnv); + TableModelUtils.insertData(databaseName(), "test", 100, 200, senderEnv); + TableModelUtils.insertData(databaseName(), "test1", 200, 300, receiverEnv); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { @@ -246,9 +331,10 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { extractorAttributes.put("source.inclusion.exclusion", ""); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("database-name", databaseName()); extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("source.forwarding-pipe-requests", "false"); - extractorAttributes.put("user", "root"); + extractorAttributes.put("user", SessionConfig.DEFAULT_USER); connectorAttributes.put("connector", "iotdb-thrift-connector"); connectorAttributes.put("connector.batch.enable", "true"); @@ -265,14 +351,14 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); } - insertData("test", "test1", 300, 400, senderEnv); + insertData(databaseName(), "test1", 300, 400, senderEnv); TestUtils.assertDataEventuallyOnEnv( receiverEnv, TableModelUtils.getQuerySql("test"), TableModelUtils.generateHeaderResults(), TableModelUtils.generateExpectedResults(0, 200), - "test", + databaseName(), handleFailure); TestUtils.assertDataEventuallyOnEnv( @@ -280,13 +366,13 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { TableModelUtils.getQuerySql("test1"), TableModelUtils.generateHeaderResults(), TableModelUtils.generateExpectedResults(200, 400), - "test", + databaseName(), handleFailure); } private void createDataBaseAndTable(BaseEnv baseEnv) { - TableModelUtils.createDataBaseAndTable(baseEnv, "test", "test"); - TableModelUtils.createDataBaseAndTable(baseEnv, "test1", "test"); + TableModelUtils.createDataBaseAndTable(baseEnv, "test", databaseName()); + TableModelUtils.createDataBaseAndTable(baseEnv, "test1", databaseName()); } private void insertData( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java index 335a3a7e8e0a5..4bf2af31141bd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java @@ -33,14 +33,16 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.PipeTestUtils.RetryFlushHandler; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; -import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.sql.Connection; @@ -51,21 +53,21 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; import static org.awaitility.Awaitility.await; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeAutoDropIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeAutoDropIT extends AbstractPipeTableModelSharedClusterIT { - @Override - @Before - public void setUp() { - super.setUp(); + @Rule public final TestName testName = new TestName(); + + private String databaseName() { + return "autodrop_" + Integer.toUnsignedString(testName.getMethodName().hashCode(), 16); } + @Override protected void setupConfig() { // Enable auto split senderEnv @@ -73,6 +75,11 @@ protected void setupConfig() { .getCommonConfig() .setAutoCreateSchemaEnabled(true) .setDatanodeMemoryProportion("3:3:1:1:1:0") + // Keep multi-region completion/conflict coverage without CPU-based tiny-file expansion. + .setDataRegionGroupExtensionPolicy("CUSTOM") + .setDefaultDataRegionGroupNumPerDatabase(3) + // Auto-drop depends on completion reports; use a short bounded heartbeat in this IT. + .setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(1) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setEnforceStrongPassword(false) @@ -83,6 +90,11 @@ protected void setupConfig() { .getCommonConfig() .setAutoCreateSchemaEnabled(true) .setDatanodeMemoryProportion("3:3:1:1:1:0") + // Keep multi-region completion/conflict coverage without CPU-based tiny-file expansion. + .setDataRegionGroupExtensionPolicy("CUSTOM") + .setDefaultDataRegionGroupNumPerDatabase(3) + // Auto-drop depends on completion reports; use a short bounded heartbeat in this IT. + .setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(1) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setEnforceStrongPassword(false) @@ -96,16 +108,11 @@ protected void setupConfig() { senderEnv.getConfig().getConfigNodeConfig().setLeaderDistributionPolicy("HASH"); } + // A full-sync pipe must deliver 100 rows and automatically remove its finite history sub-pipe. @Test public void testAutoDropInHistoricalTransfer() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - // Create an ordinary full sync pipe final String sql = String.format("create pipe a2b ('node-urls'='%s')", receiverDataNode.getIpAndPortString()); @@ -116,24 +123,26 @@ public void testAutoDropInHistoricalTransfer() throws Exception { fail(e.getMessage()); } - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", databaseName()); + TableModelUtils.insertData(databaseName(), "test", 0, 100, senderEnv); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - TableModelUtils.getQueryCountSql("test"), - "_col0,", - Collections.singleton("100,"), - "test", - handleFailure); + try (final RetryFlushHandler handleFailure = new RetryFlushHandler(senderEnv, receiverEnv)) { + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + TableModelUtils.getQueryCountSql("test"), + "_col0,", + Collections.singleton("100,"), + databaseName(), + handleFailure); + } try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { // Fetch a fresh table-model snapshot on every poll while history pipes are being removed. await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(600, TimeUnit.SECONDS) .untilAsserted( () -> { @@ -155,24 +164,20 @@ public void testAutoDropInHistoricalTransfer() throws Exception { } } + // A snapshot restricted to timestamps 0..49 must deliver exactly 50 of 100 rows and then drop. @Test public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final RetryFlushHandler handleFailure = new RetryFlushHandler(senderEnv, receiverEnv)) { - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", databaseName()); + TableModelUtils.insertData(databaseName(), "test", 0, 100, senderEnv); final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -183,7 +188,8 @@ public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("start-time", "0"); sourceAttributes.put("end-time", "49"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); + sourceAttributes.put("database-name", databaseName()); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.batch.enable", "false"); @@ -205,7 +211,7 @@ public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { TableModelUtils.getQueryCountSql("test"), "_col0,", Collections.singleton("50,"), - "test", + databaseName(), handleFailure); } @@ -213,8 +219,8 @@ public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(600, TimeUnit.SECONDS) .untilAsserted( () -> { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java index 1e3961bd61a09..dd8f5526b7491 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java @@ -36,9 +36,11 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.env.AbstractEnv; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestLogger; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.PipeTestUtils; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; @@ -51,6 +53,7 @@ import org.junit.runner.RunWith; import java.io.IOException; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -70,8 +73,6 @@ @Category({MultiClusterIT2DualTableManualEnhanced.class}) public class IoTDBPipeClusterIT extends AbstractPipeTableModelDualManualIT { - private static final double SYNC_LAG_DELTA = 0.001; - @Override @Before public void setUp() { @@ -84,6 +85,9 @@ public void setUp() { .getCommonConfig() .setAutoCreateSchemaEnabled(true) .setDatanodeMemoryProportion("3:3:1:1:1:0") + // Keep AUTO region expansion tied to the test topology instead of the host CPU count. + // Dozens of tiny regions exhaust the IT nodes' WAL/load memory and obscure failover tests. + .setDataRegionPerDataNode(2) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) @@ -93,13 +97,17 @@ public void setUp() { senderEnv .getConfig() .getDataNodeConfig() - .setMetricReporterType(Collections.singletonList("PROMETHEUS")); + // The small test files need little schema-analysis memory. Avoid 4 MiB reservations per + // concurrent load competing for the 256 MiB IT node's much smaller query memory pool. + .setLoadTsFileAnalyzeSchemaMemorySizeInBytes(256 * 1024L) + .setMetricReporterType(Collections.singletonList("JMX")); receiverEnv .getConfig() .getCommonConfig() .setAutoCreateSchemaEnabled(true) .setDatanodeMemoryProportion("3:3:1:1:1:0") + .setDataRegionPerDataNode(2) .setDataReplicationFactor(2) .setSchemaReplicationFactor(3) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) @@ -111,10 +119,10 @@ public void setUp() { receiverEnv .getConfig() .getDataNodeConfig() - .setMetricReporterType(Collections.singletonList("PROMETHEUS")); + .setLoadTsFileAnalyzeSchemaMemorySizeInBytes(256 * 1024L) + .setMetricReporterType(Collections.singletonList("JMX")); - senderEnv.initClusterEnvironment(3, 3, 180); - receiverEnv.initClusterEnvironment(3, 3, 180); + executeOnBothEnvs(env -> env.initClusterEnvironment(3, 3, 180)); } @Test @@ -261,8 +269,10 @@ private void testWithAllParameters(final String realtimeMode) throws Exception { } } - // This function has a certain probability of triggering replica asynchrony. To ensure the success - // of the test, it will be retried 5 times. The test will be ignored after five retries. + // Stopping a DataRegion leader and restarting both clusters must preserve the 300 source rows; + // existing and newly created pipes must continue to apply the inclusive timestamp limit of 300. + // Read live replication gauges on every replica before flushing/restarting to avoid stale zeros. + // Retry replica-asynchrony exceptions up to five times, then report the final failure. @Test public void testPipeAfterDataRegionLeaderStop() { for (int retry = 0; retry < 5; retry++) { @@ -326,9 +336,8 @@ public void testPipeAfterDataRegionLeaderStop() { fail(); } - TableModelUtils.insertData("test", "test", 200, 300, senderEnv); - - TableModelUtils.insertData("test1", "test1", 200, 300, senderEnv); + insertDataAfterLeaderRestart("test"); + insertDataAfterLeaderRestart("test1"); TableModelUtils.assertData("test", "test", 0, 300, receiverEnv, handleFailure); flushTableDataRegionReplicasAfterReplicationComplete( @@ -337,13 +346,7 @@ public void testPipeAfterDataRegionLeaderStop() { receiverEnv, Collections.singletonList("test")); } - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } + executeOnBothEnvs(TestUtils::restartCluster); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { @@ -388,12 +391,44 @@ public void testPipeAfterDataRegionLeaderStop() { return; } catch (final Exception e) { if (retry < 4) { + IoTDBTestLogger.logger.warn( + "Retrying table pipe leader-stop scenario after attempt {}", retry + 1, e); this.tearDown(); + } else { + throw new AssertionError("Pipe replication failed after five leader-stop attempts", e); } } } } + private void insertDataAfterLeaderRestart(final String database) { + // A Running node can still have stale internal RPC connections on other DataNodes. Retry only + // those transport failures instead of rebuilding both clusters and replaying the whole case. + // These inserts use fixed timestamps and values, so retrying a partial batch only overwrites + // the same rows; flush is also safe to repeat. + await() + .pollInSameThread() + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .atMost(1, TimeUnit.MINUTES) + .untilAsserted( + () -> { + try { + TableModelUtils.insertData(database, database, 200, 300, senderEnv); + } catch (final RuntimeException e) { + if (e.getCause() instanceof SQLException sqlException + && sqlException.getMessage() != null + && (sqlException.getMessage().contains("Connection reset by peer") + || sqlException.getMessage().contains("Connection refused"))) { + throw new AssertionError( + "DataNode internal RPC connections have not recovered after leader restart", + e); + } + throw e; + } + }); + } + private int restartTableDataRegionLeader( final SyncConfigNodeIServiceClient client, final String database) throws TException { final List leaderRegionInfoList = @@ -454,7 +489,8 @@ private void waitForTableDataRegionReplicationComplete( // IoTConsensus accepts writes on any replica, including a previous leader during failover. // Check every replica's outgoing replication before flushing or restarting the cluster. await() - .pollInterval(500, TimeUnit.MILLISECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .atMost(2, TimeUnit.MINUTES) .untilAsserted( () -> { @@ -465,21 +501,7 @@ private void waitForTableDataRegionReplicationComplete( "No table DataRegion found for databases " + databases, regionInfoList.isEmpty()); - for (final TRegionInfo regionInfo : regionInfoList) { - final DataNodeWrapper replicaNode = - findDataNodeWrapperByPort(env, regionInfo.getClientRpcPort()); - final String metricsUrl = - "http://" - + replicaNode.getIp() - + ":" - + replicaNode.getMetricPort() - + "/metrics"; - final String metricsContent = env.getUrlContent(metricsUrl, null); - Assert.assertNotNull( - "Failed to fetch metrics from replica DataNode at " + metricsUrl, - metricsContent); - assertSyncLagIsZero(metricsContent, buildDataRegionTag(regionInfo), metricsUrl); - } + PipeTestUtils.assertDataRegionReplicationComplete(env, regionInfoList); } }); } @@ -508,40 +530,8 @@ private List showTableDataRegions( return showRegionResp.getRegionInfoList(); } - private DataNodeWrapper findDataNodeWrapperByPort(final BaseEnv env, final int port) { - for (final DataNodeWrapper dataNodeWrapper : env.getDataNodeWrapperList()) { - if (dataNodeWrapper.getPort() == port) { - return dataNodeWrapper; - } - } - fail("Failed to find DataNodeWrapper for client rpc port " + port); - return null; - } - - private String buildDataRegionTag(final TRegionInfo regionInfo) { - return "DataRegion[" + regionInfo.getConsensusGroupId().getId() + "]"; - } - - private void assertSyncLagIsZero( - final String metricsContent, final String dataRegionTag, final String metricsUrl) { - for (final String line : metricsContent.split("\\R")) { - if (!line.startsWith("iot_consensus{") - || !line.contains("type=\"syncLag\"") - || !line.contains("region=\"" + dataRegionTag + "\"")) { - continue; - } - final int lastSpaceIndex = line.lastIndexOf(' '); - Assert.assertTrue("Malformed syncLag metric line: " + line, lastSpaceIndex > 0); - Assert.assertEquals( - "Expected syncLag of " + dataRegionTag + " to be 0 at " + metricsUrl + " but got " + line, - 0.0, - Double.parseDouble(line.substring(lastSpaceIndex + 1)), - SYNC_LAG_DELTA); - return; - } - fail("No syncLag metric found for " + dataRegionTag + " at " + metricsUrl); - } - + // After adding a sender DataNode, writes through that node and a restart of both clusters must + // preserve replication; the existing and new pipe must deliver all 400 rows in their tables. @Test public void testPipeAfterRegisterNewDataNode() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -591,12 +581,7 @@ public void testPipeAfterRegisterNewDataNode() throws Exception { TableModelUtils.insertData("test1", "test1", 100, 200, senderEnv); - try { - senderEnv.registerNewDataNode(true); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } + senderEnv.registerNewDataNode(true); final DataNodeWrapper newDataNode = senderEnv.getDataNodeWrapper(senderEnv.getDataNodeWrapperList().size() - 1); @@ -605,13 +590,7 @@ public void testPipeAfterRegisterNewDataNode() throws Exception { TableModelUtils.assertData("test", "test", 0, 300, receiverEnv, handleFailure); } - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } + executeOnBothEnvs(TestUtils::restartCluster); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java index 8497bbdcf0f70..1f9a2880792c1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java @@ -30,14 +30,16 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.PipeTestUtils.RetryFlushHandler; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; -import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.sql.Connection; @@ -45,21 +47,28 @@ import java.sql.Statement; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeDoubleLivingIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeDoubleLivingIT extends AbstractPipeTableModelSharedClusterIT { - @Override - @Before - public void setUp() { - super.setUp(); + @Rule public final TestName testName = new TestName(); + + private String databaseName() { + return "double_" + testName.getMethodName().toLowerCase(Locale.ROOT); + } + + private String treeDatabasePath() { + return "root." + databaseName(); } + // Double-living accepts disabling either model but rejects forwarding that would create a cycle. @Test public void testDoubleLivingInvalidForwardingParameter() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -75,10 +84,6 @@ public void testDoubleLivingInvalidForwardingParameter() throws Exception { + " with sink (" + "'node-urls'='%s')", "p1", receiverDataNode.getIpAndPortString())); - } - - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { statement.execute( String.format( "create pipe %s" @@ -88,21 +93,21 @@ public void testDoubleLivingInvalidForwardingParameter() throws Exception { + " with sink (" + "'node-urls'='%s')", "p2", receiverDataNode.getIpAndPortString())); - } - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe %s" - + " with source (" - + "'forwarding-pipe-requests'='true'," - + "'mode.double-living'='true')" - + " with sink (" - + "'node-urls'='%s')", - "p3", receiverDataNode.getIpAndPortString())); - fail(); - } catch (final SQLException ignored) { + try { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'forwarding-pipe-requests'='true'," + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p3", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + // Only rejection of the conflicting CREATE is expected; connection failures must fail. + } } try (final SyncConfigNodeIServiceClient client = @@ -116,162 +121,107 @@ public void testDoubleLivingInvalidForwardingParameter() throws Exception { } } - // combination of - // org.apache.iotdb.pipe.it.tablemodel.autocreate.IoTDBPipeLifeCycleIT.testDoubleLiving and - // org.apache.iotdb.pipe.it.autocreate.IoTDBPipeLifeCycleIT.testDoubleLiving + // Both models must merge disjoint history/realtime ranges from both clusters, then continue + // replicating after both clusters restart. All 500 rows and both pipe directions remain checked. @Test - public void testBasicDoubleLiving() { - + public void testBasicDoubleLiving() throws Exception { final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - }; - - // insertion on sender - try (Connection conn = senderEnv.getConnection()) { - for (int i = 0; i < 100; ++i) { - TestUtils.executeNonQuery( - senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), conn); - } - } catch (SQLException e) { - throw new RuntimeException(e); + // Retain each JDBC session for writes and flushes until the explicit cluster restart. + try (final Connection senderConnection = senderEnv.getConnection(); + final Connection receiverConnection = receiverEnv.getConnection(); + final Statement sender = senderConnection.createStatement(); + final Statement receiver = receiverConnection.createStatement(); + final RetryFlushHandler handleFailure = new RetryFlushHandler(receiverEnv, senderEnv)) { + insertTreeData(sender, 0, 100); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", databaseName()); + TableModelUtils.insertData(databaseName(), "test", 0, 100, senderEnv); + sender.execute("flush"); + + createDoubleLivingPipe(senderEnv, "p1", receiverDataNode); + insertTreeData(sender, 100, 200); + insertTreeData(receiver, 200, 300); + TableModelUtils.insertData(databaseName(), "test", 100, 200, senderEnv); + TableModelUtils.assertData(databaseName(), "test", 0, 200, receiverEnv, handleFailure); + TableModelUtils.insertData(databaseName(), "test", 200, 300, receiverEnv); + sender.execute("flush"); + + createDoubleLivingPipe(receiverEnv, "p2", senderDataNode); + insertTreeData(receiver, 300, 400); + TableModelUtils.insertData(databaseName(), "test", 300, 400, receiverEnv); + receiver.execute("flush"); + assertDoubleLivingData(400, handleFailure); } - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - - TestUtils.executeNonQuery(senderEnv, "flush", null); - - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe %s" - + " with source (" - + "'database-name'='test'," - + "'table-name'='test'," - + "'path'='root.db.d1.s1'," - + "'mode.double-living'='true')" - + " with sink (" - + "'batch.enable'='false'," - + "'node-urls'='%s')", - "p1", receiverDataNode.getIpAndPortString())); - } catch (final SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); + // Independent restarts can overlap. Propagate a restart failure so recovery is always verified. + CompletableFuture.allOf( + CompletableFuture.runAsync(() -> TestUtils.restartCluster(senderEnv)), + CompletableFuture.runAsync(() -> TestUtils.restartCluster(receiverEnv))) + .join(); + + // The pre-restart handler is closed above. Recovery gets fresh retry Sessions on demand. + try (final Connection connection = receiverEnv.getConnection(); + final Statement receiver = connection.createStatement(); + final RetryFlushHandler handleFailure = new RetryFlushHandler(receiverEnv, senderEnv)) { + insertTreeData(receiver, 400, 500); + TableModelUtils.insertData(databaseName(), "test", 400, 500, receiverEnv); + receiver.execute("flush"); + assertDoubleLivingData(500, handleFailure); } + } - try (Connection conn = senderEnv.getConnection()) { - // insertion on sender - for (int i = 100; i < 200; ++i) { - TestUtils.executeNonQuery( - senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), conn); - } - } catch (SQLException e) { - throw new RuntimeException(e); - } - - try (Connection conn = receiverEnv.getConnection()) { - // insertion on receiver - for (int i = 200; i < 300; ++i) { - TestUtils.executeNonQuery( - receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), conn); - } - } catch (SQLException e) { - throw new RuntimeException(e); - } - - TableModelUtils.insertData("test", "test", 100, 200, senderEnv); - - TableModelUtils.assertData("test", "test", 0, 200, receiverEnv, handleFailure); - - TableModelUtils.insertData("test", "test", 200, 300, receiverEnv); - - TestUtils.executeNonQuery(senderEnv, "flush", null); - - try (final Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + private void createDoubleLivingPipe( + final BaseEnv env, final String pipeName, final DataNodeWrapper target) throws SQLException { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( String.format( "create pipe %s" + " with source (" - + "'database-name'='test'," + + "'database-name'='%s'," + "'table-name'='test'," - + "'path'='root.db.d1.s1'," + + "'path'='%s.d1.s1'," + "'mode.double-living'='true')" + " with sink (" + "'batch.enable'='false'," + "'node-urls'='%s')", - "p2", senderDataNode.getIpAndPortString())); - } catch (final SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); + pipeName, databaseName(), treeDatabasePath(), target.getIpAndPortString())); } + } - try (Connection conn = receiverEnv.getConnection()) { - // insertion on receiver - for (int i = 300; i < 400; ++i) { - TestUtils.executeNonQuery( - receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), conn); + private void insertTreeData(final Statement statement, final int start, final int end) + throws SQLException { + // Ten rows per INSERT reduce RPCs to one tenth without changing data or phase boundaries. + for (int batchStart = start; batchStart < end; batchStart += 10) { + final StringBuilder sql = + new StringBuilder("insert into " + treeDatabasePath() + ".d1(time, s1) values "); + for (int i = batchStart; i < Math.min(batchStart + 10, end); ++i) { + if (i > batchStart) { + sql.append(','); + } + sql.append('(').append(i).append(",1)"); } - } catch (SQLException e) { - throw new RuntimeException(e); + statement.execute(sql.toString()); } + } - TableModelUtils.insertData("test", "test", 300, 400, receiverEnv); - - TestUtils.executeNonQuery(receiverEnv, "flush", null); - - // check result + private void assertDoubleLivingData(final int end, final Consumer handleFailure) { final Set expectedResSet = new HashSet<>(); - for (int i = 0; i < 400; ++i) { + for (int i = 0; i < end; ++i) { expectedResSet.add(i + ",1.0,"); } - TestUtils.assertDataEventuallyOnEnv( - senderEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); - TableModelUtils.assertData("test", "test", 0, 400, senderEnv, handleFailure); - TableModelUtils.assertData("test", "test", 0, 400, receiverEnv, handleFailure); - - // restart cluster - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; + for (final BaseEnv env : new BaseEnv[] {senderEnv, receiverEnv}) { + TestUtils.assertDataEventuallyOnEnv( + env, + "select * from " + treeDatabasePath() + ".**", + "Time," + treeDatabasePath() + ".d1.s1,", + expectedResSet); + TableModelUtils.assertData(databaseName(), "test", 0, end, env, handleFailure); } - - try (Connection conn = receiverEnv.getConnection()) { - // insertion on receiver - for (int i = 400; i < 500; ++i) { - TestUtils.executeNonQuery( - receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), conn); - } - } catch (SQLException e) { - throw new RuntimeException(e); - } - - TableModelUtils.insertData("test", "test", 400, 500, receiverEnv); - - TestUtils.executeNonQuery(receiverEnv, "flush", null); - - // check result - for (int i = 400; i < 500; ++i) { - expectedResSet.add(i + ",1.0,"); - } - TestUtils.assertDataEventuallyOnEnv( - senderEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); - TableModelUtils.assertData("test", "test", 0, 500, senderEnv, handleFailure); - TableModelUtils.assertData("test", "test", 0, 500, receiverEnv, handleFailure); } + // Creating a double-living pipe exposes one pipe per dialect; dropping each via the opposite + // dialect must leave exactly one pipe visible per dialect. @Test public void testDoubleLivingIsolation() throws Exception { final String treePipeName = "treePipe"; @@ -279,10 +229,13 @@ public void testDoubleLivingIsolation() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - // Create tree pipe - try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute( + // Retain a real SQL session per dialect so all visibility checks still exercise both parsers. + try (final Connection treeConnection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Connection tableConnection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement treeStatement = treeConnection.createStatement(); + final Statement tableStatement = tableConnection.createStatement()) { + // Create tree pipe + treeStatement.execute( String.format( "create pipe %s" + " with source (" @@ -290,21 +243,14 @@ public void testDoubleLivingIsolation() throws Exception { + " with sink (" + "'node-urls'='%s')", treePipeName, receiverDataNode.getIpAndPortString())); - } catch (final SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - // Show tree pipe by tree session - Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(treeStatement)); - // Show table pipe by table session - Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(tableStatement)); - // Create table pipe - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute( + // Create table pipe + tableStatement.execute( String.format( "create pipe %s" + " with source (" @@ -312,34 +258,34 @@ public void testDoubleLivingIsolation() throws Exception { + " with sink (" + "'node-urls'='%s')", tablePipeName, receiverDataNode.getIpAndPortString())); - } catch (final SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - // Show tree pipe by tree session - Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); - - // Show table pipe by table session - Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + // Show tree pipe by tree session + Assert.assertEquals(2, TableModelUtils.showPipesCount(treeStatement)); + + // Show table pipe by table session + Assert.assertEquals(2, TableModelUtils.showPipesCount(tableStatement)); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Drop pipe + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + } - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - // Drop pipe - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) - .getCode()); + Assert.assertEquals(1, TableModelUtils.showPipesCount(treeStatement)); + Assert.assertEquals(1, TableModelUtils.showPipesCount(tableStatement)); } - - Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); - Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); } + // A double-living pipe creates separate tree/table entries with the same name, distinct + // creation times, and the correct dialect in each extractor configuration. @Test public void testDoubleLivingCreatesSameNamePipesWithSeparateDialects() throws Exception { final String pipeName = "double_living_same_name_pipe"; diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java index 170ef173565aa..5505c7fe68b37 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java @@ -28,48 +28,50 @@ import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; -import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeIdempotentIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeIdempotentIT extends AbstractPipeTableModelSharedClusterIT { - @Override - @Before - public void setUp() { - super.setUp(); - } + @Rule public final TestName testName = new TestName(); + // Creating a table already present on the receiver must not block later database events. @Test public void testCreateTableIdempotent() throws Exception { testTableConfigIdempotent(Collections.emptyList(), "create table test()"); } + // Creating a tree-backed view already present on the receiver must allow sync to continue. @Test public void testCreateTableViewIdempotent() throws Exception { testTableConfigIdempotent( Collections.emptyList(), "create view test(s1 int32 field) restrict as root.a.**"); } + // Adding the same tag to both copies of a table must not block subsequent database events. @Test public void testAlterTableAddColumnIdempotent() throws Exception { testTableConfigIdempotent( Collections.singletonList("create table test()"), "alter table test add column a tag"); } + // Adding an existing tag to the receiver's view must be treated as an idempotent conflict. @Test public void testAlterViewAddColumnIdempotent() throws Exception { testTableConfigIdempotent( @@ -77,6 +79,7 @@ public void testAlterViewAddColumnIdempotent() throws Exception { "alter view test add column a tag"); } + // Reapplying TTL=100 to a table must preserve pipe progress after the duplicate ALTER. @Test public void testAlterTableSetPropertiesIdempotent() throws Exception { testTableConfigIdempotent( @@ -84,6 +87,7 @@ public void testAlterTableSetPropertiesIdempotent() throws Exception { "alter table test set properties ttl=100"); } + // Reapplying TTL=100 to a view must preserve pipe progress after the duplicate ALTER. @Test public void testAlterViewSetPropertiesIdempotent() throws Exception { testTableConfigIdempotent( @@ -91,6 +95,7 @@ public void testAlterViewSetPropertiesIdempotent() throws Exception { "alter view test set properties ttl=100"); } + // Dropping an attribute already removed on the receiver must not block subsequent events. @Test public void testAlterTableDropColumnIdempotent() throws Exception { testTableConfigIdempotent( @@ -98,6 +103,7 @@ public void testAlterTableDropColumnIdempotent() throws Exception { "alter table test drop column b"); } + // Dropping a view field already removed on the receiver must allow sync to continue. @Test public void testAlterViewDropColumnIdempotent() throws Exception { testTableConfigIdempotent( @@ -105,11 +111,13 @@ public void testAlterViewDropColumnIdempotent() throws Exception { "alter view test drop column s1"); } + // Dropping a table already absent on the receiver must not stall later database creation. @Test public void testDropTableIdempotent() throws Exception { testTableConfigIdempotent(Collections.singletonList("create table test()"), "drop table test"); } + // Dropping a view already absent on the receiver must not stall later database creation. @Test public void testDropViewIdempotent() throws Exception { testTableConfigIdempotent( @@ -117,6 +125,7 @@ public void testDropViewIdempotent() throws Exception { "drop view test"); } + // Replaying a view-column rename from a to b must allow subsequent events to be applied. @Test public void testRenameViewColumnIdempotent() throws Exception { testTableConfigIdempotent( @@ -124,6 +133,7 @@ public void testRenameViewColumnIdempotent() throws Exception { "alter view test rename column a to b"); } + // Replaying a view rename from test to test1 must allow subsequent events to be applied. @Test public void testRenameViewIdempotent() throws Exception { testTableConfigIdempotent( @@ -131,79 +141,91 @@ public void testRenameViewIdempotent() throws Exception { "alter view test rename to test1"); } + // Creating the same user independently on both clusters must not block pipe progress. @Test public void testTableCreateUserIdempotent() throws Exception { - testTableConfigIdempotent(Collections.emptyList(), "create user newUser 'password123456'"); + testTableConfigIdempotent(Collections.emptyList(), "create user newUser 'IoTDB!Test2024'"); } + // Replaying deletion of a user already absent on the receiver must allow sync to continue. @Test public void testTableDropUserIdempotent() throws Exception { testTableConfigIdempotent( - Collections.singletonList("create user newUser 'password123456'"), "drop user newUser"); + Collections.singletonList("create user newUser 'IoTDB!Test2024'"), "drop user newUser"); } + // Creating the same role independently on both clusters must not block pipe progress. @Test public void testTableCreateRoleIdempotent() throws Exception { testTableConfigIdempotent(Collections.emptyList(), "create role newRole"); } + // Replaying deletion of a role already absent on the receiver must allow sync to continue. @Test public void testTableDropRoleIdempotent() throws Exception { testTableConfigIdempotent( Collections.singletonList("create role newRole"), "drop role newRole"); } + // Setting the same new password on both copies of a user must not block subsequent events. @Test public void testTableAlterUserIdempotent3() throws Exception { testTableConfigIdempotent( - Collections.singletonList("create user newUser 'password123456'"), - "alter user newUser set password 'passwd123456789'"); + Collections.singletonList("create user newUser 'IoTDB!Test2024'"), + "alter user newUser set password 'IoTDB!Alter2024'"); } + // Granting an existing role membership again must allow later database events to sync. @Test public void testTableGrantRoleToUserIdempotent() throws Exception { testTableConfigIdempotent( - Arrays.asList("create user newUser 'password123456'", "create role newRole"), + Arrays.asList("create user newUser 'IoTDB!Test2024'", "create role newRole"), "grant role newRole to newUser"); } + // Revoking an already removed role membership must allow later database events to sync. @Test public void testTableRevokeRoleFromUserIdempotent() throws Exception { testTableConfigIdempotent( Arrays.asList( - "create user newUser 'password123456'", + "create user newUser 'IoTDB!Test2024'", "create role newRole", "grant role newRole to newUser"), "revoke role newRole from newUser"); } + // Reapplying all privileges with grant option must not block subsequent database events. @Test public void testTableGrantIdempotent() throws Exception { testTableConfigIdempotent( - Collections.singletonList("create user newUser 'password123456'"), + Collections.singletonList("create user newUser 'IoTDB!Test2024'"), "grant all to user newUser with grant option"); } + // Revoking all privileges a second time must not block subsequent database events. @Test public void testTableRevokeIdempotent() throws Exception { testTableConfigIdempotent( Arrays.asList( - "create user newUser 'password123456'", "grant all to user newUser with grant option"), + "create user newUser 'IoTDB!Test2024'", "grant all to user newUser with grant option"), "revoke all from user newUser"); } + // Replaying an identical table comment must allow subsequent database events to sync. @Test public void testSetTableCommentIdempotent() throws Exception { testTableConfigIdempotent( Collections.singletonList("create table test(a tag)"), "COMMENT ON TABLE test is 'tag'"); } + // Replaying an identical tag-column comment must allow subsequent database events to sync. @Test public void testSetTableColumnCommentIdempotent() throws Exception { testTableConfigIdempotent( Collections.singletonList("create table test(a tag)"), "COMMENT ON COLUMN test.a IS 'tag'"); } + // Replaying an INT32-to-INT64 field conversion must not block subsequent database events. @Test public void testAlterColumnDataTypeIdempotent() throws Exception { testTableConfigIdempotent( @@ -214,7 +236,7 @@ public void testAlterColumnDataTypeIdempotent() throws Exception { private void testTableConfigIdempotent(final List beforeSqlList, final String testSql) throws Exception { - final String database = "test"; + final String database = database("test"); TableModelUtils.createDatabase(senderEnv, database); final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -259,17 +281,22 @@ private void testTableConfigIdempotent(final List beforeSqlList, final S // Pipe transfers config events in order. Use a database event as a progress marker to ensure // that all preparation statements have been applied on receiverEnv. - TableModelUtils.createDatabase(senderEnv, "test1"); - TableModelUtils.hasDataBase("test1", receiverEnv); + TableModelUtils.createDatabase(senderEnv, database("test1")); + TableModelUtils.hasDataBase(database("test1"), receiverEnv); TestUtils.executeNonQuery(database, BaseEnv.TABLE_SQL_DIALECT, receiverEnv, testSql, null); // Create an idempotent conflict TestUtils.executeNonQuery(database, BaseEnv.TABLE_SQL_DIALECT, senderEnv, testSql, null); - TableModelUtils.createDatabase(senderEnv, "test2"); + TableModelUtils.createDatabase(senderEnv, database("test2")); + + // The method-specific marker proves progress without counting other tests' databases. + TableModelUtils.hasDataBase(database("test2"), receiverEnv); + } - // Assume that the "database" is executed on receiverEnv - TestUtils.assertDataSizeEventuallyOnEnv(receiverEnv, "show databases", 4, null); + private String database(final String name) { + // Database deletion is asynchronous; never reuse another method's database incarnation. + return name + "_" + testName.getMethodName().toLowerCase(Locale.ROOT); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java index 5d0d59609e826..5af6f74c25f75 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java @@ -29,13 +29,14 @@ import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; -import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.sql.Connection; @@ -45,6 +46,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -53,14 +55,12 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeMetaIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeMetaIT extends AbstractPipeTableModelSharedClusterIT { - @Override - @Before - public void setUp() { - super.setUp(); - } + @Rule public final TestName testName = new TestName(); + // Sync matching table schema, device attributes and deletions; exclude unmatched + // tables/databases. @Test public void testTableSync() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -78,7 +78,7 @@ public void testTableSync() throws Exception { extractorAttributes.put("extractor.capture.tree", "false"); extractorAttributes.put("extractor.capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); - extractorAttributes.put("extractor.database-name", "test"); + extractorAttributes.put("extractor.database-name", database("test")); extractorAttributes.put("extractor.table-name", "t.*[0-9]"); extractorAttributes.put("user", "root"); @@ -94,7 +94,7 @@ public void testTableSync() throws Exception { Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - final String dbName = "test"; + final String dbName = database("test"); // Keep rows alive while asynchronous metadata and data synchronization finish. final long databaseTtl = TimeUnit.DAYS.toMillis(1); TableModelUtils.createDatabase(senderEnv, dbName, databaseTtl); @@ -112,11 +112,11 @@ public void testTableSync() throws Exception { "create table noTransferTable(a tag, b attribute, c int32) with (ttl=3000)"), null); - TableModelUtils.createDatabase(senderEnv, "noTransferDatabase", 300); + TableModelUtils.createDatabase(senderEnv, database("noTransferDatabase"), 300); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "show tables from test", + "show tables from " + database("test"), "TableName,TTL(ms),", Collections.singleton("table1," + databaseTtl + ","), dbName); @@ -160,7 +160,7 @@ public void testTableSync() throws Exception { // Will not include no-transfer table TestUtils.assertDataAlwaysOnEnv( receiverEnv, - "show tables from test", + "show tables from " + database("test"), "TableName,TTL(ms),", Collections.singleton("table1," + databaseTtl + ","), dbName); @@ -182,7 +182,7 @@ public void testTableSync() throws Exception { TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "show tables from test", + "show tables from " + database("test"), "TableName,TTL(ms),", Collections.emptySet(), dbName); @@ -203,6 +203,7 @@ public void testTableSync() throws Exception { } } + // A table-only pipe must keep the receiver's tree catalog empty after tree writes on the sender. @Test public void testNoTree() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -237,9 +238,9 @@ public void testNoTree() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "create database root.test", - "create timeSeries root.test.d1.s1 int32", - "insert into root.test.d1 (s1) values (1)")); + "create database " + treeDatabasePath(), + "create timeSeries " + treeDatabasePath() + ".d1.s1 int32", + "insert into " + treeDatabasePath() + ".d1 (s1) values (1)")); TestUtils.assertDataAlwaysOnEnv( receiverEnv, @@ -249,6 +250,7 @@ public void testNoTree() throws Exception { } } + // A tree-only pipe must keep the receiver's table catalog empty after table DDL on the sender. @Test public void testNoTable() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -283,7 +285,7 @@ public void testNoTable() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - final String dbName = "test"; + final String dbName = database("test"); TableModelUtils.createDatabase(senderEnv, dbName, 300); TestUtils.executeNonQueries( @@ -306,6 +308,7 @@ public void testNoTable() throws Exception { } } + // Sync historical and realtime grants for testUser, retaining only the selected table privileges. @Test public void testAuth() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -318,7 +321,8 @@ public void testAuth() throws Exception { TestUtils.executeNonQueries( senderEnv, - Arrays.asList("create user testUser 'password'", "grant all on root.** to user testUser"), + Arrays.asList( + "create user testUser 'IoTDB!Test2024'", "grant all on root.** to user testUser"), null); TestUtils.executeNonQueries( @@ -327,7 +331,7 @@ public void testAuth() throws Exception { senderEnv, Arrays.asList( "grant create on db.tb to user testUser", - "grant drop on database test to user testUser"), + "grant drop on database " + database("test") + " to user testUser"), null); final Map extractorAttributes = new HashMap<>(); @@ -338,7 +342,7 @@ public void testAuth() throws Exception { extractorAttributes.put("extractor.capture.tree", "false"); extractorAttributes.put("extractor.capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); - extractorAttributes.put("extractor.database-name", "test"); + extractorAttributes.put("extractor.database-name", database("test")); extractorAttributes.put("extractor.table-name", "t.*[0-9]"); extractorAttributes.put("user", "root"); @@ -374,11 +378,12 @@ public void testAuth() throws Exception { ",,SYSTEM,false,", ",,SECURITY,false,", ",*.*,ALTER,true,", - ",test.*,DROP,false,")), + "," + database("test") + ".*,DROP,false,")), "information_schema"); } } + // Each dialect must accept its own schema inclusion and reject the other model's schema path. @Test public void testValidation() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -425,13 +430,16 @@ public void testValidation() throws Exception { } } + // Merge complementary device attributes from a historical snapshot without losing existing + // values. @Test public void testAttributeSync() { + final String database = database("test"); TestUtils.executeNonQueries( receiverEnv, Arrays.asList( - "create database test", - "use test", + "create database " + database, + "use " + database, "create table table1(a tag, b attribute, c attribute, d int32)", "insert into table1 (time, a, b, c, d) values(1, 1, null, 1, 1), (2, 2, 2, null, 2)"), BaseEnv.TABLE_SQL_DIALECT); @@ -439,8 +447,8 @@ public void testAttributeSync() { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "create database test", - "use test", + "create database " + database, + "use " + database, "create table table1(a tag, b attribute, c attribute, d int32)", "insert into table1 (time, a, b, c, d) values(1, 1, 1, null, 1), (2, 2, null, 2, 2)", String.format( @@ -454,6 +462,14 @@ public void testAttributeSync() { "show devices from table1", "a,b,c,", new HashSet<>(Arrays.asList("1,1,1,", "2,2,2,")), - "test"); + database); + } + + private String database(final String name) { + return (name + "_" + testName.getMethodName()).toLowerCase(Locale.ROOT); + } + + private String treeDatabasePath() { + return "root.test_" + testName.getMethodName().toLowerCase(Locale.ROOT); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java index 5470b61c368cc..c55462718df52 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java @@ -23,26 +23,32 @@ import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeNullValueIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeNullValueIT extends AbstractPipeTableModelSharedClusterIT { + + @Rule public final TestName testName = new TestName(); @Override @Before @@ -63,90 +69,110 @@ private void testInsertNullValueTemplate( final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); + final String database = database("test"); final Consumer handleFailure = o -> { TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); }; - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + // Retain the history/realtime boundary and both flushes while sharing one Tablet Session. + // SQL scenarios keep using JDBC and do not open an unused Session. + try (final ITableSession session = + insertType == InsertType.SESSION_INSERT_TABLET + ? senderEnv.getTableSessionConnection() + : null) { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test", database); + + if (insertType == InsertType.SESSION_INSERT_TABLET) { + TableModelUtils.insertTablet(database, "test", 0, 200, session, true); + } else if (insertType == InsertType.SQL_INSERT) { + TableModelUtils.insertData(database, "test", 0, 200, senderEnv, true); + } + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); + extractorAttributes.put("realtime-mode", realtime); + extractorAttributes.put("user", "root"); + if (withParsing) { + extractorAttributes.put("start-time", "150"); + extractorAttributes.put("end-time", "249"); + extractorAttributes.put("database-name", database); + extractorAttributes.put("table-name", "test"); + } + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("test", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } if (insertType == InsertType.SESSION_INSERT_TABLET) { - TableModelUtils.insertTablet("test", "test", 0, 200, senderEnv, true); + TableModelUtils.insertTablet(database, "test", 200, 400, session, true); } else if (insertType == InsertType.SQL_INSERT) { - TableModelUtils.insertData("test", "test", 0, 200, senderEnv, true); + TableModelUtils.insertData(database, "test", 200, 400, senderEnv, true); } - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("__system.sql-dialect", "table"); - extractorAttributes.put("realtime-mode", realtime); - extractorAttributes.put("user", "root"); if (withParsing) { - extractorAttributes.put("start-time", "150"); - extractorAttributes.put("end-time", "249"); - extractorAttributes.put("database-name", "test"); - extractorAttributes.put("table-name", "test"); + TableModelUtils.assertCountData(database, "test", 100, receiverEnv, handleFailure); + return; } - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("test", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - } - - if (insertType == InsertType.SESSION_INSERT_TABLET) { - TableModelUtils.insertTablet("test", "test", 200, 400, senderEnv, true); - } else if (insertType == InsertType.SQL_INSERT) { - TableModelUtils.insertData("test", "test", 200, 400, senderEnv, true); + TableModelUtils.assertCountData(database, "test", 400, receiverEnv, handleFailure); } + } - if (withParsing) { - TableModelUtils.assertCountData("test", "test", 100, receiverEnv, handleFailure); - return; - } - TableModelUtils.assertCountData("test", "test", 400, receiverEnv, handleFailure); + private String database(final String name) { + return (name + "_" + testName.getMethodName()).toLowerCase(Locale.ROOT); } // ---------------------- // // Scenario 1: SQL Insert // // ---------------------- // + // SQL inserts containing null values in forced-log mode must replicate 100 rows in [150, 249]. @Test public void testSQLInsertWithParsingForcedLog() throws Exception { testInsertNullValueTemplate(InsertType.SQL_INSERT, true, "forced-log"); } + // SQL inserts containing null values in forced-log mode must replicate all 400 history/realtime + // rows. @Test public void testSQLInsertWithoutParsingForcedLog() throws Exception { testInsertNullValueTemplate(InsertType.SQL_INSERT, false, "forced-log"); } + // SQL inserts containing null values in file mode must replicate 100 rows in [150, 249]. @Test public void testSQLInsertWithParsingFile() throws Exception { testInsertNullValueTemplate(InsertType.SQL_INSERT, true, "file"); } + // SQL inserts containing null values in file mode must replicate all 400 history/realtime rows. @Test public void testSQLInsertWithoutParsingFile() throws Exception { testInsertNullValueTemplate(InsertType.SQL_INSERT, false, "file"); } + // SQL inserts containing null values in stream mode must replicate 100 rows in [150, 249]. @Test public void testSQLInsertWithParsingStream() throws Exception { testInsertNullValueTemplate(InsertType.SQL_INSERT, true, "stream"); } + // SQL inserts containing null values in stream mode must replicate all 400 history/realtime + // rows. @Test public void testSQLInsertWithoutParsingStream() throws Exception { testInsertNullValueTemplate(InsertType.SQL_INSERT, false, "stream"); @@ -155,31 +181,41 @@ public void testSQLInsertWithoutParsingStream() throws Exception { // --------------------------------- // // Scenario 2: Session Insert Tablet // // --------------------------------- // + // Session Tablets containing null values in forced-log mode must replicate 100 rows in [150, + // 249]. @Test public void testSessionInsertTabletWithParsingForcedLog() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, "forced-log"); } + // Session Tablets containing null values in forced-log mode must replicate all 400 + // history/realtime rows. @Test public void testSessionInsertTabletWithoutParsingForcedLog() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, "forced-log"); } + // Session Tablets containing null values in file mode must replicate 100 rows in [150, 249]. @Test public void testSessionInsertTabletWithParsingFile() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, "file"); } + // Session Tablets containing null values in file mode must replicate all 400 history/realtime + // rows. @Test public void testSessionInsertTabletWithoutParsingFile() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, "file"); } + // Session Tablets containing null values in stream mode must replicate 100 rows in [150, 249]. @Test public void testSessionInsertTabletWithParsingStream() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, "stream"); } + // Session Tablets containing null values in stream mode must replicate all 400 history/realtime + // rows. @Test public void testSessionInsertTabletWithoutParsingStream() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, "stream"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkCompressionIT.java index fe2b854863da7..6fa8cb172b647 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkCompressionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkCompressionIT.java @@ -27,20 +27,21 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionConfig; -import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; -import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.sql.Connection; @@ -50,6 +51,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Consumer; @@ -57,16 +59,16 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeSinkCompressionIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeSinkCompressionIT extends AbstractPipeTableModelSharedClusterIT { + + @Rule public final TestName testName = new TestName(); @Override - @Before - public void setUp() { + protected void setupConfig() { + // One LZMA2 encoder needs about 100 MB in addition to the node and in-flight callbacks. + // Give the sender headroom above the default 256 MB while bounding sink concurrency below. + senderEnv.getConfig().getDataNodeJVMConfig().setMaxHeapSize(512); // Override to enable air-gap - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - senderEnv .getConfig() .getCommonConfig() @@ -74,6 +76,10 @@ public void setUp() { .setDatanodeMemoryProportion("3:3:1:1:1:0") .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + // Compression cases write only a few rows per database. Keep one DataRegion per database + // so each case exercises compression without creating dozens of tiny TsFiles. + .setDataRegionGroupExtensionPolicy("CUSTOM") + .setDefaultDataRegionGroupNumPerDatabase(1) .setDnConnectionTimeoutMs(600000) .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false) @@ -87,40 +93,51 @@ public void setUp() { .setPipeAirGapReceiverEnabled(true) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionGroupExtensionPolicy("CUSTOM") + .setDefaultDataRegionGroupNumPerDatabase(1) .setDnConnectionTimeoutMs(600000) .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false) .setPipeAutoSplitFullEnabled(false); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); } + // The async thrift sink with stream extraction, batching, and Snappy must preserve tree and + // table data. @Test public void testCompression1() throws Exception { doTest("iotdb-thrift-connector", "stream", true, "snappy"); } + // The async thrift sink with batch extraction and Snappy/LZMA2 compression must preserve tree + // and table data. @Test public void testCompression2() throws Exception { doTest("iotdb-thrift-connector", "batch", true, "snappy, lzma2"); } + // The sync thrift sink with stream extraction and repeated Snappy compression must preserve + // tree and table data. @Test public void testCompression3() throws Exception { doTest("iotdb-thrift-sync-connector", "stream", false, "snappy, snappy"); } + // The sync thrift sink with batch extraction and gzip/Zstd compression must preserve tree and + // table data. @Test public void testCompression4() throws Exception { doTest("iotdb-thrift-sync-connector", "batch", true, "gzip, zstd"); } + // The air-gap sink with stream extraction and LZMA2/LZ4 compression must preserve tree and + // table data. @Test public void testCompression5() throws Exception { doTest("iotdb-air-gap-connector", "stream", false, "lzma2, lz4"); } + // The air-gap sink with batch extraction and LZMA2 compression must preserve tree and table + // data. @Test public void testCompression6() throws Exception { doTest("iotdb-air-gap-connector", "batch", true, "lzma2"); @@ -137,23 +154,20 @@ private void doTest( ? receiverDataNode.getPipeAirGapReceiverPort() : receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; + final String database = database("test"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test", database); + TableModelUtils.insertData(database, "test", 0, 50, senderEnv, true); - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.insertData("test", "test", 0, 50, senderEnv, true); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (final RetryFlushHandler handleFailure = new RetryFlushHandler(); + final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { TestUtils.executeNonQueries( senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s1) values (2010-01-01T10:00:00+08:00, 1)", - "insert into root.db.d1(time, s1) values (2010-01-02T10:00:00+08:00, 2)", - "flush"), + isolateDatabasePaths( + Arrays.asList( + "insert into root.db.d1(time, s1) values (2010-01-01T10:00:00+08:00, 1)", + "insert into root.db.d1(time, s1) values (2010-01-02T10:00:00+08:00, 2)", + "flush")), null); final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -165,16 +179,23 @@ private void doTest( extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("capture.tree", "true"); extractorAttributes.put("mode.double-living", "true"); - extractorAttributes.put("user", "root"); + extractorAttributes.put("extractor.database-name", database); + extractorAttributes.put("extractor.table-name", "test"); + // The default pattern format matches literal prefixes; a wildcard would exclude the data. + extractorAttributes.put("extractor.pattern", databasePath() + "."); + extractorAttributes.put("user", SessionConfig.DEFAULT_USER); processorAttributes.put("processor", "do-nothing-processor"); connectorAttributes.put("connector", connectorType); connectorAttributes.put("connector.batch.enable", useBatchMode ? "true" : "false"); + // LZMA2 allocates a large dictionary per task. Bound its peak memory for these functional + // round trips while retaining every codec and batching combination. + connectorAttributes.put("connector.parallel.tasks", "1"); connectorAttributes.put("connector.ip", receiverIp); connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.user", "root"); - connectorAttributes.put("connector.password", "root"); + connectorAttributes.put("connector.user", SessionConfig.DEFAULT_USER); + connectorAttributes.put("connector.password", SessionConfig.DEFAULT_PASSWORD); connectorAttributes.put("connector.compressor", compressionTypes); final TSStatus status = @@ -189,36 +210,39 @@ private void doTest( TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.s1),", + isolateDatabasePaths("select count(*) from root.db.**"), + isolateDatabasePaths("count(root.db.d1.s1),"), Collections.singleton("2,"), handleFailure); TestUtils.executeNonQueries( senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s1) values (3, 3)", - "insert into root.db.d1(time, s1) values (4, 4)", - "insert into root.db.d1(time, s1) values (5, 5)", - "insert into root.db.d1(time, s1) values (6, 6)", - "insert into root.db.d1(time, s1) values (7, 7)", - "insert into root.db.d1(time, s1) values (8, 8)", - "flush"), + isolateDatabasePaths( + Arrays.asList( + "insert into root.db.d1(time, s1) values (3, 3)", + "insert into root.db.d1(time, s1) values (4, 4)", + "insert into root.db.d1(time, s1) values (5, 5)", + "insert into root.db.d1(time, s1) values (6, 6)", + "insert into root.db.d1(time, s1) values (7, 7)", + "insert into root.db.d1(time, s1) values (8, 8)", + "flush")), null); - TableModelUtils.insertData("test", "test", 50, 100, senderEnv, true); + TableModelUtils.insertData(database, "test", 50, 100, senderEnv, true); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.s1),", + isolateDatabasePaths("select count(*) from root.db.**"), + isolateDatabasePaths("count(root.db.d1.s1),"), Collections.singleton("8,"), handleFailure); - TableModelUtils.assertCountData("test", "test", 100, receiverEnv, handleFailure); + TableModelUtils.assertCountData(database, "test", 100, receiverEnv, handleFailure); } } + // Accept Zstd levels -131072, 3, and 22, reject levels -131073 and 23 on the same connection, + // retain only the three valid pipes, and replicate data through a valid pipe. @Test public void testZstdCompressorLevel() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -226,113 +250,119 @@ public void testZstdCompressorLevel() throws Exception { final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (final RetryFlushHandler handleFailure = new RetryFlushHandler(); + final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { TestUtils.executeNonQueries( senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s1) values (1, 1)", - "insert into root.db.d1(time, s2) values (1, 1)", - "insert into root.db.d1(time, s3) values (1, 1)", - "insert into root.db.d1(time, s4) values (1, 1)", - "insert into root.db.d1(time, s5) values (1, 1)", - "flush"), + isolateDatabasePaths( + Arrays.asList( + "insert into root.db.d1(time, s1) values (1, 1)", + "insert into root.db.d1(time, s2) values (1, 1)", + "insert into root.db.d1(time, s3) values (1, 1)", + "insert into root.db.d1(time, s4) values (1, 1)", + "insert into root.db.d1(time, s5) values (1, 1)", + "flush")), null); - // Create 5 pipes with different zstd compression levels, p4 and p5 should fail. + // Reuse the SQL client for all five levels, including after a rejected CREATE PIPE. try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( String.format( "create pipe p1" - + " with extractor ('extractor.pattern'='root.db.d1.s1','table-name'='test1','capture.table'='true','capture.tree'='true','mode.double-living'='true')" + + " with extractor ('extractor.pattern'='" + + isolateDatabasePaths("root.db.d1.s1") + + "','extractor.database-name'='" + + database("test") + + "','table-name'='test1','capture.table'='true','capture.tree'='true','mode.double-living'='true')" + " with connector (" + "'connector.ip'='%s'," + "'connector.port'='%s'," + "'connector.compressor'='zstd, zstd'," + "'connector.compressor.zstd.level'='3')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } + receiverIp, + receiverPort)); - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { statement.execute( String.format( "create pipe p2" - + " with extractor ('extractor.pattern'='root.db.d1.s2','table-name'='test2','capture.table'='true','capture.tree'='true','mode.double-living'='true')" + + " with extractor ('extractor.pattern'='" + + isolateDatabasePaths("root.db.d1.s2") + + "','extractor.database-name'='" + + database("test") + + "','table-name'='test2','capture.table'='true','capture.tree'='true','mode.double-living'='true')" + " with connector (" + "'connector.ip'='%s'," + "'connector.port'='%s'," + "'connector.compressor'='zstd, zstd'," + "'connector.compressor.zstd.level'='22'," + "'connector.rate-limit-bytes-per-second'='2048.0')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } + receiverIp, + receiverPort)); - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { statement.execute( String.format( "create pipe p3" - + " with extractor ('extractor.pattern'='root.db.d1.s3','table-name'='test3','capture.table'='true','capture.tree'='true','mode.double-living'='true')" + + " with extractor ('extractor.pattern'='" + + isolateDatabasePaths("root.db.d1.s3") + + "','extractor.database-name'='" + + database("test") + + "','table-name'='test3','capture.table'='true','capture.tree'='true','mode.double-living'='true')" + " with connector (" + "'connector.ip'='%s'," + "'connector.port'='%s'," + "'connector.compressor'='zstd, zstd'," + "'connector.compressor.zstd.level'='-131072')", - receiverIp, receiverPort)); - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p4" - + " with extractor ('extractor.pattern'='root.db.d1.s4','table-name'='test4','capture.table'='true','capture.tree'='true','mode.double-living'='true')" - + " with connector (" - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.compressor'='zstd, zstd'," - + "'connector.compressor.zstd.level'='-131073')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException e) { - // Make sure the error message in IoTDBConnector.java is returned - Assert.assertTrue(e.getMessage().contains("Zstd compression level should be in the range")); - } - - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe p5" - + " with extractor ('extractor.pattern'='root.db.d1.s5','table-name'='test5','capture.table'='true','capture.tree'='true','mode.double-living'='true')" - + " with connector (" - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.compressor'='zstd, zstd'," - + "'connector.compressor.zstd.level'='23')", - receiverIp, receiverPort)); - fail(); - } catch (SQLException e) { - // Make sure the error message in IoTDBConnector.java is returned - Assert.assertTrue(e.getMessage().contains("Zstd compression level should be in the range")); + receiverIp, + receiverPort)); + + try { + statement.execute( + String.format( + "create pipe p4" + + " with extractor ('extractor.pattern'='" + + isolateDatabasePaths("root.db.d1.s4") + + "','extractor.database-name'='" + + database("test") + + "','table-name'='test4','capture.table'='true','capture.tree'='true','mode.double-living'='true')" + + " with connector (" + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.compressor'='zstd, zstd'," + + "'connector.compressor.zstd.level'='-131073')", + receiverIp, + receiverPort)); + fail(); + } catch (SQLException e) { + // Make sure the error message in IoTDBConnector.java is returned + Assert.assertTrue( + e.getMessage().contains("Zstd compression level should be in the range")); + } + + try { + statement.execute( + String.format( + "create pipe p5" + + " with extractor ('extractor.pattern'='" + + isolateDatabasePaths("root.db.d1.s5") + + "','extractor.database-name'='" + + database("test") + + "','table-name'='test5','capture.table'='true','capture.tree'='true','mode.double-living'='true')" + + " with connector (" + + "'connector.ip'='%s'," + + "'connector.port'='%s'," + + "'connector.compressor'='zstd, zstd'," + + "'connector.compressor.zstd.level'='23')", + receiverIp, + receiverPort)); + fail(); + } catch (SQLException e) { + // Make sure the error message in IoTDBConnector.java is returned + Assert.assertTrue( + e.getMessage().contains("Zstd compression level should be in the range")); + } } final List showPipeResult = @@ -346,16 +376,67 @@ public void testZstdCompressorLevel() throws Exception { .filter(info -> !info.id.startsWith(PipeStaticMeta.SYSTEM_PIPE_PREFIX)) .count()); - TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); - TableModelUtils.insertData("test", "test1", 0, 50, senderEnv, true); + TableModelUtils.createDataBaseAndTable(senderEnv, "test1", database("test")); + TableModelUtils.insertData(database("test"), "test1", 0, 50, senderEnv, true); + + TableModelUtils.createDataBaseAndTable(senderEnv, "test2", database("test")); + TableModelUtils.insertData(database("test"), "test2", 0, 50, senderEnv, true); - TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); - TableModelUtils.insertData("test", "test2", 0, 50, senderEnv, true); + TableModelUtils.createDataBaseAndTable(senderEnv, "test3", database("test")); + TableModelUtils.insertData(database("test"), "test3", 0, 50, senderEnv, true); - TableModelUtils.createDataBaseAndTable(senderEnv, "test3", "test"); - TableModelUtils.insertData("test", "test3", 0, 50, senderEnv, true); + TableModelUtils.assertCountData(database("test"), "test1", 50, receiverEnv, handleFailure); + } + } - TableModelUtils.assertCountData("test", "test1", 50, receiverEnv, handleFailure); + private final class RetryFlushHandler implements Consumer, AutoCloseable { + // Index 0 owns the sender Session and index 1 owns the receiver Session. Open lazily so a + // successful first assertion poll needs no retry clients. + private final ISession[] sessions = new ISession[2]; + + @Override + public void accept(final String ignored) { + // Assertions poll every 100 ms; reuse connections instead of opening multi-node JDBC + // connections for both environments on each failed poll. + for (int i = 0; i < sessions.length; i++) { + try { + if (sessions[i] == null) { + sessions[i] = (i == 0 ? senderEnv : receiverEnv).getSessionConnection(); + } + sessions[i].executeNonQueryStatement("flush"); + } catch (Exception e) { + throw new AssertionError(e); + } + } } + + @Override + public void close() throws Exception { + try { + if (sessions[0] != null) { + sessions[0].close(); + } + } finally { + if (sessions[1] != null) { + sessions[1].close(); + } + } + } + } + + private String database(final String name) { + return (name + "_" + testName.getMethodName()).toLowerCase(Locale.ROOT); + } + + private String databasePath() { + return "root.db_" + testName.getMethodName().toLowerCase(Locale.ROOT); + } + + private String isolateDatabasePaths(final String value) { + return value.replace("root.db", databasePath()); + } + + private List isolateDatabasePaths(final List values) { + return values.stream().map(this::isolateDatabasePaths).toList(); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkParallelIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkParallelIT.java index 6749d3f8aff25..9677ba7daaf3c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkParallelIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkParallelIT.java @@ -26,6 +26,8 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.PipeTestUtils.RetryFlushHandler; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; @@ -41,7 +43,6 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualEnhanced.class}) @@ -53,22 +54,30 @@ public void setUp() { super.setUp(); } + @Override + protected void setupConfig() { + super.setupConfig(); + // Retain multiple Regions for the three parallel tasks without generating many tiny files + // from CPU-based defaults or exhausting query memory on concurrent schema analysis. + for (final BaseEnv env : new BaseEnv[] {senderEnv, receiverEnv}) { + env.getConfig().getCommonConfig().setDataRegionPerDataNode(2); + env.getConfig().getDataNodeConfig().setLoadTsFileAnalyzeSchemaMemorySizeInBytes(256 * 1024L); + } + } + + // Three parallel batched tasks must deliver ten historical and ten realtime table rows plus four + // tree rows, with per-Region serialization disabled and double-living mode enabled. @Test public void testIoTConnectorParallel() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + TableModelUtils.insertData("test", "test", 0, 10, senderEnv); final Set expectedResSet = new HashSet<>(); try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final RetryFlushHandler handleFailure = new RetryFlushHandler(senderEnv, receiverEnv)) { final Map extractorAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); final Map connectorAttributes = new HashMap<>(); @@ -105,7 +114,7 @@ public void testIoTConnectorParallel() throws Exception { "insert into root.sg1.d1(time, s1) values (3, 4)", "flush"), null); - TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + TableModelUtils.insertData("test", "test", 10, 20, senderEnv); expectedResSet.add("0,1.0,"); expectedResSet.add("1,2.0,"); expectedResSet.add("2,3.0,"); @@ -117,7 +126,7 @@ public void testIoTConnectorParallel() throws Exception { expectedResSet, handleFailure); - TableModelUtils.assertCountData("test", "test", 200, receiverEnv, handleFailure); + TableModelUtils.assertCountData("test", "test", 20, receiverEnv, handleFailure); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java index 2da24a820e017..9c9f8996212f8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java @@ -20,13 +20,11 @@ package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; import org.apache.iotdb.commons.utils.function.CheckedTriConsumer; -import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.db.pipe.receiver.transform.converter.ValueConverter; import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; -import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; @@ -50,13 +48,10 @@ import java.time.LocalDate; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Random; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; @@ -73,8 +68,10 @@ public void setUp() { super.setUp(); } - private static final int generateDataSize = 1000; + // One hundred rows retain every type pair and all thirteen string samples with fewer repeats. + private static final int generateDataSize = 100; + // Convert a 100-row tablet across all 100 type pairs and check every received value and null. @Test public void insertTablet() { prepareTypeConversionTest( @@ -84,6 +81,7 @@ public void insertTablet() { false); } + // TsFile delivery should convert the same type pairs when receiver conversion is supported. @Test @Ignore("The receiver conversion is currently banned, will ignore conflict") public void insertTabletReceiveByTsFile() { @@ -105,7 +103,6 @@ private SessionDataSet query( } sql = sql + param + "time from " + tableName + " ORDER BY time ASC"; - session.executeNonQueryStatement("use test"); return session.executeQueryStatement(sql); } @@ -115,50 +112,42 @@ private void prepareTypeConversionTest( List> measurementSchemas = generateMeasurementSchemas(); Tablet tablet = generateTabletAndMeasurementSchema(measurementSchemas, "test"); - createDatabaseAndTable(measurementSchemas, true, tablet.getColumnTypes(), senderEnv); - createDatabaseAndTable(measurementSchemas, false, tablet.getColumnTypes(), receiverEnv); try (ITableSession senderSession = senderEnv.getTableSessionConnection(); ITableSession receiverSession = receiverEnv.getTableSessionConnection()) { - senderSession.executeNonQueryStatement("use test"); - receiverSession.executeNonQueryStatement("use test"); + // Reuse these sessions for DDL, both writes, pipe creation, flush and receiver queries. + createDatabaseAndTable(measurementSchemas, true, tablet.getColumnTypes(), senderSession); + createDatabaseAndTable(measurementSchemas, false, tablet.getColumnTypes(), receiverSession); if (isTsFile) { // Send TsFile data to receiver executeDataWriteOperation.accept(senderSession, receiverSession, tablet); senderSession.executeNonQueryStatement("flush"); - createDataPipe(true); + createDataPipe(senderSession, true); } else { // Send Tablet data to receiver // Write once to create data regions, guarantee that no any tsFiles will be sent executeDataWriteOperation.accept(senderSession, receiverSession, tablet); - createDataPipe(false); + createDataPipe(senderSession, false); // The actual implementation logic of inserting data executeDataWriteOperation.accept(senderSession, receiverSession, tablet); senderSession.executeNonQueryStatement("flush"); } - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - // Verify receiver data + // The sender was flushed above; bounded queries can observe delivery without flushing either + // cluster again or opening additional connections on every failed poll. long timeoutSeconds = 600; List> expectedValues = generateTabletResultSetForTable(tablet, measurementSchemas); await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(timeoutSeconds, TimeUnit.SECONDS) .untilAsserted( () -> { - try { - validateResultSet( - query(receiverSession, tablet.getSchemas(), tablet.getTableName()), - expectedValues, - tablet.getTimestamps()); - } catch (Exception | Error e) { - handleFailure.accept(e.getMessage()); + try (SessionDataSet dataSet = + query(receiverSession, tablet.getSchemas(), tablet.getTableName())) { + validateResultSet(dataSet, expectedValues, tablet.getTimestamps()); + } catch (Exception e) { fail(e.getMessage()); } }); @@ -172,7 +161,8 @@ private void createDatabaseAndTable( List> measurementSchemas, boolean isLeft, List categories, - BaseEnv env) { + ITableSession session) + throws IoTDBConnectionException, StatementExecutionException { StringBuilder builder = new StringBuilder(); for (int i = 0; i < measurementSchemas.size(); i++) { final MeasurementSchema measurement = @@ -185,15 +175,13 @@ private void createDatabaseAndTable( builder.deleteCharAt(builder.length() - 1); String tableCreation = String.format("create table if not exists test (%s)", builder.toString()); - TestUtils.executeNonQueries( - null, - "table", - env, - Arrays.asList("create database if not exists test", "use test", tableCreation), - null); + session.executeNonQueryStatement("create database if not exists test"); + session.executeNonQueryStatement("use test"); + session.executeNonQueryStatement(tableCreation); } - private void createDataPipe(boolean isTSFile) { + private void createDataPipe(ITableSession session, boolean isTSFile) + throws IoTDBConnectionException, StatementExecutionException { String sql = String.format( "create pipe test" @@ -205,8 +193,7 @@ private void createDataPipe(boolean isTSFile) { receiverEnv.getIP(), receiverEnv.getPort(), isTSFile ? "tsfile" : "tablet"); - TestUtils.executeNonQueries( - null, BaseEnv.TABLE_SQL_DIALECT, senderEnv, Collections.singletonList(sql), null); + session.executeNonQueryStatement(sql); } private void validateResultSet( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionSemanticIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionSemanticIT.java index 6704d37b87e2e..2ef5b44be75ca 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionSemanticIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionSemanticIT.java @@ -20,14 +20,12 @@ package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.TypeConversionSemanticCase; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelSharedClusterIT; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -35,10 +33,11 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.concurrent.CompletableFuture; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeTypeConversionSemanticIT extends AbstractPipeTableModelDualManualIT { +public class IoTDBPipeTypeConversionSemanticIT extends AbstractPipeTableModelSharedClusterIT { private static final String DATABASE = "pipe_type_conversion"; private static final String TABLE = "semantic_conversion"; @@ -56,17 +55,6 @@ public class IoTDBPipeTypeConversionSemanticIT extends AbstractPipeTableModelDua "int32_to_timestamp", "int32_to_date"); - @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - setupConfig(); - senderEnv.initClusterEnvironment(1, 1); - receiverEnv.initClusterEnvironment(1, 1); - } - @Override protected void setupConfig() { super.setupConfig(); @@ -82,6 +70,13 @@ protected void setupConfig() { .setSchemaReplicationFactor(1); } + @Override + protected CompletableFuture initializeClusters() { + return CompletableFuture.allOf( + CompletableFuture.runAsync(() -> senderEnv.initClusterEnvironment(1, 1)), + CompletableFuture.runAsync(() -> receiverEnv.initClusterEnvironment(1, 1))); + } + @Test public void testPipeReceiverTypeConversionSemantics() { createDatabaseAndTable(senderEnv, TABLE, TypeConversionSemanticCase.CASES, true); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java index 24e9b04d6ff0d..00d281e870fbf 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java @@ -32,7 +32,9 @@ import java.io.File; import java.util.Objects; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; public abstract class AbstractPipeDualTreeModelAutoIT { @@ -45,8 +47,16 @@ public void setUp() { senderEnv = MultiEnvFactory.getEnv(0); receiverEnv = MultiEnvFactory.getEnv(1); setupConfig(); - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); + executeOnBothEnvs(BaseEnv::initClusterEnvironment); + } + + protected void executeOnBothEnvs(final Consumer action) { + // The clusters own separate node directories and ports. Wait for both tasks, including when one + // fails, so teardown cannot race with a cluster that is still starting or restarting. + CompletableFuture.allOf( + CompletableFuture.runAsync(() -> action.accept(senderEnv)), + CompletableFuture.runAsync(() -> action.accept(receiverEnv))) + .join(); } protected void setupConfig() { @@ -81,15 +91,20 @@ protected void setupConfig() { @After public final void tearDown() { + cleanEnvironment(); + } + + protected void cleanEnvironment() { senderEnv.cleanClusterEnvironment(); receiverEnv.cleanClusterEnvironment(); } protected static void awaitUntilFlush(BaseEnv env) { + // Flush and check for persisted files immediately; an already-ready snapshot needs no delay. Awaitility.await() .atMost(1, TimeUnit.MINUTES) - .pollDelay(2, TimeUnit.SECONDS) - .pollInterval(2, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .until( () -> { TestUtils.executeNonQuery(env, "flush", null); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeTreeModelSharedClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeTreeModelSharedClusterIT.java new file mode 100644 index 0000000000000..dcbe9b606e310 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeTreeModelSharedClusterIT.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.auto; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.jdbc.IoTDBSQLException; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.ExternalResource; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.junit.Assert.assertEquals; + +/** + * Shares clusters for tree pipe/data scenarios with fixed configuration. Tests that change cluster + * topology, authorization, templates or other global state need their own lifecycle/reset logic. + */ +public abstract class AbstractPipeTreeModelSharedClusterIT extends AbstractPipeDualTreeModelAutoIT { + + private static BaseEnv sharedSender; + private static BaseEnv sharedReceiver; + private static CompletableFuture startup; + + @ClassRule + public static final ExternalResource CLUSTERS = + new ExternalResource() { + @Override + protected void after() { + // Always release both clusters, including a partially initialized or failed environment. + try { + if (sharedSender != null) { + sharedSender.cleanClusterEnvironment(); + } + } finally { + try { + if (sharedReceiver != null) { + sharedReceiver.cleanClusterEnvironment(); + } + } finally { + sharedSender = null; + sharedReceiver = null; + startup = null; + } + } + } + }; + + @Override + @Before + public void setUp() { + if (sharedSender == null) { + MultiEnvFactory.createEnv(2); + sharedSender = MultiEnvFactory.getEnv(0); + sharedReceiver = MultiEnvFactory.getEnv(1); + senderEnv = sharedSender; + receiverEnv = sharedReceiver; + setupConfig(); + // Await both independent starts before running any scenario or attempting cleanup. + startup = initializeClusters(); + } + senderEnv = sharedSender; + receiverEnv = sharedReceiver; + startup.join(); + } + + protected CompletableFuture initializeClusters() { + return CompletableFuture.allOf( + CompletableFuture.runAsync(senderEnv::initClusterEnvironment), + CompletableFuture.runAsync(receiverEnv::initClusterEnvironment)); + } + + @Override + protected void cleanEnvironment() { + if (startup == null || startup.isCompletedExceptionally()) { + return; + } + try { + // Stop replication on both sides before deleting data, so a live pipe cannot recreate it. + dropTreePipes(senderEnv); + dropTreePipes(receiverEnv); + deleteTreeDatabases(senderEnv); + deleteTreeDatabases(receiverEnv); + } catch (Exception e) { + throw new AssertionError("Failed to reset shared tree pipe test clusters", e); + } + } + + private static void dropTreePipes(final BaseEnv env) throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { + final TShowPipeResp response = + client.showPipe( + new TShowPipeReq().setIsTableModel(false).setUserName(SessionConfig.DEFAULT_USER)); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + for (final TShowPipeInfo pipe : response.getPipeInfoList()) { + if (!pipe.getId().startsWith(PipeStaticMeta.SYSTEM_PIPE_PREFIX)) { + final int statusCode = + client + .dropPipeExtended(new TDropPipeReq(pipe.getId()).setIsTableModel(false)) + .getCode(); + // An in-flight DROP PIPE can finish after the SHOW PIPE snapshot. + if (statusCode != TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode()) { + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), statusCode); + } + } + } + } + } + + private static void deleteTreeDatabases(final BaseEnv env) throws Exception { + try (final Connection connection = env.getConnection(); + final Statement statement = connection.createStatement()) { + final List databases = new ArrayList<>(); + try (final ResultSet result = statement.executeQuery("show databases")) { + while (result.next()) { + databases.add(result.getString(1)); + } + } + for (final String database : databases) { + try { + statement.execute("delete database " + database); + } catch (final IoTDBSQLException e) { + // An in-flight metadata DROP can finish after the SHOW DATABASES snapshot. + if (e.getErrorCode() != TSStatusCode.PATH_NOT_EXIST.getStatusCode()) { + throw e; + } + } + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java index 0821a75cf7b83..ec6168ca7fb2a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java @@ -27,7 +27,8 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.pipe.it.dual.PipeTestUtils; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.junit.Assert; import org.junit.Before; @@ -48,7 +49,7 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeAlterIT extends AbstractPipeDualTreeModelAutoIT { +public class IoTDBPipeAlterIT extends AbstractPipeTreeModelSharedClusterIT { @Override @Before @@ -59,12 +60,18 @@ public void setUp() { @Override protected void setupConfig() { super.setupConfig(); + senderEnv + .getConfig() + .getDataNodeConfig() + .setMetricReporterType(Collections.singletonList("JMX")); senderEnv .getConfig() .getCommonConfig() .setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(600); } + // MODIFY/REPLACE must preserve stopped/running state, advance creation time, and apply + // source/processor/sink changes, including empty-attribute resets. @Test public void testBasicAlterPipe() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -74,404 +81,366 @@ public void testBasicAlterPipe() throws Exception { String.format( "create pipe a2b with source ('source'='iotdb-source', 'source.pattern'='root.test1', 'source.realtime.mode'='stream') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", receiverDataNode.getIpAndPortString()); + // This sequence keeps the same cluster and dialect; retain both clients through every + // operation while preserving the metadata check after each change. try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + final Statement statement = connection.createStatement(); + final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { statement.execute(sql); - } catch (SQLException e) { - fail(e.getMessage()); - } - // show pipe - long lastCreationTime; - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test1")); - Assert.assertTrue( - showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); - Assert.assertTrue( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Record last creation time - lastCreationTime = showPipeResult.get(0).creationTime; - } - - // Stop pipe - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // show pipe + long lastCreationTime; + { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue( + showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test1")); + Assert.assertTrue( + showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Record last creation time + lastCreationTime = showPipeResult.get(0).creationTime; + } + + // Stop pipe statement.execute("stop pipe a2b"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("STOPPED", showPipeResult.get(0).state); - } - // Alter pipe (modify) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Show pipe + { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + } + + // Alter pipe (modify) statement.execute("alter pipe a2b modify source ('source.pattern'='root.test2')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("STOPPED", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("__system.sql-dialect=tree")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test2")); - Assert.assertTrue( - showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); - Assert.assertTrue( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - // Alter pipe (replace) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Show pipe + { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue( + showPipeResult.get(0).pipeExtractor.contains("__system.sql-dialect=tree")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue( + showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test2")); + Assert.assertTrue( + showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (replace) statement.execute( "alter pipe a2b replace source ('source'='iotdb-source', 'source.path'='root.test1.**')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("STOPPED", showPipeResult.get(0).state); - // check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test2")); - Assert.assertFalse( - showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); - Assert.assertTrue( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Alter pipe (modify) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Show pipe + { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + // check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue( + showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertFalse( + showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test2")); + Assert.assertFalse( + showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify) statement.execute("alter pipe a2b modify sink ('sink.batch.enable'='false')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("STOPPED", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=false")); - Assert.assertTrue( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - // Start pipe - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Show pipe + { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("STOPPED", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue( + showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=false")); + Assert.assertTrue( + showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Start pipe statement.execute("start pipe a2b"); - } catch (SQLException e) { - fail(e.getMessage()); - } - // Alter pipe (replace) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Alter pipe (replace) statement.execute( "alter pipe a2b replace processor ('processor'='tumbling-time-sampling-processor')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=false")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Alter pipe (modify) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Show pipe + { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue( + showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeProcessor + .contains("processor=tumbling-time-sampling-processor")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=false")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify) statement.execute("alter pipe a2b modify sink ('connector.batch.enable'='true')"); - } catch (SQLException e) { - fail(e.getMessage()); - } - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Alter pipe (modify empty) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Show pipe + { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue( + showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeProcessor + .contains("processor=tumbling-time-sampling-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify empty) statement.execute("alter pipe a2b modify source ()"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - // Alter pipe (replace empty) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // show pipe + { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue( + showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeProcessor + .contains("processor=tumbling-time-sampling-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (replace empty) statement.execute("alter pipe a2b replace source ()"); - } catch (SQLException e) { - fail(e.getMessage()); - } - // show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // check configurations - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - - // Alter pipe (replace empty) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // show pipe + { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertFalse( + showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeProcessor + .contains("processor=tumbling-time-sampling-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (replace empty) statement.execute("alter pipe a2b replace processor ()"); - } catch (SQLException e) { - fail(e.getMessage()); - } - - // show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // check configurations - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertFalse( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - lastCreationTime = showPipeResult.get(0).creationTime; - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } - // Alter pipe (modify empty) - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // show pipe + { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // check configurations + Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertFalse( + showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertFalse( + showPipeResult + .get(0) + .pipeProcessor + .contains("processor=tumbling-time-sampling-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + lastCreationTime = showPipeResult.get(0).creationTime; + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } + + // Alter pipe (modify empty) statement.execute("alter pipe a2b modify sink ()"); + + // Show pipe + { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, showPipeResult.size()); + // Check status + Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + // Check configurations + Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); + Assert.assertFalse( + showPipeResult + .get(0) + .pipeProcessor + .contains("processor=tumbling-time-sampling-processor")); + Assert.assertTrue( + showPipeResult + .get(0) + .pipeConnector + .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); + // Check creation time and record last creation time + Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); + // Check exception message + Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); + } } catch (SQLException e) { fail(e.getMessage()); } - - // Show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = - client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); - // Check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); - // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); - Assert.assertFalse( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); - Assert.assertTrue( - showPipeResult - .get(0) - .pipeConnector - .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); - // Check creation time and record last creation time - Assert.assertTrue(showPipeResult.get(0).creationTime > lastCreationTime); - // Check exception message - Assert.assertEquals("", showPipeResult.get(0).exceptionMessage); - } } + // ALTER must reject a missing pipe, and the same SQL connection must still be usable to create + // it. @Test public void testAlterPipeFailure() { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -483,18 +452,17 @@ public void testAlterPipeFailure() { receiverDataNode.getIpAndPortString()); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { - statement.execute(sql); - fail(); - } catch (SQLException ignore) { - } - - // Create pipe - sql = - String.format( - "create pipe a2b with sink ('node-urls'='%s', 'batch.enable'='false')", - receiverDataNode.getIpAndPortString()); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { + statement.execute(sql); + fail(); + } catch (SQLException ignore) { + } + + // Create pipe + sql = + String.format( + "create pipe a2b with sink ('node-urls'='%s', 'batch.enable'='false')", + receiverDataNode.getIpAndPortString()); statement.execute(sql); } catch (SQLException e) { fail(e.getMessage()); @@ -580,8 +548,10 @@ public void testAlterPipeSourceAndProcessor() { Collections.singleton("3,")); } + // With history disabled, completing the data barrier must leave historical series absent; + // enabling history through ALTER must then transfer the existing series. @Test - public void testAlterPipeRealtime() { + public void testAlterPipeRealtime() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); // Insert data on sender @@ -605,7 +575,8 @@ public void testAlterPipeRealtime() { fail(e.getMessage()); } - TestUtils.assertDataAlwaysOnEnv( + PipeTestUtils.awaitDataPipeCompletion(senderEnv, "a2b"); + TestUtils.assertDataEventuallyOnEnv( receiverEnv, "count timeSeries root.db.**", "count(timeseries),", @@ -625,8 +596,10 @@ public void testAlterPipeRealtime() { Collections.singleton("1,")); } + // After deleting already replicated rows on the receiver, enabling sink batching must deliver a + // new row without replaying either committed row, checked after the altered pipe has completed. @Test - public void testAlterPipeDoesNotResendCommittedData() { + public void testAlterPipeDoesNotResendCommittedData() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); TestUtils.executeNonQueries( @@ -644,6 +617,7 @@ public void testAlterPipeDoesNotResendCommittedData() { final Set oldData = new HashSet<>(Arrays.asList("1,1.0,", "2,2.0,")); TestUtils.assertDataEventuallyOnEnv( receiverEnv, "select * from root.db.d1", "Time,root.db.d1.s1,", oldData); + PipeTestUtils.awaitDataPipeCompletion(senderEnv, "a2b"); TestUtils.executeNonQuery( receiverEnv, "delete from root.db.d1.s1 where time >= 1 and time <= 2", null); @@ -652,15 +626,13 @@ public void testAlterPipeDoesNotResendCommittedData() { TestUtils.executeNonQuery( senderEnv, "alter pipe a2b modify sink ('sink.batch.enable'='true')", null); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, "select * from root.db.d1", "Time,root.db.d1.s1,", Collections.emptySet()); - + // Wait for the altered sink's committed barrier before checking the exact result, so pending + // batches cannot replay the deleted rows after the assertion. TestUtils.executeNonQueries( senderEnv, Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"), null); final Set newData = Collections.singleton("3,3.0,"); + PipeTestUtils.awaitDataPipeCompletion(senderEnv, "a2b"); TestUtils.assertDataEventuallyOnEnv( receiverEnv, "select * from root.db.d1", "Time,root.db.d1.s1,", newData); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, "select * from root.db.d1", "Time,root.db.d1.s1,", newData); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAutoSplitIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAutoSplitIT.java index c737ce32ec704..3011a6909379b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAutoSplitIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAutoSplitIT.java @@ -30,6 +30,7 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.junit.Assert; @@ -80,10 +81,11 @@ public void setUp() { .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false) .setPipeAutoSplitFullEnabled(true); // set auto split to true - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); + executeOnBothEnvs(BaseEnv::initClusterEnvironment); } + // Only full-sync pipes split; preserve sink options and replicate the row remaining after + // deletion. @Test public void testWriteBackSinkWithTargetDatabase() throws Exception { TestUtils.executeNonQueries( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java index 30cee019f660a..4314a98be0aae 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java @@ -22,11 +22,16 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.pipe.it.dual.PipeTestUtils; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.http.client.methods.HttpPost; @@ -41,36 +46,52 @@ import java.nio.charset.Charset; import java.sql.Connection; -import java.sql.ResultSet; import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import static org.apache.iotdb.db.it.IoTDBRestServiceIT.getHttpPost; import static org.awaitility.Awaitility.await; +/** Shares fixed-topology clusters while isolating each test method in distinct databases. */ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeDataSinkIT extends AbstractPipeDualTreeModelAutoIT { +public class IoTDBPipeDataSinkIT extends AbstractPipeTreeModelSharedClusterIT { + + private static final AtomicInteger TEST_SEQUENCE = new AtomicInteger(); + + private int testId; @Override @Before public void setUp() { super.setUp(); + // Database deletion is asynchronous; unique names prevent a prior region incarnation leaking + // rows into the next test while the shared clusters remain running. + testId = TEST_SEQUENCE.incrementAndGet(); } @Override protected void setupConfig() { super.setupConfig(); + senderEnv + .getConfig() + .getDataNodeConfig() + .setMetricReporterType(Collections.singletonList("JMX")); senderEnv.getConfig().getDataNodeConfig().setEnableRestService(true); senderEnv.getConfig().getCommonConfig().setPipeAutoSplitFullEnabled(true); } + // Realtime-only replication starts after timestamp 0 is written and must transfer timestamp 1. @Test public void testThriftSinkWithRealtimeFirstDisabled() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -85,7 +106,8 @@ public void testThriftSinkWithRealtimeFirstDisabled() throws Exception { // Because the failures will randomly generate due to resource limitation TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (0, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (0, 1)", "flush")), null); final Map sourceAttributes = new HashMap<>(); @@ -113,22 +135,26 @@ public void testThriftSinkWithRealtimeFirstDisabled() throws Exception { // Because the failures will randomly generate due to resource limitation TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush")), null); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.vehicle.**", - "Time,root.vehicle.d0.s1,", - Collections.unmodifiableSet(new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,")))); + isolateDatabasePaths("select * from root.vehicle.**"), + isolateDatabasePaths("Time,root.vehicle.d0.s1,"), + isolateDatabasePaths( + Collections.unmodifiableSet(new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,"))))); } } + // Tablet sink format transfers realtime points across initial and recreated pipe instances. @Test public void testSinkTabletFormat() throws Exception { testSinkFormat("tablet"); } + // TsFile sink format transfers realtime points across initial and recreated pipe instances. @Test public void testSinkTsFileFormat() throws Exception { testSinkFormat("tsfile"); @@ -147,7 +173,8 @@ private void testSinkFormat(final String format) throws Exception { // Because the failures will randomly generate due to resource limitation TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush")), null); final Map sourceAttributes = new HashMap<>(); @@ -177,14 +204,16 @@ private void testSinkFormat(final String format) throws Exception { // Because the failures will randomly generate due to resource limitation TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush")), null); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.vehicle.**", - "Time,root.vehicle.d0.s1,", - Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,")))); + isolateDatabasePaths("select * from root.vehicle.**"), + isolateDatabasePaths("Time,root.vehicle.d0.s1,"), + isolateDatabasePaths( + Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,"))))); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("testPipe").getCode()); @@ -202,21 +231,25 @@ private void testSinkFormat(final String format) throws Exception { // Because the failures will randomly generate due to resource limitation TestUtils.executeNonQueries( senderEnv, - Arrays.asList( - "insert into root.vehicle.d0(time, s1) values (4, 1)", - "insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)", - "flush"), + isolateDatabasePaths( + Arrays.asList( + "insert into root.vehicle.d0(time, s1) values (4, 1)", + "insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)", + "flush")), null); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.vehicle.**", - "Time,root.vehicle.d0.s1,", - Collections.unmodifiableSet( - new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,")))); + isolateDatabasePaths("select * from root.vehicle.**"), + isolateDatabasePaths("Time,root.vehicle.d0.s1,"), + isolateDatabasePaths( + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,"))))); } } + // The legacy sink must transfer the first row in its own database even when earlier tests left + // asynchronous regions pending cleanup. @Test public void testLegacySink() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -238,17 +271,19 @@ public void testLegacySink() throws Exception { // Because the failures will randomly generate due to resource limitation TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (0, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (0, 1)", "flush")), null); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.vehicle.**", - "Time,root.vehicle.d0.s1,", - Collections.singleton("0,1.0,")); + isolateDatabasePaths("select * from root.vehicle.**"), + isolateDatabasePaths("Time,root.vehicle.d0.s1,"), + isolateDatabasePaths(Collections.singleton("0,1.0,"))); } } + // A history and realtime log pipe must create receiver series from source DDL and inserts. @Test public void testReceiverAutoCreateByLog() throws Exception { testReceiverAutoCreate( @@ -260,6 +295,7 @@ public void testReceiverAutoCreateByLog() throws Exception { }); } + // A history and realtime file pipe must create receiver series from source DDL and inserts. @Test public void testReceiverAutoCreateByFile() throws Exception { testReceiverAutoCreate( @@ -271,13 +307,14 @@ public void testReceiverAutoCreateByFile() throws Exception { }); } + // A source path pattern limits receiver auto-creation to the two matching devices. @Test public void testReceiverAutoCreateWithPattern() throws Exception { testReceiverAutoCreate( new HashMap() { { put("source.realtime.mode", "batch"); - put("source.path", "root.ln.wf01.wt0*.*"); + put("source.path", isolateDatabasePaths("root.ln.wf01.wt0*.*")); put("user", "root"); } }); @@ -317,57 +354,61 @@ private void testReceiverAutoCreate(final Map sourceAttributes) // Because the failures will randomly generate due to resource limitation TestUtils.executeNonQueries( senderEnv, - Arrays.asList( - "create timeSeries root.ln.wf01.wt01.boolean boolean", - "create timeSeries root.ln.wf01.wt01.int32 int32", - "create timeSeries root.ln.wf01.wt01.int64 int64", - "create timeSeries root.ln.wf01.wt01.float float", - "create timeSeries root.ln.wf01.wt01.double double", - "create timeSeries root.ln.wf01.wt01.time_stamp timestamp", - "create timeSeries root.ln.wf01.wt01.date date", - "create timeSeries root.ln.wf01.wt01.text text", - "create timeSeries root.ln.wf01.wt01.string string", - "create timeSeries root.ln.wf01.wt01.blob blob", - "create aligned timeSeries root.ln.wf01.wt02(int32 int32, boolean boolean)", - "insert into root.ln.wf01.wt01(time, boolean, int32, int64, float, double, time_stamp, date, text, string, blob) values (20000, false, 123, 321, 13.3, 14.4, now(), '2000-12-13', 'abc', 'def', X'f103')", - "insert into root.ln.wf01.wt02(time, int32, boolean) values (20000, 123, false)", - // For pattern parse - "insert into root.ln.wf01.wt11(time, redundant_data) values (20000, -1)", - "flush"), + isolateDatabasePaths( + Arrays.asList( + "create timeSeries root.ln.wf01.wt01.boolean boolean", + "create timeSeries root.ln.wf01.wt01.int32 int32", + "create timeSeries root.ln.wf01.wt01.int64 int64", + "create timeSeries root.ln.wf01.wt01.float float", + "create timeSeries root.ln.wf01.wt01.double double", + "create timeSeries root.ln.wf01.wt01.time_stamp timestamp", + "create timeSeries root.ln.wf01.wt01.date date", + "create timeSeries root.ln.wf01.wt01.text text", + "create timeSeries root.ln.wf01.wt01.string string", + "create timeSeries root.ln.wf01.wt01.blob blob", + "create aligned timeSeries root.ln.wf01.wt02(int32 int32, boolean boolean)", + "insert into root.ln.wf01.wt01(time, boolean, int32, int64, float, double, time_stamp, date, text, string, blob) values (20000, false, 123, 321, 13.3, 14.4, now(), '2000-12-13', 'abc', 'def', X'f103')", + "insert into root.ln.wf01.wt02(time, int32, boolean) values (20000, 123, false)", + // For pattern parse + "insert into root.ln.wf01.wt11(time, redundant_data) values (20000, -1)", + "flush")), null); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "show timeSeries root.ln.wf01.wt01.*", + isolateDatabasePaths("show timeSeries root.ln.wf01.wt01.*"), "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", - Collections.unmodifiableSet( - new HashSet<>( - Arrays.asList( - "root.ln.wf01.wt01.boolean,null,root.ln,BOOLEAN,RLE,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.int32,null,root.ln,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.int64,null,root.ln,INT64,TS_2DIFF,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.float,null,root.ln,FLOAT,GORILLA,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.double,null,root.ln,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.time_stamp,null,root.ln,TIMESTAMP,TS_2DIFF,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.date,null,root.ln,DATE,TS_2DIFF,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.text,null,root.ln,TEXT,PLAIN,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.string,null,root.ln,STRING,PLAIN,LZ4,null,null,null,null,BASE,", - "root.ln.wf01.wt01.blob,null,root.ln,BLOB,PLAIN,LZ4,null,null,null,null,BASE,"))), + isolateDatabasePaths( + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + "root.ln.wf01.wt01.boolean,null,root.ln,BOOLEAN,RLE,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.int32,null,root.ln,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.int64,null,root.ln,INT64,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.float,null,root.ln,FLOAT,GORILLA,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.double,null,root.ln,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.time_stamp,null,root.ln,TIMESTAMP,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.date,null,root.ln,DATE,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.text,null,root.ln,TEXT,PLAIN,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.string,null,root.ln,STRING,PLAIN,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.blob,null,root.ln,BLOB,PLAIN,LZ4,null,null,null,null,BASE,")))), handleFailure); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "show devices root.ln.wf01.wt02", + isolateDatabasePaths("show devices root.ln.wf01.wt02"), "Device,IsAligned,Template,TTL(ms),", - Collections.singleton("root.ln.wf01.wt02,true,null,INF,"), + isolateDatabasePaths(Collections.singleton("root.ln.wf01.wt02,true,null,INF,")), handleFailure); } } + // Synchronous receiver loading preserves the same five timestamps across repeated pipe starts. @Test public void testSyncLoadTsFile() throws Exception { testReceiverLoadTsFile("sync"); } + // Asynchronous receiver loading preserves the same five timestamps across repeated pipe starts. @Test public void testAsyncLoadTsFile() throws Exception { testReceiverLoadTsFile("async"); @@ -386,7 +427,8 @@ private void testReceiverLoadTsFile(final String loadTsFileStrategy) throws Exce // Because the failures will randomly generate due to resource limitation TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush")), null); final Map sourceAttributes = new HashMap<>(); @@ -415,14 +457,16 @@ private void testReceiverLoadTsFile(final String loadTsFileStrategy) throws Exce // Because the failures will randomly generate due to resource limitation TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush")), null); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.vehicle.**", - "Time,root.vehicle.d0.s1,", - Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,")))); + isolateDatabasePaths("select * from root.vehicle.**"), + isolateDatabasePaths("Time,root.vehicle.d0.s1,"), + isolateDatabasePaths( + Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,"))))); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("testPipe").getCode()); @@ -440,26 +484,30 @@ private void testReceiverLoadTsFile(final String loadTsFileStrategy) throws Exce // Because the failures will randomly generate due to resource limitation TestUtils.executeNonQueries( senderEnv, - Arrays.asList( - "insert into root.vehicle.d0(time, s1) values (4, 1)", - "insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)", - "flush"), + isolateDatabasePaths( + Arrays.asList( + "insert into root.vehicle.d0(time, s1) values (4, 1)", + "insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)", + "flush")), null); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.vehicle.**", - "Time,root.vehicle.d0.s1,", - Collections.unmodifiableSet( - new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,")))); + isolateDatabasePaths("select * from root.vehicle.**"), + isolateDatabasePaths("Time,root.vehicle.d0.s1,"), + isolateDatabasePaths( + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,"))))); } } + // Synchronous load without checksum verification transfers data after the source flushes. @Test public void testSyncLoadTsFileWithoutVerify() throws Exception { testLoadTsFileWithoutVerify("sync"); } + // Asynchronous load without checksum verification transfers data after the source flushes. @Test public void testAsyncLoadTsFileWithoutVerify() throws Exception { testLoadTsFileWithoutVerify("async"); @@ -478,7 +526,8 @@ private void testLoadTsFileWithoutVerify(final String loadTsFileStrategy) throws // Because the failures will randomly generate due to resource limitation TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush")), null); final Map sourceAttributes = new HashMap<>(); @@ -506,17 +555,20 @@ private void testLoadTsFileWithoutVerify(final String loadTsFileStrategy) throws TestUtils.executeNonQueries( senderEnv, - Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush"), + isolateDatabasePaths( + Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush")), null); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.vehicle.**", - "Time,root.vehicle.d0.s1,", - Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,")))); + isolateDatabasePaths("select * from root.vehicle.**"), + isolateDatabasePaths("Time,root.vehicle.d0.s1,"), + isolateDatabasePaths( + Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,"))))); } } + // Partial REST inserts with duplicate timestamps remain visible only in this test's databases. @Test public void testSpecialPartialInsert() throws Exception { try (final Connection connection = senderEnv.getConnection(); @@ -537,7 +589,8 @@ public void testSpecialPartialInsert() throws Exception { + senderEnv.getDataNodeWrapper(0).getRestServicePort() + "/rest/v2/insertRecords"); String json = - "{\"timestamps\":[1635232113960,1635232151960,1635232143960,1635232143960],\"measurements_list\":[[\"s33\",\"s44\"],[\"s55\",\"s66\"],[\"s77\",\"s88\"],[\"s771\",\"s881\"]],\"data_types_list\":[[\"INT32\",\"INT64\"],[\"FLOAT\",\"DOUBLE\"],[\"FLOAT\",\"DOUBLE\"],[\"BOOLEAN\",\"TEXT\"]],\"values_list\":[[1,false],[2.1,2],[4,6],[false,\"cccccc\"]],\"is_aligned\":false,\"devices\":[\"root.s1\",\"root.s1\",\"root.s1\",\"root.s3\"]}"; + isolateDatabasePaths( + "{\"timestamps\":[1635232113960,1635232151960,1635232143960,1635232143960],\"measurements_list\":[[\"s33\",\"s44\"],[\"s55\",\"s66\"],[\"s77\",\"s88\"],[\"s771\",\"s881\"]],\"data_types_list\":[[\"INT32\",\"INT64\"],[\"FLOAT\",\"DOUBLE\"],[\"FLOAT\",\"DOUBLE\"],[\"BOOLEAN\",\"TEXT\"]],\"values_list\":[[1,false],[2.1,2],[4,6],[false,\"cccccc\"]],\"is_aligned\":false,\"devices\":[\"root.s1\",\"root.s1\",\"root.s1\",\"root.s3\"]}"); httpPost.setEntity(new StringEntity(json, Charset.defaultCharset())); for (int i = 0; i < 30; i++) { try { @@ -557,38 +610,42 @@ public void testSpecialPartialInsert() throws Exception { TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select s88, s77, s66, s55, s44, s33 from root.s1", - "Time,root.s1.s88,root.s1.s77,root.s1.s66,root.s1.s55,root.s1.s33,", - new HashSet<>( - Arrays.asList( - "1635232113960,null,null,null,null,1,", - "1635232151960,null,null,2.0,2.1,null,", - "1635232143960,6.0,4.0,null,null,null,"))); + isolateDatabasePaths("select s88, s77, s66, s55, s44, s33 from root.s1"), + isolateDatabasePaths("Time,root.s1.s88,root.s1.s77,root.s1.s66,root.s1.s55,root.s1.s33,"), + isolateDatabasePaths( + new HashSet<>( + Arrays.asList( + "1635232113960,null,null,null,null,1,", + "1635232151960,null,null,2.0,2.1,null,", + "1635232143960,6.0,4.0,null,null,null,")))); } + // Deletions in a non-aligned series must survive historical transfers: the receiver must retain + // exactly five, then four series, including after the historical sub-pipe is auto-dropped. @Test - public void testTransferMods() { + public void testTransferMods() throws Exception { TestUtils.executeNonQueries( senderEnv, - Arrays.asList( - "create database root.sg_nonAligned", - "create TIMESERIES root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`.s0 with datatype=boolean, encoding=RLE,compressor=snappy", - "create timeseries root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`.s1 with datatype=int32, encoding=PLAIN,compressor=LZ4", - "create timeseries root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`.s2 with datatype=int64,encoding=gorilla,compressor=uncompressed", - "create timeseries root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`.s3 with datatype=float,encoding=chimp,compressor=gzip", - "create timeseries root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`.s4 with datatype=double,encoding=ts_2diff,compressor=zstd", - "create timeseries root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`.s5 with datatype=text,encoding=dictionary,compressor=lzma2", - "insert into root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`(time,s1, s2,s3,s4,s0,s5) values(1,1,10,5.39,5.51234,true,''),(11,null,20,5.39,15.51234,false,'第2条 device_nonAligned'),(21,3,null,5.39,25.51234,true,'第3条device_nonAligned'),(31,4,40,null,35.51234,true,'第4条device_nonAligned'),(41,5,50,5.39,null,false,'第5条device_nonAligned'),(51,6,60,5.39,55.51234,null,'第6条device_nonAligned'),(61,7,70,5.39,65.51234,false,null),(71,8,80,5.39,75.51234,false,'第8条device_nonAligned'),(81,9,90,5.39,85.51234,false,'第9条device_nonAligned'),(91,10,100,5.39,95.51234,false,'第10条device_nonAligned'),(101,11,110,5.39,105.51234,false,'第11条device_nonAligned'),(111,12,120,5.39,115.51234,false,'第12条device_nonAligned'),(121,13,130,5.39,125.51234,false,'第13条device_nonAligned'),(131,14,140,5.39,135.51234,false,'第14条device_nonAligned'),(141,15,150,5.39,145.51234,false,'第15条'),(151,16,160,5.39,155.51234,false,'第16条'),(161,17,170,5.39,165.51234,false,'第17条'),(171,18,180,5.39,175.51234,false,'第18条'),(181,19,190,5.39,185.51234,false,'第19条'),(191,20,200,5.39,195.51234,false,'第20条'),(201,21,210,5.39,null,false,'第21条')", - "flush", - "delete timeseries root.sg_nonAligned.非对齐序列带有encoding和压缩方式.s0", - String.format( - "create pipe test with source ('source.realtime.mode'='stream','inclusion'='data','path'='root.sg_nonAligned.非对齐序列带有encoding和压缩方式.**','source.realtime.enable'='true','mods.enable'='true') with sink ('sink'='iotdb-thrift-sink', 'sink.node-urls'='%s')", - receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString()))); + isolateDatabasePaths( + Arrays.asList( + "create database root.sg_nonAligned", + "create TIMESERIES root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`.s0 with datatype=boolean, encoding=RLE,compressor=snappy", + "create timeseries root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`.s1 with datatype=int32, encoding=PLAIN,compressor=LZ4", + "create timeseries root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`.s2 with datatype=int64,encoding=gorilla,compressor=uncompressed", + "create timeseries root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`.s3 with datatype=float,encoding=chimp,compressor=gzip", + "create timeseries root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`.s4 with datatype=double,encoding=ts_2diff,compressor=zstd", + "create timeseries root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`.s5 with datatype=text,encoding=dictionary,compressor=lzma2", + "insert into root.sg_nonAligned.`非对齐序列带有encoding和压缩方式`(time,s1, s2,s3,s4,s0,s5) values(1,1,10,5.39,5.51234,true,''),(11,null,20,5.39,15.51234,false,'第2条 device_nonAligned'),(21,3,null,5.39,25.51234,true,'第3条device_nonAligned'),(31,4,40,null,35.51234,true,'第4条device_nonAligned'),(41,5,50,5.39,null,false,'第5条device_nonAligned'),(51,6,60,5.39,55.51234,null,'第6条device_nonAligned'),(61,7,70,5.39,65.51234,false,null),(71,8,80,5.39,75.51234,false,'第8条device_nonAligned'),(81,9,90,5.39,85.51234,false,'第9条device_nonAligned'),(91,10,100,5.39,95.51234,false,'第10条device_nonAligned'),(101,11,110,5.39,105.51234,false,'第11条device_nonAligned'),(111,12,120,5.39,115.51234,false,'第12条device_nonAligned'),(121,13,130,5.39,125.51234,false,'第13条device_nonAligned'),(131,14,140,5.39,135.51234,false,'第14条device_nonAligned'),(141,15,150,5.39,145.51234,false,'第15条'),(151,16,160,5.39,155.51234,false,'第16条'),(161,17,170,5.39,165.51234,false,'第17条'),(171,18,180,5.39,175.51234,false,'第18条'),(181,19,190,5.39,185.51234,false,'第19条'),(191,20,200,5.39,195.51234,false,'第20条'),(201,21,210,5.39,null,false,'第21条')", + "flush", + "delete timeseries root.sg_nonAligned.非对齐序列带有encoding和压缩方式.s0", + String.format( + "create pipe test with source ('source.realtime.mode'='stream','inclusion'='data','path'='root.sg_nonAligned.非对齐序列带有encoding和压缩方式.**','source.realtime.enable'='true','mods.enable'='true') with sink ('sink'='iotdb-thrift-sink', 'sink.node-urls'='%s')", + receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())))); awaitHistoricalPipeCompletion("test"); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "count timeseries root.sg_nonAligned.非对齐序列带有encoding和压缩方式.*", + isolateDatabasePaths("count timeseries root.sg_nonAligned.非对齐序列带有encoding和压缩方式.*"), "count(timeseries),", Collections.singleton("5,")); @@ -596,61 +653,94 @@ public void testTransferMods() { senderEnv, Arrays.asList("drop pipe if exists test_history", "drop pipe if exists test_realtime")); - TestUtils.executeNonQuery(receiverEnv, "drop database root.**"); + TestUtils.executeNonQuery(receiverEnv, "drop database " + database("sg_nonAligned")); TestUtils.executeNonQueries( senderEnv, - Arrays.asList( - "delete timeseries root.sg_nonAligned.非对齐序列带有encoding和压缩方式.s1", - String.format( - "create pipe test with source ('source.realtime.mode'='stream','inclusion'='data','path'='root.sg_nonAligned.非对齐序列带有encoding和压缩方式.**','source.realtime.enable'='true','mods.enable'='true') with sink ('sink'='iotdb-thrift-sink', 'sink.node-urls'='%s')", - receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString()))); + isolateDatabasePaths( + Arrays.asList( + "delete timeseries root.sg_nonAligned.非对齐序列带有encoding和压缩方式.s1", + String.format( + "create pipe test with source ('source.realtime.mode'='stream','inclusion'='data','path'='root.sg_nonAligned.非对齐序列带有encoding和压缩方式.**','source.realtime.enable'='true','mods.enable'='true') with sink ('sink'='iotdb-thrift-sink', 'sink.node-urls'='%s')", + receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())))); awaitHistoricalPipeCompletion("test"); + PipeTestUtils.awaitDataPipeCompletion(senderEnv, "test_realtime"); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "count timeseries root.sg_nonAligned.非对齐序列带有encoding和压缩方式.*", + isolateDatabasePaths("count timeseries root.sg_nonAligned.非对齐序列带有encoding和压缩方式.*"), "count(timeseries),", Collections.singleton("4,"), 15); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "count timeseries root.sg_nonAligned.非对齐序列带有encoding和压缩方式.*", - "count(timeseries),", - Collections.singleton("4,")); } - private void awaitHistoricalPipeCompletion(final String pipeName) { - await() - .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) - .atMost(600L, TimeUnit.SECONDS) - .untilAsserted( - () -> { - boolean sourcePipeFound = false; - boolean sourcePipeCompleted = true; - boolean realtimePipeFound = false; - - for (final DataNodeWrapper dataNode : senderEnv.getDataNodeWrapperList()) { - try (final Connection connection = senderEnv.getConnection(dataNode); - final Statement statement = connection.createStatement(); - final ResultSet resultSet = statement.executeQuery("show pipes")) { - while (resultSet.next()) { - final String id = resultSet.getString(1); - if (id.equals(pipeName) || id.equals(pipeName + "_history")) { - sourcePipeFound = true; - sourcePipeCompleted &= "0".equals(resultSet.getString(8)); - } else if (id.equals(pipeName + "_realtime")) { - realtimePipeFound = true; - } + private void awaitHistoricalPipeCompletion(final String pipeName) throws Exception { + // SHOW PIPES on any DataNode is forwarded to the ConfigNode. Query that same metadata once + // per poll through a retained client, instead of opening SQL connections to every DataNode. + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final TShowPipeReq request = + new TShowPipeReq().setIsTableModel(false).setUserName(SessionConfig.DEFAULT_USER); + await() + .alias("Historical sub-pipe completion of " + pipeName) + .pollInSameThread() + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) + .atMost(600L, TimeUnit.SECONDS) + .untilAsserted( + () -> { + final TShowPipeResp response = client.showPipe(request); + Assert.assertEquals( + response.toString(), + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + response.getStatus().getCode()); + boolean historicalPipeFound = false; + boolean realtimePipeFound = false; + + for (final TShowPipeInfo pipe : response.getPipeInfoList()) { + if (pipeName.equals(pipe.getId()) + || (pipeName + "_history").equals(pipe.getId())) { + historicalPipeFound = true; + } else if ((pipeName + "_realtime").equals(pipe.getId())) { + realtimePipeFound = true; } } - } - // The historical sub-pipe may be auto-dropped after all events are committed. - Assert.assertTrue(sourcePipeFound || realtimePipeFound); - Assert.assertTrue(!sourcePipeFound || sourcePipeCompleted); - }); + // Auto-splitting is enabled for this class. A zero remaining-event estimate can + // precede capture; auto-removal requires completion reports from all target + // regions. + Assert.assertTrue("Missing realtime sub-pipe " + pipeName, realtimePipeFound); + Assert.assertFalse("Historical sub-pipe is still running", historicalPipeFound); + }); + } + } + + private String isolateDatabasePaths(final String value) { + return value + .replace("root.vehicle", database("vehicle")) + .replace("root.ln", database("ln")) + .replace("root.s1", database("s1")) + .replace("root.s3", database("s3")) + .replace("root.sg_nonAligned", database("sg_nonAligned")); + } + + private List isolateDatabasePaths(final List values) { + final List isolatedValues = new ArrayList<>(values.size()); + for (final String value : values) { + isolatedValues.add(isolateDatabasePaths(value)); + } + return isolatedValues; + } + + private Set isolateDatabasePaths(final Set values) { + final Set isolatedValues = new HashSet<>(values.size()); + for (final String value : values) { + isolatedValues.add(isolateDatabasePaths(value)); + } + return isolatedValues; + } + + private String database(final String name) { + return "root." + name + "_test_" + testId; } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeIPv6IT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeIPv6IT.java index 716d188c2b137..f3b8e8e61a98b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeIPv6IT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeIPv6IT.java @@ -71,8 +71,7 @@ public void setUp() { senderEnv = MultiEnvFactory.getEnv(0); receiverEnv = MultiEnvFactory.getEnv(1); setupConfig(); - senderEnv.initClusterEnvironment(1, 1); - receiverEnv.initClusterEnvironment(1, 1); + executeOnBothEnvs(env -> env.initClusterEnvironment(1, 1)); } @Override @@ -90,6 +89,7 @@ protected void setupConfig() { .setSchemaReplicationFactor(1); } + // Both thrift sinks must replicate their own row through bracketed IPv6 URLs on 1C1D clusters. @Test public void testSyncAndAsyncSinksThroughIPv6Loopback() throws Exception { assertDataNodesUseIPv6(senderEnv.getDataNodeWrapperList()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java index 1ef15e2577d7d..1eb606d393a83 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java @@ -24,12 +24,15 @@ import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.env.AbstractEnv; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; +import org.awaitility.Awaitility; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -38,12 +41,14 @@ import java.sql.Connection; import java.sql.SQLException; +import java.sql.Statement; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import static org.apache.iotdb.db.it.utils.TestUtils.assertNonQueryTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.assertTestFail; @@ -55,7 +60,9 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeLifeCycleIT extends AbstractPipeDualTreeModelAutoIT { +public class IoTDBPipeLifeCycleIT extends AbstractPipeTreeModelSharedClusterIT { + + private boolean permissionUserCreated; @Override @Before @@ -63,6 +70,14 @@ public void setUp() { super.setUp(); } + @Override + protected void cleanEnvironment() { + super.cleanEnvironment(); + if (permissionUserCreated) { + executeNonQueryWithRetry(senderEnv, "drop user test"); + } + } + @Test public void testLifeCycleWithHistoryEnabled() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -134,6 +149,8 @@ public void testLifeCycleWithHistoryEnabled() throws Exception { } } + // With history disabled, old data must stay excluded, new data/schema/database changes must be + // replicated, and changes made while stopped must arrive only after restarting the pipe. @Test public void testLifeCycleWithHistoryDisabled() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -219,25 +236,41 @@ public void testLifeCycleWithHistoryDisabled() throws Exception { "flush"), null); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "select s1 from root.db.d1", - "Time,root.db.d1.s1,", - Collections.singleton("2,2.0,")); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "count timeseries root.ln0.**", - "count(timeseries),", - Collections.singleton("0,")); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "count timeseries root.db.**", - "count(timeseries),", - Collections.singleton("2,")); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, "count databases root.ln0", "count,", Collections.singleton("0,")); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, "count databases root.db", "count,", Collections.singleton("1,")); + // Observe all stopped-pipe invariants in the same ten-second window, using one connection. + // failFast preserves continuous checking: a transient leak must fail the test immediately. + try (final Connection connection = receiverEnv.getConnection(); + final Statement statement = connection.createStatement()) { + Awaitility.await() + .pollInSameThread() + .pollDelay(0, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .during(10, TimeUnit.SECONDS) + .atMost(15, TimeUnit.SECONDS) + .failFast( + () -> { + TestUtils.assertResultSetEqual( + statement.executeQuery("select s1 from root.db.d1"), + "Time,root.db.d1.s1,", + Collections.singleton("2,2.0,")); + TestUtils.assertResultSetEqual( + statement.executeQuery("count timeseries root.ln0.**"), + "count(timeseries),", + Collections.singleton("0,")); + TestUtils.assertResultSetEqual( + statement.executeQuery("count timeseries root.db.**"), + "count(timeseries),", + Collections.singleton("2,")); + TestUtils.assertResultSetEqual( + statement.executeQuery("count databases root.ln0"), + "count,", + Collections.singleton("0,")); + TestUtils.assertResultSetEqual( + statement.executeQuery("count databases root.db"), + "count,", + Collections.singleton("1,")); + }) + .until(() -> true); + } Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); @@ -460,6 +493,8 @@ public void testLifeCycleHybridMode() throws Exception { } } + // A running pipe must recover after both clusters restart and replicate a third row in addition + // to the history and realtime rows that were already received before the restart. @Test public void testLifeCycleWithClusterRestart() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -512,13 +547,7 @@ public void testLifeCycleWithClusterRestart() throws Exception { receiverEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); } - try { - TestUtils.restartCluster(receiverEnv); - TestUtils.restartCluster(senderEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } + executeOnBothEnvs(TestUtils::restartCluster); try (final SyncConfigNodeIServiceClient ignored = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { @@ -534,6 +563,8 @@ public void testLifeCycleWithClusterRestart() throws Exception { } } + // Once realtime transfer is established, stop the receiver and enqueue the remaining rows during + // its outage; restarting all receiver nodes must recover the pipe and deliver all 100 rows. @Test public void testReceiverRestartWhenTransferring() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -565,42 +596,21 @@ public void testReceiverRestartWhenTransferring() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - final Thread t = - new Thread( - () -> { - Connection connection = null; - try { - connection = senderEnv.getConnection(); - } catch (SQLException e) { - // ignore - } - try { - for (int i = 0; i < 100; ++i) { - TestUtils.executeNonQuery( - senderEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - Thread.sleep(100); - } - } catch (InterruptedException ignored) { - } - }); - t.start(); - - try { - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - try { - t.interrupt(); - t.join(); - } catch (Throwable ignored) { - } - return; - } + insertData(senderEnv, 0, 1); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.db.**", + "count(root.db.d1.s1),", + Collections.singleton("1,")); - t.join(); - TestUtils.executeNonQuery(senderEnv, "flush", null); + // Explicitly enqueue events while the receiver is offline instead of relying on a writer's + // sleep interval to overlap the restart. Failures propagate on the test thread. + receiverEnv.shutdownAllDataNodes(); + receiverEnv.shutdownAllConfigNodes(); + insertData(senderEnv, 1, 100); + receiverEnv.startAllConfigNodes(); + receiverEnv.startAllDataNodes(); + ((AbstractEnv) receiverEnv).checkClusterStatusWithoutUnknown(); TestUtils.assertDataEventuallyOnEnv( receiverEnv, @@ -671,6 +681,8 @@ public void testReceiverAlreadyHaveTimeSeries() throws Exception { } } + // Bidirectional pipes with forwarding disabled must converge on all 400 history/realtime rows, + // survive a restart of both clusters, then converge on 600 rows without a forwarding cycle. @Test public void testDoubleLiving() throws Exception { // Double living is two clusters with pipes connecting each other. @@ -682,11 +694,7 @@ public void testDoubleLiving() throws Exception { final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - for (int i = 0; i < 100; ++i) { - TestUtils.executeNonQuery( - senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), null); - } - TestUtils.executeNonQuery(senderEnv, "flush", null); + insertData(senderEnv, 0, 100); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { @@ -713,25 +721,8 @@ public void testDoubleLiving() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); } - try (Connection connection = senderEnv.getConnection()) { - for (int i = 100; i < 200; ++i) { - TestUtils.executeNonQuery( - senderEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - } - TestUtils.executeNonQuery(senderEnv, "flush", connection); - } - - try (Connection connection = receiverEnv.getConnection()) { - for (int i = 200; i < 300; ++i) { - TestUtils.executeNonQuery( - receiverEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - } - TestUtils.executeNonQuery(receiverEnv, "flush", connection); - } + insertData(senderEnv, 100, 200); + insertData(receiverEnv, 200, 300); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { @@ -758,15 +749,7 @@ public void testDoubleLiving() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); } - try (Connection connection = receiverEnv.getConnection()) { - for (int i = 300; i < 400; ++i) { - TestUtils.executeNonQuery( - receiverEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - } - TestUtils.executeNonQuery(receiverEnv, "flush", connection); - } + insertData(receiverEnv, 300, 400); final Set expectedResSet = new HashSet<>(); for (int i = 0; i < 400; ++i) { @@ -778,33 +761,16 @@ public void testDoubleLiving() throws Exception { TestUtils.assertDataEventuallyOnEnv( receiverEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } + // Flush remotely received realtime rows before restart so recovery starts from persisted files. + TestUtils.executeNonQuery(senderEnv, "flush", null); + TestUtils.executeNonQuery(receiverEnv, "flush", null); - try (Connection connection = senderEnv.getConnection()) { - for (int i = 400; i < 500; ++i) { - TestUtils.executeNonQuery( - senderEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - } - TestUtils.executeNonQuery(senderEnv, "flush", connection); - } + // Keep one side of the bidirectional pipes online while the other cluster recovers. + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); - try (Connection connection = receiverEnv.getConnection()) { - for (int i = 500; i < 600; ++i) { - TestUtils.executeNonQuery( - receiverEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - } - TestUtils.executeNonQuery(receiverEnv, "flush", connection); - } + insertData(senderEnv, 400, 500); + insertData(receiverEnv, 500, 600); for (int i = 400; i < 600; ++i) { expectedResSet.add(i + ",1.0,"); @@ -815,9 +781,24 @@ public void testDoubleLiving() throws Exception { receiverEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); } + private static void insertData(final BaseEnv env, final int start, final int end) + throws SQLException { + // Preserve per-statement realtime events while avoiding a new JDBC session for every insert. + try (final Connection connection = env.getConnection(); + final Statement statement = connection.createStatement()) { + for (int i = start; i < end; ++i) { + statement.execute(String.format("insert into root.db.d1(time, s1) values (%s, 1)", i)); + } + statement.execute("flush"); + } + } + + // A user without SYSTEM may list pipes but cannot manage pipes/plugins; granting SYSTEM allows + // pipe lifecycle operations while an untrusted plugin URI must still be rejected. @Test public void testPermission() { - createUser(senderEnv, "test", "test123123456"); + createUser(senderEnv, "test", "IoTDB!Test2024"); + permissionUserCreated = true; assertNonQueryTestFail( senderEnv, @@ -829,46 +810,46 @@ public void testPermission() { + ")", "803: No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456"); + "IoTDB!Test2024"); assertNonQueryTestFail( senderEnv, "drop pipe testPipe", "803: No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456"); + "IoTDB!Test2024"); // Will not throw exception - executeQueryWithRetry(senderEnv, "show pipes", "test", "test123123456"); + executeQueryWithRetry(senderEnv, "show pipes", "test", "IoTDB!Test2024"); assertNonQueryTestFail( senderEnv, "start pipe testPipe", "803: No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456"); + "IoTDB!Test2024"); assertNonQueryTestFail( senderEnv, "stop pipe testPipe", "803: No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456"); + "IoTDB!Test2024"); assertNonQueryTestFail( senderEnv, "create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'", "803: No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456"); + "IoTDB!Test2024"); assertNonQueryTestFail( senderEnv, "drop pipePlugin TestProcessor", "803: No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456"); + "IoTDB!Test2024"); assertTestFail( senderEnv, "show pipePlugins", "803: No permissions for this operation, please add privilege SYSTEM", "test", - "test123123456"); + "IoTDB!Test2024"); grantUserSystemPrivileges(senderEnv, "test", PrivilegeType.SYSTEM); @@ -876,20 +857,20 @@ public void testPermission() { senderEnv, "create pipe testPipe\n" + "with sink (\n" + " 'sink'='write-back-sink'\n" + ")", "test", - "test123123456"); - executeQueryWithRetry(senderEnv, "show pipes", "test", "test123123456"); + "IoTDB!Test2024"); + executeQueryWithRetry(senderEnv, "show pipes", "test", "IoTDB!Test2024"); executeNonQueriesWithRetry( senderEnv, Arrays.asList("start pipe testPipe", "stop pipe testPipe", "drop pipe testPipe"), "test", - "test123123456"); + "IoTDB!Test2024"); assertNonQueryTestFail( senderEnv, "create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'", "701: Untrusted uri xxx", "test", - "test123123456"); - executeQueryWithRetry(senderEnv, "show pipe plugins", "test", "test123123456"); + "IoTDB!Test2024"); + executeQueryWithRetry(senderEnv, "show pipe plugins", "test", "IoTDB!Test2024"); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeMutualSSLIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeMutualSSLIT.java index 6362cca567cc7..cef7fa9e46cd9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeMutualSSLIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeMutualSSLIT.java @@ -46,6 +46,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Properties; import java.util.concurrent.TimeUnit; @RunWith(IoTDBTestRunner.class) @@ -76,6 +77,7 @@ protected void setupConfig() { .setSslProtocol(SessionConfig.DEFAULT_SSL_PROTOCOL); } + // Mutual TLS must deliver two historical rows and one realtime row using the configured stores. @Test public void testPipeCanTransferWithMutualSSL() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -116,35 +118,25 @@ public void testPipeCanTransferWithMutualSSL() throws Exception { Arrays.asList("insert into root.pipe_mtls.d1(time, s1) values (3, 33)", "flush"), null); + // Keep store paths in JDBC properties: Windows drive-letter colons in URL parameters would + // otherwise be parsed as the host/port separator. + final Properties connectionProperties = new Properties(); + connectionProperties.setProperty("user", SessionConfig.DEFAULT_USER); + connectionProperties.setProperty("password", SessionConfig.DEFAULT_PASSWORD); + connectionProperties.setProperty(Config.USE_SSL, "true"); + connectionProperties.setProperty(Config.TRUST_STORE, trustStorePath()); + connectionProperties.setProperty(Config.TRUST_STORE_PWD, STORE_PASSWORD); + connectionProperties.setProperty(Config.KEY_STORE, keyStorePath()); + connectionProperties.setProperty(Config.KEY_STORE_PWD, STORE_PASSWORD); try (final Connection connection = DriverManager.getConnection( - Config.IOTDB_URL_PREFIX - + receiverDataNode.getIpAndPortString() - + "?" - + Config.USE_SSL - + "=true&" - + Config.TRUST_STORE - + "=" - + trustStorePath() - + "&" - + Config.TRUST_STORE_PWD - + "=" - + STORE_PASSWORD - + "&" - + Config.KEY_STORE - + "=" - + keyStorePath() - + "&" - + Config.KEY_STORE_PWD - + "=" - + STORE_PASSWORD, - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD); + Config.IOTDB_URL_PREFIX + receiverDataNode.getIpAndPortString() + "/", + connectionProperties); final Statement statement = connection.createStatement()) { Awaitility.await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(600, TimeUnit.SECONDS) .untilAsserted( () -> diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java index ead03f73c6dd1..b8e8837eac9ea 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java @@ -54,6 +54,9 @@ public void setUp() { MultiEnvFactory.createEnv(2); senderEnv = MultiEnvFactory.getEnv(0); receiverEnv = MultiEnvFactory.getEnv(1); + // Keep automatic region allocation without scaling these tiny protocol workloads by host CPU. + senderEnv.getConfig().getCommonConfig().setDataRegionPerDataNode(2); + receiverEnv.getConfig().getCommonConfig().setDataRegionPerDataNode(2); } private void innerSetUp( @@ -96,10 +99,11 @@ private void innerSetUp( senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); - senderEnv.initClusterEnvironment(configNodesNum, dataNodesNum); - receiverEnv.initClusterEnvironment(configNodesNum, dataNodesNum); + executeOnBothEnvs(env -> env.initClusterEnvironment(configNodesNum, dataNodesNum)); } + // Replicate historical and realtime rows between two Ratis/Ratis/IoT 1C1D clusters; + // stopping the pipe must prevent the third row from reaching the receiver. @Test public void test1C1DWithRatisRatisIot() throws Exception { innerSetUp( @@ -113,6 +117,8 @@ public void test1C1DWithRatisRatisIot() throws Exception { doTest(); } + // Use Simple consensus for the 1C1D receiver's ConfigNode and SchemaRegion and IoT for data; + // replication must deliver the first two rows and exclude the row written after stopping. @Test public void test1C1DWithSimpleSimpleIot() throws Exception { innerSetUp( @@ -126,6 +132,8 @@ public void test1C1DWithSimpleSimpleIot() throws Exception { doTest(); } + // Use Simple data consensus on both 1C1D clusters with Ratis for schema/config; the receiver + // must contain two rows and remain unchanged after the pipe is stopped. @Test public void test1C1DWithRatisRatisSimple() throws Exception { innerSetUp( @@ -139,6 +147,8 @@ public void test1C1DWithRatisRatisSimple() throws Exception { doTest(); } + // Replicate history and realtime data between two 3C3D clusters with three schema/data replicas; + // stopping the pipe must prevent the next row from reaching the receiver. @Test public void test3C3DWith3SchemaRegionFactor3DataRegionFactor() throws Exception { innerSetUp( @@ -152,6 +162,8 @@ public void test3C3DWith3SchemaRegionFactor3DataRegionFactor() throws Exception doTest(); } + // Replicate historical and realtime rows between 3C3D clusters with three schema replicas + // and two data replicas; stopping the pipe must prevent the third row from being transferred. @Test public void test3C3DWith3SchemaRegionFactor2DataRegionFactor() throws Exception { innerSetUp( @@ -165,6 +177,8 @@ public void test3C3DWith3SchemaRegionFactor2DataRegionFactor() throws Exception doTest(); } + // Replicate one row from 3C3D to 1C1D, stop that pipe, then send a second row back through a + // pipe on the smaller cluster; the original sender must contain both rows. @Test public void testPipeOnBothSenderAndReceiver() throws Exception { senderEnv @@ -196,8 +210,11 @@ public void testPipeOnBothSenderAndReceiver() throws Exception { senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); - senderEnv.initClusterEnvironment(3, 3); - receiverEnv.initClusterEnvironment(1, 1); + executeOnBothEnvs( + env -> { + final int nodeCount = env == senderEnv ? 3 : 1; + env.initClusterEnvironment(nodeCount, nodeCount); + }); final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); @@ -343,16 +360,22 @@ private void doTest() throws Exception { } } + // A synchronous sink using three receiver node URLs must transfer historical mods, realtime + // rows and metadata, then transfer the third surviving row through a file-mode pipe. @Test public void testSyncConnectorUseNodeUrls() throws Exception { doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_SYNC_CONNECTOR.getPipePluginName()); } + // An asynchronous sink using three receiver node URLs must transfer historical mods, realtime + // rows and metadata, ending with three surviving rows after file-mode replication. @Test public void testAsyncConnectorUseNodeUrls() throws Exception { doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName()); } + // An air-gap sink using the three receiver ports must transfer historical mods, realtime rows + // and metadata, ending with three surviving rows after file-mode replication. @Test public void testAirGapConnectorUseNodeUrls() throws Exception { doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()); @@ -393,8 +416,7 @@ private void doTestUseNodeUrls(String sinkName) throws Exception { senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); - senderEnv.initClusterEnvironment(1, 1); - receiverEnv.initClusterEnvironment(1, 3); + executeOnBothEnvs(env -> env.initClusterEnvironment(1, env == senderEnv ? 1 : 3)); final StringBuilder nodeUrlsBuilder = new StringBuilder(); for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeReceiverAutoCreateDisabledIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeReceiverAutoCreateDisabledIT.java index c5d2e3b178195..4a48feb8d74d2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeReceiverAutoCreateDisabledIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeReceiverAutoCreateDisabledIT.java @@ -20,13 +20,11 @@ package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -39,20 +37,18 @@ import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.CompletableFuture; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeReceiverAutoCreateDisabledIT extends AbstractPipeDualTreeModelAutoIT { +public class IoTDBPipeReceiverAutoCreateDisabledIT extends AbstractPipeTreeModelSharedClusterIT { @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - setupConfig(); - senderEnv.initClusterEnvironment(1, 1); - receiverEnv.initClusterEnvironment(1, 1); + protected CompletableFuture initializeClusters() { + // Preserve the original 1C1D topology while reusing it across all receiver-schema scenarios. + return CompletableFuture.allOf( + CompletableFuture.runAsync(() -> senderEnv.initClusterEnvironment(1, 1)), + CompletableFuture.runAsync(() -> receiverEnv.initClusterEnvironment(1, 1))); } @Override diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java index 6f1405782823c..3db7bff4f8b9c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java @@ -28,12 +28,12 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionConfig; -import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.pipe.it.dual.PipeTestUtils; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; @@ -41,7 +41,6 @@ import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -61,14 +60,15 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeSourceIT extends AbstractPipeDualTreeModelAutoIT { - - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); +public class IoTDBPipeSourceIT extends AbstractPipeTreeModelSharedClusterIT { + @Override + protected void setupConfig() { + super.setupConfig(); + senderEnv + .getConfig() + .getDataNodeConfig() + .setMetricReporterType(Collections.singletonList("JMX")); senderEnv .getConfig() .getCommonConfig() @@ -99,11 +99,10 @@ public void setUp() { // 10 min, assert that the operations will not time out senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); } + // Each valid history/realtime time-boundary parameter creates a pipe successfully; reuse one + // tree-model JDBC connection for all ten CREATE PIPE statements and retain count checkpoints. @Test public void testExtractorValidParameter() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -111,225 +110,182 @@ public void testExtractorValidParameter() throws Exception { final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - // ---------------------------------------------------------- // - // Scenario 1: when 'source.history.enable' is set to true // - // ---------------------------------------------------------- // - - // Scenario 1.1: test when 'source.history.start-time' and 'source.history.end-time' are - // not set - final String p1_1 = - String.format( - "create pipe p1_1" - + " with source (" - + "'source.history.enable'='true')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { + // ---------------------------------------------------------- // + // Scenario 1: when 'source.history.enable' is set to true // + // ---------------------------------------------------------- // + + // Scenario 1.1: test when 'source.history.start-time' and 'source.history.end-time' are + // not set + final String p1_1 = + String.format( + "create pipe p1_1" + + " with source (" + + "'source.history.enable'='true')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p1_1); - } catch (final SQLException e) { - fail(e.getMessage()); - } - // Scenario 1.2: test when only 'source.history.start-time' is set - final String p1_2 = - String.format( - "create pipe p1_2" - + " with source (" - + "'source.history.enable'='true'," - + "'source.history.start-time'='2000.01.01T08:00:00')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Scenario 1.2: test when only 'source.history.start-time' is set + final String p1_2 = + String.format( + "create pipe p1_2" + + " with source (" + + "'source.history.enable'='true'," + + "'source.history.start-time'='2000.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p1_2); - } catch (final SQLException e) { - fail(e.getMessage()); - } - // Scenario 1.3: test when only 'source.history.end-time' is set - final String p1_3 = - String.format( - "create pipe p1_3" - + " with source (" - + "'source.history.enable'='true'," - + "'source.history.end-time'='2000.01.01T08:00:00')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Scenario 1.3: test when only 'source.history.end-time' is set + final String p1_3 = + String.format( + "create pipe p1_3" + + " with source (" + + "'source.history.enable'='true'," + + "'source.history.end-time'='2000.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p1_3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - // Scenario 1.4: test when 'source.history.start-time' equals 'source.history.end-time' - final String p1_4 = - String.format( - "create pipe p1_4" - + " with source (" - + "'source.history.enable'='true'," - + "'source.history.start-time'='2000.01.01T08:00:00'," - + "'source.history.end-time'='2000.01.01T08:00:00')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Scenario 1.4: test when 'source.history.start-time' equals 'source.history.end-time' + final String p1_4 = + String.format( + "create pipe p1_4" + + " with source (" + + "'source.history.enable'='true'," + + "'source.history.start-time'='2000.01.01T08:00:00'," + + "'source.history.end-time'='2000.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p1_4); - } catch (final SQLException e) { - fail(e.getMessage()); - } - // Scenario 1.5: test when 'source.history.end-time' is future time - final String p1_5 = - String.format( - "create pipe p1_5" - + " with source (" - + "'source.history.enable'='true'," - + "'source.history.start-time'='2000.01.01T08:00:00'," - + "'source.history.end-time'='2100.01.01T08:00:00')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Scenario 1.5: test when 'source.history.end-time' is future time + final String p1_5 = + String.format( + "create pipe p1_5" + + " with source (" + + "'source.history.enable'='true'," + + "'source.history.start-time'='2000.01.01T08:00:00'," + + "'source.history.end-time'='2100.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p1_5); - } catch (final SQLException e) { - fail(e.getMessage()); - } - assertPipeCount(5); - - // ---------------------------------------------------------------------- // - // Scenario 2: when 'source.start-time' or 'source.end-time' is specified // - // ---------------------------------------------------------------------- // - - // Scenario 2.1: test when only 'source.start-time' is set - final String p2_1 = - String.format( - "create pipe p2_1" - + " with source (" - + "'source.start-time'='2000.01.01T08:00:00')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + assertPipeCount(5); + + // ---------------------------------------------------------------------- // + // Scenario 2: when 'source.start-time' or 'source.end-time' is specified // + // ---------------------------------------------------------------------- // + + // Scenario 2.1: test when only 'source.start-time' is set + final String p2_1 = + String.format( + "create pipe p2_1" + + " with source (" + + "'source.start-time'='2000.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p2_1); - } catch (final SQLException e) { - fail(e.getMessage()); - } - // Scenario 2.2: test when only 'source.end-time' is set - final String p2_2 = - String.format( - "create pipe p2_2" - + " with source (" - + "'source.end-time'='2000.01.01T08:00:00')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Scenario 2.2: test when only 'source.end-time' is set + final String p2_2 = + String.format( + "create pipe p2_2" + + " with source (" + + "'source.end-time'='2000.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p2_2); - } catch (final SQLException e) { - fail(e.getMessage()); - } - // Scenario 2.3: test when 'source.start-time' equals 'source.end-time' - final String p2_3 = - String.format( - "create pipe p2_3" - + " with source (" - + "'source.start-time'='2000.01.01T08:00:00'," - + "'source.end-time'='2000.01.01T08:00:00')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Scenario 2.3: test when 'source.start-time' equals 'source.end-time' + final String p2_3 = + String.format( + "create pipe p2_3" + + " with source (" + + "'source.start-time'='2000.01.01T08:00:00'," + + "'source.end-time'='2000.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p2_3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - // Scenario 2.4: test when only when 'source.start-time' is less than 'source.end-time' - final String p2_4 = - String.format( - "create pipe p2_4" - + " with source (" - + "'source.start-time'='2000.01.01T08:00:00'," - + "'source.end-time'='2100.01.01T08:00:00')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Scenario 2.4: test when only when 'source.start-time' is less than 'source.end-time' + final String p2_4 = + String.format( + "create pipe p2_4" + + " with source (" + + "'source.start-time'='2000.01.01T08:00:00'," + + "'source.end-time'='2100.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p2_4); - } catch (final SQLException e) { - fail(e.getMessage()); - } - assertPipeCount(9); - - // ---------------------------------------------------------------------- // - // Scenario 3: when 'source.start-time' or 'source.end-time' is timestamp // - // ---------------------------------------------------------------------- // - - // Scenario 3.1: test when 'source.start-time' is timestamp string - final String p3_1 = - String.format( - "create pipe p3_1" - + " with source (" - + "'source.start-time'='1000'," - + "'source.end-time'='2000.01.01T08:00:00')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + assertPipeCount(9); + + // ---------------------------------------------------------------------- // + // Scenario 3: when 'source.start-time' or 'source.end-time' is timestamp // + // ---------------------------------------------------------------------- // + + // Scenario 3.1: test when 'source.start-time' is timestamp string + final String p3_1 = + String.format( + "create pipe p3_1" + + " with source (" + + "'source.start-time'='1000'," + + "'source.end-time'='2000.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p3_1); + + assertPipeCount(10); } catch (final SQLException e) { fail(e.getMessage()); } - - assertPipeCount(10); } + // Invalid time bounds and mutually disabled history/realtime options must be rejected without + // leaving any pipe behind; expected failures share one connection across all parameter checks. @Test public void testExtractorInvalidParameter() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -337,97 +293,98 @@ public void testExtractorInvalidParameter() throws Exception { final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - // Scenario 1: invalid 'source.history.start-time' - final String formatString = - String.format( - "create pipe p1" - + " with source (" - + "'source.history.enable'='true'," - + "'source.history.start-time'=%s)" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - "%s", receiverIp, receiverPort); - - final List invalidStartTimes = - Arrays.asList("''", "null", "'null'", "'-1000-01-01T00:00:00'", "'2000-01-01T00:00:0'"); - for (final String invalidStartTime : invalidStartTimes) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(String.format(formatString, invalidStartTime)); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + // Scenario 1: invalid 'source.history.start-time' + final String formatString = + String.format( + "create pipe p1" + + " with source (" + + "'source.history.enable'='true'," + + "'source.history.start-time'=%s)" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + "%s", receiverIp, receiverPort); + + final List invalidStartTimes = + Arrays.asList("''", "null", "'null'", "'-1000-01-01T00:00:00'", "'2000-01-01T00:00:0'"); + for (final String invalidStartTime : invalidStartTimes) { + try { + statement.execute(String.format(formatString, invalidStartTime)); + fail(); + } catch (final SQLException ignored) { + } + } + assertPipeCount(0); + + // Scenario 2: can not set 'source.history.enable' and 'source.realtime.enable' both to + // false + final String p2 = + String.format( + "create pipe p2" + + " with source (" + + "'source.history.enable'='false'," + + "'source.realtime.enable'='false')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); + try { + statement.execute(p2); fail(); } catch (final SQLException ignored) { } + assertPipeCount(0); + + // Scenario 3: test when 'source.history.start-time' is greater than + // 'source.history.end-time' + final String p3 = + String.format( + "create pipe p3" + + " with source (" + + "'source.history.enable'='true'," + + "'source.history.start-time'='2001.01.01T08:00:00'," + + "'source.history.end-time'='2000.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); + try { + statement.execute(p3); + fail(); + } catch (final SQLException ignored) { + } + assertPipeCount(0); + + // Scenario 4: test when 'source.start-time' is greater than 'source.end-time' + final String p4 = + String.format( + "create pipe p4" + + " with source (" + + "'source.start-time'='2001.01.01T08:00:00'," + + "'source.end-time'='2000.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); + try { + statement.execute(p4); + fail(); + } catch (final SQLException ignored) { + } + assertPipeCount(0); + } catch (final SQLException e) { + fail(e.getMessage()); } - assertPipeCount(0); - - // Scenario 2: can not set 'source.history.enable' and 'source.realtime.enable' both to - // false - final String p2 = - String.format( - "create pipe p2" - + " with source (" - + "'source.history.enable'='false'," - + "'source.realtime.enable'='false')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p2); - fail(); - } catch (final SQLException ignored) { - } - assertPipeCount(0); - - // Scenario 3: test when 'source.history.start-time' is greater than - // 'source.history.end-time' - final String p3 = - String.format( - "create pipe p3" - + " with source (" - + "'source.history.enable'='true'," - + "'source.history.start-time'='2001.01.01T08:00:00'," - + "'source.history.end-time'='2000.01.01T08:00:00')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p3); - fail(); - } catch (final SQLException ignored) { - } - assertPipeCount(0); - - // Scenario 4: test when 'source.start-time' is greater than 'source.end-time' - final String p4 = - String.format( - "create pipe p4" - + " with source (" - + "'source.start-time'='2001.01.01T08:00:00'," - + "'source.end-time'='2000.01.01T08:00:00')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(p4); - fail(); - } catch (final SQLException ignored) { - } - assertPipeCount(0); } @Test @@ -773,6 +730,9 @@ public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws E } } + // Replicate only timestamps in [2000, 4000] from history and realtime events, including a Tablet + // with unused tail slots. After data completion, out-of-range devices must have no receiver + // series. @Test public void testExtractorTimeRangeMatch() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -875,26 +835,28 @@ public void testExtractorTimeRangeMatch() throws Exception { "flush"), null); - TestUtils.assertDataAlwaysOnEnv( + // Check the updated value as well as absence of excluded devices after every DataRegion has + // committed its flush barrier. Align by device because schema-region column order can vary. + TestUtils.executeNonQuery( + senderEnv, "insert into root.db.d5 (time, at1) values (3000, 30)", null); + PipeTestUtils.awaitDataPipeCompletion(senderEnv, "p1"); + TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(at1) from root.db.d1, root.db.d3, root.db.d5", - "count(root.db.d1.at1),count(root.db.d3.at1),count(root.db.d5.at1),", - Collections.singleton("3,3,3,"), - 600, + "select at1 from root.db.d5 where time = 3000", + "Time,root.db.d5.at1,", + Collections.singleton("3000,30,"), handleFailure); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "show timeseries root.db.d2.**", - "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", - Collections.emptySet()); - TestUtils.assertDataAlwaysOnEnv( + TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "show timeseries root.db.d4.**", - "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", - Collections.emptySet()); + "select count(at1) from root.db.** align by device", + "Device,count(at1),", + new HashSet<>(Arrays.asList("root.db.d1,3,", "root.db.d3,3,", "root.db.d5,3,")), + handleFailure); } } + // A start-time bound must not re-enable history: historical series stay absent after initial + // completion and after realtime events, while timestamps at and above 2000 are transferred. @Test public void testSourceTimeRangeRespectsHistoryDisable() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -929,7 +891,8 @@ public void testSourceTimeRangeRespectsHistoryDisable() throws Exception { new TCreatePipeReq("p1", sinkAttributes).setExtractorAttributes(sourceAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - TestUtils.assertDataAlwaysOnEnv( + PipeTestUtils.awaitDataPipeCompletion(senderEnv, "p1"); + TestUtils.assertDataEventuallyOnEnv( receiverEnv, "show timeseries root.db.history.**", "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", @@ -942,12 +905,13 @@ public void testSourceTimeRangeRespectsHistoryDisable() throws Exception { + " values (1000, 1), (2000, 2), (3000, 3)"), null); + PipeTestUtils.awaitDataPipeCompletion(senderEnv, "p1"); TestUtils.assertDataEventuallyOnEnv( receiverEnv, "select count(at1) from root.db.realtime", "count(root.db.realtime.at1),", Collections.singleton("2,")); - TestUtils.assertDataAlwaysOnEnv( + TestUtils.assertDataEventuallyOnEnv( receiverEnv, "show timeseries root.db.history.**", "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java index ce197b943560d..6eca8d9659446 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java @@ -30,7 +30,7 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -45,7 +45,7 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeSwitchStatusIT extends AbstractPipeDualTreeModelAutoIT { +public class IoTDBPipeSwitchStatusIT extends AbstractPipeTreeModelSharedClusterIT { @Override @Before diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java index 0c2f3bcb312a3..f12ad0e338a9c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java @@ -19,29 +19,42 @@ package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.consensus.ConfigRegionId; +import org.apache.iotdb.commons.pipe.agent.plugin.meta.ConfigNodePipePluginMetaKeeper; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.consensus.ratis.utils.Utils; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.cluster.env.AbstractEnv; import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.ratis.client.RaftClient; +import org.apache.ratis.client.RaftClientConfigKeys; +import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.protocol.RaftClientReply; +import org.apache.ratis.protocol.RaftGroup; +import org.apache.ratis.protocol.RaftPeer; +import org.apache.ratis.retry.RetryPolicies; +import org.apache.ratis.util.TimeDuration; import org.apache.tsfile.external.commons.lang3.SystemUtils; +import org.awaitility.Awaitility; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -55,20 +68,18 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import static org.junit.Assert.fail; +// These syntax scenarios create no data. Drop each method's pipes while retaining both clusters; +// the plugin restart scenario also restores its nodes and removes its custom plugin in finally. @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeSyntaxIT extends AbstractPipeDualTreeModelAutoIT { - - @Override - @Before - public void setUp() { - super.setUp(); - } +public class IoTDBPipeSyntaxIT extends AbstractPipeTreeModelSharedClusterIT { + // Legal identifiers, including quoted numbers and reserved words, must create and drop cleanly. @Test public void testValidPipeName() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -77,15 +88,16 @@ public void testValidPipeName() throws Exception { final int receiverPort = receiverDataNode.getPort(); try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { final List validPipeNames = Arrays.asList("Pipe_1", "null", "`33`", "`root`", "中文", "with"); final List expectedPipeNames = Arrays.asList("Pipe_1", "null", "33", "root", "中文", "with"); for (final String pipeName : validPipeNames) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe %s" @@ -111,8 +123,7 @@ public void testValidPipeName() throws Exception { } for (final String pipeName : validPipeNames) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute(String.format("drop pipe %s", pipeName)); } catch (SQLException e) { e.printStackTrace(); @@ -127,6 +138,7 @@ public void testValidPipeName() throws Exception { } } + // Reordering source and sink parameters must retain validation of the disabled history mode. @Test public void testRevertParameterOrder() { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -153,6 +165,7 @@ public void testRevertParameterOrder() { } } + // Putting the sink before the source must fail and leave no pipe registered. @Test public void testRevertStageOrder() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -161,10 +174,11 @@ public void testRevertStageOrder() throws Exception { final int receiverPort = receiverDataNode.getPort(); try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p1" @@ -188,6 +202,8 @@ public void testRevertStageOrder() throws Exception { } } + // A sink is required; omitted source/processor stages default successfully when a sink is + // present. @Test public void testMissingStage() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -196,23 +212,22 @@ public void testMissingStage() throws Exception { final int receiverPort = receiverDataNode.getPort(); try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + try { statement.execute("create pipe p1"); fail(); } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute("create pipe p2 with source ('source'='iotdb-source')"); fail(); } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( "create pipe p3" + " with source ('source'='iotdb-source')" @@ -221,8 +236,7 @@ public void testMissingStage() throws Exception { } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p4" @@ -237,8 +251,7 @@ public void testMissingStage() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p5" @@ -262,6 +275,7 @@ public void testMissingStage() throws Exception { } } + // Empty source/processor attributes are valid; unknown plugin names must not create a pipe. @Test public void testInvalidParameter() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -270,9 +284,10 @@ public void testInvalidParameter() throws Exception { final int receiverPort = receiverDataNode.getPort(); try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p1" @@ -289,8 +304,7 @@ public void testInvalidParameter() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p2" @@ -306,8 +320,7 @@ public void testInvalidParameter() throws Exception { } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p3" @@ -323,8 +336,7 @@ public void testInvalidParameter() throws Exception { } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p4" @@ -347,6 +359,7 @@ public void testInvalidParameter() throws Exception { } } + // Traversal paths, absolute paths and platform-reserved directory names must be rejected. @Test public void testDirectoryErrors() throws SQLException { try (final Connection connection = senderEnv.getConnection(); @@ -409,6 +422,7 @@ private void testDirectoryError(final String wrongDir, final Statement statement } } + // Single and double quoted stage attributes are valid; bare and backtick quoted values fail. @Test public void testBrackets() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -417,9 +431,10 @@ public void testBrackets() throws Exception { final int receiverPort = receiverDataNode.getPort(); try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe source1" @@ -435,8 +450,7 @@ public void testBrackets() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe source2" @@ -452,8 +466,7 @@ public void testBrackets() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe source3" @@ -469,8 +482,7 @@ public void testBrackets() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe source4" @@ -485,8 +497,7 @@ public void testBrackets() throws Exception { } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe source5" @@ -501,8 +512,7 @@ public void testBrackets() throws Exception { } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe processor1" @@ -518,8 +528,7 @@ public void testBrackets() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe processor2" @@ -535,8 +544,7 @@ public void testBrackets() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe processor3" @@ -552,8 +560,7 @@ public void testBrackets() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe processor4" @@ -568,8 +575,7 @@ public void testBrackets() throws Exception { } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe processor5" @@ -584,8 +590,7 @@ public void testBrackets() throws Exception { } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe sink1" @@ -600,8 +605,7 @@ public void testBrackets() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe sink2" @@ -616,8 +620,7 @@ public void testBrackets() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe sink3" @@ -632,8 +635,7 @@ public void testBrackets() throws Exception { fail(e.getMessage()); } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe sink4" @@ -647,8 +649,7 @@ public void testBrackets() throws Exception { } catch (SQLException ignored) { } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe sink5" @@ -669,6 +670,8 @@ public void testBrackets() throws Exception { } } + // SHOW PIPE must filter by name or shared sink, excluding a pipe with different batching + // settings. @Test public void testShowPipeWithWrongPipeName() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -734,6 +737,7 @@ public void testShowPipeWithWrongPipeName() throws Exception { } } + // Empty or unknown inclusion/exclusion sets must fail; a supported nonempty selection succeeds. @Test public void testInclusionPattern() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -742,10 +746,11 @@ public void testInclusionPattern() throws Exception { final int receiverPort = receiverDataNode.getPort(); try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection(); + final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { // Empty inclusion - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p2" @@ -762,8 +767,7 @@ public void testInclusionPattern() throws Exception { } // Invalid inclusion - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p3" @@ -780,8 +784,7 @@ public void testInclusionPattern() throws Exception { } // Invalid exclusion - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p4" @@ -798,8 +801,7 @@ public void testInclusionPattern() throws Exception { } // Valid - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + try { statement.execute( String.format( "create pipe p4" @@ -823,6 +825,7 @@ public void testInclusionPattern() throws Exception { } } + // The compact CREATE PIPE form must accept a sink without an explicit WITH SINK clause. @Test public void testValidPipeWithoutWithSink() { try (final Connection connection = senderEnv.getConnection(); @@ -834,6 +837,7 @@ public void testValidPipeWithoutWithSink() { } } + // Plugin URI validation and drops of absent/built-in plugins must preserve their specific errors. @Test public void testPipePluginValidation() { try (final Connection connection = senderEnv.getConnection(); @@ -894,6 +898,7 @@ public void testPipePluginValidation() { } } + // Removing an installed plugin jar and restarting must retain the plugin with a visible error. @Test public void testShowPipePluginAfterJarDeletedAndClusterRestart() throws Exception { final String pluginName = "TEST_MISSING_JAR_PROCESSOR"; @@ -916,50 +921,155 @@ public void testShowPipePluginAfterJarDeletedAndClusterRestart() throws Exceptio pluginName, pluginClassName, pluginJarPath.toUri())); } - senderEnv.shutdownAllDataNodes(); - senderEnv.shutdownAllConfigNodes(); - - deletePluginJarUnderConfigNodes(pluginName); - - senderEnv.startAllConfigNodes(); - senderEnv.startAllDataNodes(); - ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); - - boolean pluginFound = false; - boolean exceptionMessageFound = false; - SQLException lastException = null; - for (int retry = 0; retry < 10; retry++) { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement(); - final ResultSet resultSet = statement.executeQuery("show pipeplugins")) { - while (resultSet.next()) { - if (pluginName.equalsIgnoreCase(resultSet.getString("PluginName"))) { - pluginFound = true; - final String exceptionMessage = resultSet.getString("ExceptionMessage"); - exceptionMessageFound = exceptionMessage != null && !exceptionMessage.trim().isEmpty(); - break; - } + try { + takeConfigNodeSnapshots(); + awaitConfigNodeSnapshots(pluginName); + senderEnv.shutdownAllDataNodes(); + senderEnv.shutdownAllConfigNodes(); + + Awaitility.await() + .pollInSameThread() + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try { + deletePluginJarUnderConfigNodes(pluginName); + } catch (IOException e) { + // Windows can keep a just-stopped plugin class loader open briefly. + throw new AssertionError("Plugin jar is still locked after cluster shutdown", e); + } + }); + + senderEnv.startAllConfigNodes(); + senderEnv.startAllDataNodes(); + ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); + + Awaitility.await() + .pollInSameThread() + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .untilAsserted( + () -> { + boolean pluginFound = false; + boolean exceptionMessageFound = false; + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement(); + final ResultSet resultSet = statement.executeQuery("show pipeplugins")) { + while (resultSet.next()) { + if (pluginName.equalsIgnoreCase(resultSet.getString("PluginName"))) { + pluginFound = true; + final String exceptionMessage = resultSet.getString("ExceptionMessage"); + exceptionMessageFound = + exceptionMessage != null && !exceptionMessage.trim().isEmpty(); + break; + } + } + } catch (final SQLException e) { + // Only reconnecting and reading metadata is retried after the explicit restart. + throw new AssertionError("Failed to query plugins after cluster restart", e); + } + Assert.assertTrue("Expected plugin in show pipe plugins result.", pluginFound); + Assert.assertTrue( + "Expected non-empty ExceptionMessage after deleting plugin jar and restarting cluster.", + exceptionMessageFound); + }); + } finally { + // Keep the shared cluster usable even if jar removal or a restart assertion fails. + for (final ConfigNodeWrapper node : senderEnv.getConfigNodeWrapperList()) { + if (!node.isAlive()) { + node.start(); } - lastException = null; - break; - } catch (final SQLException e) { - lastException = e; - Thread.sleep(1000); + } + for (final DataNodeWrapper node : senderEnv.getDataNodeWrapperList()) { + if (!node.isAlive()) { + node.start(); + } + } + ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(String.format("drop pipePlugin if exists %s", pluginName)); } } - if (lastException != null) { - throw lastException; + } + + private void takeConfigNodeSnapshots() throws Exception { + final List locations; + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + locations = client.showCluster().getConfigNodeList(); + } + final List peers = new ArrayList<>(); + for (final TConfigNodeLocation location : locations) { + peers.add( + Utils.fromNodeInfoAndPriorityToRaftPeer( + location.getConfigNodeId(), location.getConsensusEndPoint(), 0)); + } + final RaftProperties properties = new RaftProperties(); + RaftClientConfigKeys.Rpc.setRequestTimeout( + properties, TimeDuration.valueOf(30, TimeUnit.SECONDS)); + // Snapshot only after plugin creation has released its lock. An automatic snapshot between + // procedure steps can block the state-machine updater on that lock while the procedure waits + // for its next consensus write. Persisting here also prevents log replay from restoring the + // jar. + try (final RaftClient client = + RaftClient.newBuilder() + .setRaftGroup( + RaftGroup.valueOf( + Utils.fromConsensusGroupIdToRaftGroupId(new ConfigRegionId(0)), peers)) + .setProperties(properties) + .setRetryPolicy(RetryPolicies.noRetry()) + .build()) { + for (final RaftPeer peer : peers) { + final RaftClientReply reply = + client.getSnapshotManagementApi(peer.getId()).create(true, 30_000); + Assert.assertTrue("Failed to snapshot ConfigNode: " + reply, reply.isSuccess()); + } } + } - Assert.assertTrue("Expected plugin in show pipe plugins result.", pluginFound); - Assert.assertTrue( - "Expected non-empty ExceptionMessage after deleting plugin jar and restarting cluster.", - exceptionMessageFound); + private void awaitConfigNodeSnapshots(final String pluginName) { + for (final ConfigNodeWrapper configNode : senderEnv.getConfigNodeWrapperList()) { + Awaitility.await() + .pollInSameThread() + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .atMost(30, TimeUnit.SECONDS) + .until(() -> snapshotContainsPlugin(configNode, pluginName)); + } + } - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(String.format("drop pipePlugin %s", pluginName)); + private boolean snapshotContainsPlugin( + final ConfigNodeWrapper configNode, final String pluginName) throws IOException { + final Path consensusPath = + Paths.get(configNode.getNodePath(), "data", "confignode", "consensus"); + try (final Stream paths = Files.walk(consensusPath, 3)) { + for (final Path snapshot : + paths + .filter( + path -> + path.getFileName().toString().matches("\\d+_\\d+") + && path.getParent().getFileName().toString().equals("sm")) + .toList()) { + final Path pluginMetaPath = snapshot.resolve("pipe_plugin_info.bin"); + if (!Files.exists(pluginMetaPath)) { + continue; + } + try (final InputStream input = Files.newInputStream(pluginMetaPath)) { + final ConfigNodePipePluginMetaKeeper metaKeeper = new ConfigNodePipePluginMetaKeeper(); + metaKeeper.processLoadSnapshot(input); + if (metaKeeper.containsPipePlugin(pluginName)) { + return true; + } + } catch (final IOException ignored) { + // Ratis may replace a snapshot while the directory is being inspected. + } + } } + return false; } private void deletePluginJarUnderConfigNodes(final String pluginName) throws IOException { @@ -971,17 +1081,13 @@ private void deletePluginJarUnderConfigNodes(final String pluginName) throws IOE continue; } try (final Stream children = Files.walk(pluginJarDirPath)) { - children - .filter(path -> !path.equals(pluginJarDirPath)) - .sorted(Comparator.reverseOrder()) - .forEach( - path -> { - try { - Files.deleteIfExists(path); - } catch (final IOException e) { - throw new RuntimeException(e); - } - }); + for (final Path path : + children + .filter(path -> !path.equals(pluginJarDirPath)) + .sorted(Comparator.reverseOrder()) + .toList()) { + Files.deleteIfExists(path); + } } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBShowReceiversLifecycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBShowReceiversLifecycleIT.java index f08e7f3b4e5df..76461e3e99e79 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBShowReceiversLifecycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBShowReceiversLifecycleIT.java @@ -29,7 +29,7 @@ import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.itbase.exception.InconsistentDataException; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.awaitility.Awaitility; @@ -50,7 +50,7 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBShowReceiversLifecycleIT extends AbstractPipeDualTreeModelAutoIT { +public class IoTDBShowReceiversLifecycleIT extends AbstractPipeTreeModelSharedClusterIT { @Override protected void setupConfig() { @@ -59,6 +59,7 @@ protected void setupConfig() { receiverEnv.getConfig().getCommonConfig().setPipeAirGapReceiverEnabled(true); } + // Verify SHOW RECEIVERS removes the pipe ID after an active thrift pipe is dropped. @Test public void testShowReceiversPipeIdsDisappearAfterDropPipe() throws Exception { final String database = "root.show_receivers_lifecycle"; @@ -74,6 +75,7 @@ public void testShowReceiversPipeIdsDisappearAfterDropPipe() throws Exception { assertShowReceiversDoesNotContainPipe("show receivers", BaseEnv.TREE_SQL_DIALECT, pipeName); } + // Verify SHOW RECEIVERS removes the pipe ID after an active thrift pipe is stopped. @Test public void testShowReceiversPipeIdsDisappearAfterStopPipe() throws Exception { final String database = "root.show_receivers_lifecycle_stop"; @@ -89,6 +91,7 @@ public void testShowReceiversPipeIdsDisappearAfterStopPipe() throws Exception { assertShowReceiversDoesNotContainPipe("show receivers", BaseEnv.TREE_SQL_DIALECT, pipeName); } + // Verify a pipe capturing data and schema exposes both DataNode and ConfigNode receivers. @Test public void testShowReceiversIncludesDataNodeAndConfigNodeReceivers() throws Exception { final String database = "root.show_receivers_config_node"; @@ -119,6 +122,7 @@ public void testShowReceiversIncludesDataNodeAndConfigNodeReceivers() throws Exc "show receivers", BaseEnv.TREE_SQL_DIALECT, pipeName); } + // Verify the air-gap protocol is reported after rows reach its DataNode receiver. @Test public void testShowReceiversShowsAirGapProtocol() throws Exception { final String database = "root.show_receivers_air_gap"; @@ -143,6 +147,13 @@ private void createThriftPipe(final String database, final String pipeName) thro private void createThriftPipe( final String database, final String pipeName, final String sourceInclusion) throws Exception { createPipe(database, pipeName, sourceInclusion, "iotdb-thrift-sink", false); + // Registration precedes data transfer. Wait for the initial row before stopping/dropping the + // pipe so catalog cleanup cannot race with an unfinished receiver schema creation. + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(s1) from " + database + ".d1", + "count(" + database + ".d1.s1),", + Collections.singleton("1,")); } private void createAirGapPipe(final String database, final String pipeName) throws Exception { @@ -201,10 +212,11 @@ private void createPipe( private void assertShowReceivers( final String sql, final String sqlDialect, final String pipeName) { + // Query immediately; registration can already be visible when pipe creation returns. Awaitility.await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(60L, TimeUnit.SECONDS) .ignoreExceptionsMatching(e -> e instanceof InconsistentDataException) .untilAsserted(() -> Assert.assertTrue(hasExpectedReceiver(sql, sqlDialect, pipeName))); @@ -250,8 +262,8 @@ private void assertShowReceiversDoesNotContainPipe( // Pipe connections close asynchronously, so DataNodes may observe different snapshots. Awaitility.await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(60L, TimeUnit.SECONDS) .ignoreExceptionsMatching(e -> e instanceof InconsistentDataException) .untilAsserted(() -> Assert.assertFalse(containsPipe(sql, sqlDialect, pipeName))); @@ -278,8 +290,8 @@ private void assertShowReceiversContainDataNodeAndConfigNode( final String sql, final String sqlDialect, final String pipeName) { Awaitility.await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(60L, TimeUnit.SECONDS) .ignoreExceptionsMatching(e -> e instanceof InconsistentDataException) .untilAsserted( @@ -317,8 +329,8 @@ private void assertShowReceiversContainProtocol( final String sql, final String sqlDialect, final String pipeName, final String protocol) { Awaitility.await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(60L, TimeUnit.SECONDS) .ignoreExceptionsMatching(e -> e instanceof InconsistentDataException) .untilAsserted( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java index fc5d483ea5c30..0ee56493ee4ac 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java @@ -21,37 +21,42 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; +import org.awaitility.Awaitility; import org.junit.Assert; -import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBTreePatternFormatIT extends AbstractPipeDualTreeModelAutoIT { +public class IoTDBTreePatternFormatIT extends AbstractPipeTreeModelSharedClusterIT { - @Override - @Before - public void setUp() { - super.setUp(); - } + @Rule public final TestName testName = new TestName(); + // A legacy prefix root.db.d1.s includes both s and s1, excluding other devices and databases. @Test public void testPrefixPattern() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -65,17 +70,17 @@ public void testPrefixPattern() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)", - "insert into root.db.d2(time, s) values (1, 1)", - "insert into root.db2.d1(time, s) values (1, 1)"), + isolateDatabasePaths("insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)"), + isolateDatabasePaths("insert into root.db.d2(time, s) values (1, 1)"), + isolateDatabasePaths("insert into root.db2.d1(time, s) values (1, 1)")), null); - awaitUntilFlush(senderEnv); + flushAndAwaitTsFile(); final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); final Map sinkAttributes = new HashMap<>(); - sourceAttributes.put("source.pattern", "root.db.d1.s"); + sourceAttributes.put("source.pattern", isolateDatabasePaths("root.db.d1.s")); sourceAttributes.put("source.inclusion", "data.insert"); sourceAttributes.put("user", "root"); @@ -99,12 +104,13 @@ public void testPrefixPattern() throws Exception { expectedResSet.add("1,1.0,1.0,"); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.db.**", - "Time,root.db.d1.s,root.db.d1.s1,", + isolateDatabasePaths("select * from root.db.**"), + isolateDatabasePaths("Time,root.db.d1.s,root.db.d1.s1,"), expectedResSet); } } + // The path root.**.d1.s* selects s and s1 in both databases, excluding d2 and t. @Test public void testIoTDBPattern() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -118,11 +124,11 @@ public void testIoTDBPattern() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)", - "insert into root.db.d2(time, s) values (1, 1)", - "insert into root.db2.d1(time, s) values (1, 1)"), + isolateDatabasePaths("insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)"), + isolateDatabasePaths("insert into root.db.d2(time, s) values (1, 1)"), + isolateDatabasePaths("insert into root.db2.d1(time, s) values (1, 1)")), null); - awaitUntilFlush(senderEnv); + flushAndAwaitTsFile(); final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -152,12 +158,13 @@ public void testIoTDBPattern() throws Exception { expectedResSet.add("1,1.0,1.0,1.0,"); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.db2.**,root.db.**", - "Time,root.db2.d1.s,root.db.d1.s,root.db.d1.s1,", + isolateDatabasePaths("select * from root.db2.**,root.db.**"), + isolateDatabasePaths("Time,root.db2.d1.s,root.db.d1.s,root.db.d1.s1,"), expectedResSet); } } + // Legacy pattern + iotdb format must select the same d1.s* historical data as the path option. @Test public void testIoTDBPatternWithLegacySyntax() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -171,11 +178,11 @@ public void testIoTDBPatternWithLegacySyntax() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)", - "insert into root.db.d2(time, s) values (1, 1)", - "insert into root.db2.d1(time, s) values (1, 1)"), + isolateDatabasePaths("insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)"), + isolateDatabasePaths("insert into root.db.d2(time, s) values (1, 1)"), + isolateDatabasePaths("insert into root.db2.d1(time, s) values (1, 1)")), null); - awaitUntilFlush(senderEnv); + flushAndAwaitTsFile(); final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -206,14 +213,46 @@ public void testIoTDBPatternWithLegacySyntax() throws Exception { expectedResSet.add("1,1.0,1.0,1.0,"); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select * from root.db2.**,root.db.**", - "Time,root.db2.d1.s,root.db.d1.s,root.db.d1.s1,", + isolateDatabasePaths("select * from root.db2.**,root.db.**"), + isolateDatabasePaths("Time,root.db2.d1.s,root.db.d1.s,root.db.d1.s1,"), expectedResSet); } } //////////////////////////// Multiple & Exclusion //////////////////////////// + private void flushAndAwaitTsFile() { + // Shared clusters retain parent directories after database deletion. Check an actual TsFile + // after FLUSH, without imposing the old two-second delay on every history/realtime scenario. + Awaitility.await() + .pollDelay(Duration.ZERO) + .pollInterval(Duration.ofMillis(100)) + .atMost(Duration.ofMinutes(1)) + .until( + () -> { + TestUtils.executeNonQuery(senderEnv, "flush", null); + for (final DataNodeWrapper node : senderEnv.getDataNodeWrapperList()) { + final Path dataDirectory = + Path.of( + node.getNodePath(), + IoTDBConstant.DATA_FOLDER_NAME, + "datanode", + IoTDBConstant.DATA_FOLDER_NAME); + if (Files.isDirectory(dataDirectory)) { + try (final Stream files = Files.walk(dataDirectory)) { + if (files.anyMatch( + file -> + Files.isRegularFile(file) + && file.getFileName().toString().endsWith(".tsfile"))) { + return true; + } + } + } + } + return false; + }); + } + private void testPipeWithMultiplePatterns( final Map sourceAttributes, final List insertQueries, @@ -242,7 +281,7 @@ private void testPipeWithMultiplePatterns( // 3. Handle historical data insertion (if applicable) if (isHistorical) { TestUtils.executeNonQueries(senderEnv, insertQueries, null); - awaitUntilFlush(senderEnv); + flushAndAwaitTsFile(); } // 4. Create the pipe @@ -260,7 +299,7 @@ private void testPipeWithMultiplePatterns( // 6. Handle realtime data insertion (if applicable) if (!isHistorical) { TestUtils.executeNonQueries(senderEnv, insertQueries, null); - awaitUntilFlush(senderEnv); + flushAndAwaitTsFile(); } // 7. Validate data eventually arrives on the receiver @@ -269,19 +308,21 @@ private void testPipeWithMultiplePatterns( } } + // Historical data matching either inclusion pattern must sync while root.db3 stays excluded. @Test public void testMultipleIoTDBPatternHistoricalData() throws Exception { final Map sourceAttributes = new HashMap<>(); - sourceAttributes.put("source.pattern.inclusion", "root.db.**, root.db2.d1.*"); + sourceAttributes.put( + "source.pattern.inclusion", isolateDatabasePaths("root.db.**, root.db2.d1.*")); sourceAttributes.put("source.inclusion", "data.insert"); sourceAttributes.put("user", "root"); final List insertQueries = Arrays.asList( - "insert into root.db.d1(time, s, s1) values (1, 1, 1)", - "insert into root.db.d2(time, s) values (2, 2)", - "insert into root.db2.d1(time, s, t) values (3, 3, 3)", - "insert into root.db3.d1(time, s) values (4, 4)"); + isolateDatabasePaths("insert into root.db.d1(time, s, s1) values (1, 1, 1)"), + isolateDatabasePaths("insert into root.db.d2(time, s) values (2, 2)"), + isolateDatabasePaths("insert into root.db2.d1(time, s, t) values (3, 3, 3)"), + isolateDatabasePaths("insert into root.db3.d1(time, s) values (4, 4)")); final Set expectedResSet = new HashSet<>(); expectedResSet.add("1,null,null,1.0,1.0,null,"); @@ -292,24 +333,27 @@ public void testMultipleIoTDBPatternHistoricalData() throws Exception { sourceAttributes, insertQueries, true, // isHistorical = true - "select * from root.db2.**,root.db.**", - "Time,root.db2.d1.s,root.db2.d1.t,root.db.d1.s,root.db.d1.s1,root.db.d2.s,", + isolateDatabasePaths("select * from root.db2.**,root.db.**"), + isolateDatabasePaths( + "Time,root.db2.d1.s,root.db2.d1.t,root.db.d1.s,root.db.d1.s1,root.db.d2.s,"), expectedResSet); } + // Realtime data matching either inclusion pattern must sync while root.db3 stays excluded. @Test public void testMultipleIoTDBPatternRealtimeData() throws Exception { final Map sourceAttributes = new HashMap<>(); - sourceAttributes.put("source.pattern.inclusion", "root.db.**, root.db2.d1.*"); + sourceAttributes.put( + "source.pattern.inclusion", isolateDatabasePaths("root.db.**, root.db2.d1.*")); sourceAttributes.put("source.inclusion", "data.insert"); sourceAttributes.put("user", "root"); final List insertQueries = Arrays.asList( - "insert into root.db.d1(time, s, s1) values (1, 1, 1)", - "insert into root.db.d2(time, s) values (2, 2)", - "insert into root.db2.d1(time, s, t) values (3, 3, 3)", - "insert into root.db3.d1(time, s) values (4, 4)"); + isolateDatabasePaths("insert into root.db.d1(time, s, s1) values (1, 1, 1)"), + isolateDatabasePaths("insert into root.db.d2(time, s) values (2, 2)"), + isolateDatabasePaths("insert into root.db2.d1(time, s, t) values (3, 3, 3)"), + isolateDatabasePaths("insert into root.db3.d1(time, s) values (4, 4)")); final Set expectedResSet = new HashSet<>(); expectedResSet.add("1,null,null,1.0,1.0,null,"); @@ -320,30 +364,33 @@ public void testMultipleIoTDBPatternRealtimeData() throws Exception { sourceAttributes, insertQueries, false, // isHistorical = false - "select * from root.db2.**,root.db.**", - "Time,root.db2.d1.s,root.db2.d1.t,root.db.d1.s,root.db.d1.s1,root.db.d2.s,", + isolateDatabasePaths("select * from root.db2.**,root.db.**"), + isolateDatabasePaths( + "Time,root.db2.d1.s,root.db2.d1.t,root.db.d1.s,root.db.d1.s1,root.db.d2.s,"), expectedResSet); } + // Historical inclusion root.db.** must exclude s* under d1 and every measurement under d3. @Test public void testIoTDBPatternWithExclusionHistoricalData() throws Exception { final Map sourceAttributes = new HashMap<>(); // Inclusion: Match everything under root.db - sourceAttributes.put("source.pattern.inclusion", "root.db.**"); + sourceAttributes.put("source.pattern.inclusion", isolateDatabasePaths("root.db.**")); // Exclusion: Exclude root.db.d1.s* and root.db.d3.* - sourceAttributes.put("source.pattern.exclusion", "root.db.d1.s*, root.db.d3.*"); + sourceAttributes.put( + "source.pattern.exclusion", isolateDatabasePaths("root.db.d1.s*, root.db.d3.*")); sourceAttributes.put("source.inclusion", "data.insert"); sourceAttributes.put("user", "root"); final List insertQueries = Arrays.asList( // s, s1 excluded, t matches - "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)", + isolateDatabasePaths("insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)"), // s matches - "insert into root.db.d2(time, s) values (2, 2)", + isolateDatabasePaths("insert into root.db.d2(time, s) values (2, 2)"), // s excluded - "insert into root.db.d3(time, s) values (3, 3)", - "insert into root.db1.d1(time, s) values (4, 4)"); + isolateDatabasePaths("insert into root.db.d3(time, s) values (3, 3)"), + isolateDatabasePaths("insert into root.db1.d1(time, s) values (4, 4)")); final Set expectedResSet = new HashSet<>(); expectedResSet.add("1,1.0,null,"); @@ -353,25 +400,27 @@ public void testIoTDBPatternWithExclusionHistoricalData() throws Exception { sourceAttributes, insertQueries, true, // isHistorical = true - "select * from root.db.**", - "Time,root.db.d1.t,root.db.d2.s,", + isolateDatabasePaths("select * from root.db.**"), + isolateDatabasePaths("Time,root.db.d1.t,root.db.d2.s,"), expectedResSet); } + // Realtime inclusion root.db.** must exclude s* under d1 and every measurement under d3. @Test public void testIoTDBPatternWithExclusionRealtimeData() throws Exception { final Map sourceAttributes = new HashMap<>(); - sourceAttributes.put("source.pattern.inclusion", "root.db.**"); - sourceAttributes.put("source.pattern.exclusion", "root.db.d1.s*, root.db.d3.*"); + sourceAttributes.put("source.pattern.inclusion", isolateDatabasePaths("root.db.**")); + sourceAttributes.put( + "source.pattern.exclusion", isolateDatabasePaths("root.db.d1.s*, root.db.d3.*")); sourceAttributes.put("source.inclusion", "data.insert"); sourceAttributes.put("user", "root"); final List insertQueries = Arrays.asList( - "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)", - "insert into root.db.d2(time, s) values (2, 2)", - "insert into root.db.d3(time, s) values (3, 3)", - "insert into root.db1.d1(time, s) values (4, 4)"); + isolateDatabasePaths("insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)"), + isolateDatabasePaths("insert into root.db.d2(time, s) values (2, 2)"), + isolateDatabasePaths("insert into root.db.d3(time, s) values (3, 3)"), + isolateDatabasePaths("insert into root.db1.d1(time, s) values (4, 4)")); final Set expectedResSet = new HashSet<>(); expectedResSet.add("1,1.0,null,"); @@ -381,24 +430,26 @@ public void testIoTDBPatternWithExclusionRealtimeData() throws Exception { sourceAttributes, insertQueries, false, // isHistorical = false - "select * from root.db.**", - "Time,root.db.d1.t,root.db.d2.s,", + isolateDatabasePaths("select * from root.db.**"), + isolateDatabasePaths("Time,root.db.d1.t,root.db.d2.s,"), expectedResSet); } + // With only historical exclusions, unmatched databases still sync while d1.s* and d3 stay out. @Test public void testIoTDBPatternWithExclusionOnlyHistoricalData() throws Exception { final Map sourceAttributes = new HashMap<>(); - sourceAttributes.put("source.pattern.exclusion", "root.db.d1.s*, root.db.d3.*"); + sourceAttributes.put( + "source.pattern.exclusion", isolateDatabasePaths("root.db.d1.s*, root.db.d3.*")); sourceAttributes.put("source.inclusion", "data.insert"); sourceAttributes.put("user", "root"); final List insertQueries = Arrays.asList( - "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)", - "insert into root.db.d2(time, s) values (2, 2)", - "insert into root.db.d3(time, s) values (3, 3)", - "insert into root.db1.d1(time, s) values (4, 4)"); + isolateDatabasePaths("insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)"), + isolateDatabasePaths("insert into root.db.d2(time, s) values (2, 2)"), + isolateDatabasePaths("insert into root.db.d3(time, s) values (3, 3)"), + isolateDatabasePaths("insert into root.db1.d1(time, s) values (4, 4)")); final Set expectedResSet = new HashSet<>(); expectedResSet.add("1,null,1.0,null,"); @@ -409,24 +460,26 @@ public void testIoTDBPatternWithExclusionOnlyHistoricalData() throws Exception { sourceAttributes, insertQueries, true, // isHistorical = true - "select * from root.db1.**,root.db.**", - "Time,root.db1.d1.s,root.db.d1.t,root.db.d2.s,", + isolateDatabasePaths("select * from root.db1.**,root.db.**"), + isolateDatabasePaths("Time,root.db1.d1.s,root.db.d1.t,root.db.d2.s,"), expectedResSet); } + // With only realtime exclusions, unmatched databases still sync while d1.s* and d3 stay out. @Test public void testIoTDBPatternWithExclusionOnlyRealtimeData() throws Exception { final Map sourceAttributes = new HashMap<>(); - sourceAttributes.put("source.pattern.exclusion", "root.db.d1.s*, root.db.d3.*"); + sourceAttributes.put( + "source.pattern.exclusion", isolateDatabasePaths("root.db.d1.s*, root.db.d3.*")); sourceAttributes.put("source.inclusion", "data.insert"); sourceAttributes.put("user", "root"); final List insertQueries = Arrays.asList( - "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)", - "insert into root.db.d2(time, s) values (2, 2)", - "insert into root.db.d3(time, s) values (3, 3)", - "insert into root.db1.d1(time, s) values (4, 4)"); + isolateDatabasePaths("insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)"), + isolateDatabasePaths("insert into root.db.d2(time, s) values (2, 2)"), + isolateDatabasePaths("insert into root.db.d3(time, s) values (3, 3)"), + isolateDatabasePaths("insert into root.db1.d1(time, s) values (4, 4)")); final Set expectedResSet = new HashSet<>(); expectedResSet.add("1,null,1.0,null,"); @@ -437,26 +490,28 @@ public void testIoTDBPatternWithExclusionOnlyRealtimeData() throws Exception { sourceAttributes, insertQueries, false, // isHistorical = false - "select * from root.db1.**,root.db.**", - "Time,root.db1.d1.s,root.db.d1.t,root.db.d2.s,", + isolateDatabasePaths("select * from root.db1.**,root.db.**"), + isolateDatabasePaths("Time,root.db1.d1.s,root.db.d1.t,root.db.d2.s,"), expectedResSet); } + // Historical exclusion of d_20 must take precedence over both wildcard and exact inclusions. @Test public void testIoTDBPatternWithInclusionAndExclusionHistoricalData() throws Exception { final Map sourceAttributes = new HashMap<>(); sourceAttributes.put( - "source.pattern.inclusion", "root.test.g_0.d_2*.**,root.test.g_0.d_20.s_0"); - sourceAttributes.put("source.pattern.exclusion", "root.test.g_0.d_20.**"); + "source.pattern.inclusion", + isolateDatabasePaths("root.test.g_0.d_2*.**,root.test.g_0.d_20.s_0")); + sourceAttributes.put("source.pattern.exclusion", isolateDatabasePaths("root.test.g_0.d_20.**")); sourceAttributes.put("source.inclusion", "data.insert"); sourceAttributes.put("user", "root"); final List insertQueries = Arrays.asList( - "insert into root.test.g_0.d_21(time, s_1) values (1, 1)", - "insert into root.test.g_0.d_22(time, s_1) values (2, 2)", - "insert into root.test.g_0.d_20(time, s_0, s_1) values (3, 3, 3)", - "insert into root.test.g_0.d_30(time, s_1) values (4, 4)"); + isolateDatabasePaths("insert into root.test.g_0.d_21(time, s_1) values (1, 1)"), + isolateDatabasePaths("insert into root.test.g_0.d_22(time, s_1) values (2, 2)"), + isolateDatabasePaths("insert into root.test.g_0.d_20(time, s_0, s_1) values (3, 3, 3)"), + isolateDatabasePaths("insert into root.test.g_0.d_30(time, s_1) values (4, 4)")); final Set expectedResSet = new HashSet<>(); expectedResSet.add("1,1.0,null,"); @@ -466,26 +521,28 @@ public void testIoTDBPatternWithInclusionAndExclusionHistoricalData() throws Exc sourceAttributes, insertQueries, true, // isHistorical = true - "select * from root.test.**", - "Time,root.test.g_0.d_21.s_1,root.test.g_0.d_22.s_1,", + isolateDatabasePaths("select * from root.test.**"), + isolateDatabasePaths("Time,root.test.g_0.d_21.s_1,root.test.g_0.d_22.s_1,"), expectedResSet); } + // Realtime exclusion of d_20 must take precedence over both wildcard and exact inclusions. @Test public void testIoTDBPatternWithInclusionAndExclusionRealtimeData() throws Exception { final Map sourceAttributes = new HashMap<>(); sourceAttributes.put( - "source.pattern.inclusion", "root.test.g_0.d_2*.**,root.test.g_0.d_20.s_0"); - sourceAttributes.put("source.pattern.exclusion", "root.test.g_0.d_20.**"); + "source.pattern.inclusion", + isolateDatabasePaths("root.test.g_0.d_2*.**,root.test.g_0.d_20.s_0")); + sourceAttributes.put("source.pattern.exclusion", isolateDatabasePaths("root.test.g_0.d_20.**")); sourceAttributes.put("source.inclusion", "data.insert"); sourceAttributes.put("user", "root"); final List insertQueries = Arrays.asList( - "insert into root.test.g_0.d_21(time, s_1) values (1, 1)", - "insert into root.test.g_0.d_22(time, s_1) values (2, 2)", - "insert into root.test.g_0.d_20(time, s_0, s_1) values (3, 3, 3)", - "insert into root.test.g_0.d_30(time, s_1) values (4, 4)"); + isolateDatabasePaths("insert into root.test.g_0.d_21(time, s_1) values (1, 1)"), + isolateDatabasePaths("insert into root.test.g_0.d_22(time, s_1) values (2, 2)"), + isolateDatabasePaths("insert into root.test.g_0.d_20(time, s_0, s_1) values (3, 3, 3)"), + isolateDatabasePaths("insert into root.test.g_0.d_30(time, s_1) values (4, 4)")); final Set expectedResSet = new HashSet<>(); expectedResSet.add("1,1.0,null,"); @@ -495,8 +552,22 @@ public void testIoTDBPatternWithInclusionAndExclusionRealtimeData() throws Excep sourceAttributes, insertQueries, false, // isHistorical = false - "select * from root.test.**", - "Time,root.test.g_0.d_21.s_1,root.test.g_0.d_22.s_1,", + isolateDatabasePaths("select * from root.test.**"), + isolateDatabasePaths("Time,root.test.g_0.d_21.s_1,root.test.g_0.d_22.s_1,"), expectedResSet); } + + private String isolateDatabasePaths(final String value) { + // Suffix the database node only, retaining wildcards and the devices/measurements under test. + return value.replaceAll("root\\.(db[0-9]*|test)(?=[.,]|$)", "$0_" + databaseSuffix()); + } + + private String databaseSuffix() { + // Stay below the receiver's 64-character database path limit, including root and the prefix. + // Retain a hash when shortening names so methods with the same prefix use different databases. + final String method = testName.getMethodName().toLowerCase(Locale.ROOT); + return method.length() <= 40 + ? method + : method.substring(0, 31) + "_" + Integer.toUnsignedString(method.hashCode(), 16); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java index 5cc8bc5d74105..90347e45e1c1c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java @@ -24,36 +24,54 @@ import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeAutoConflictIT extends AbstractPipeDualTreeModelAutoIT { +public class IoTDBPipeAutoConflictIT extends AbstractPipeTreeModelSharedClusterIT { + @Rule public final TestName testName = new TestName(); + + private ISession senderSession; + private ISession receiverSession; + private boolean clustersReady; + + @Override @Before public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); + super.setUp(); + clustersReady = true; + } + + @Override + protected void setupConfig() { senderEnv .getConfig() @@ -79,11 +97,10 @@ public void setUp() { // 10 min, assert that the operations will not time out senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); } + // Both IOT-consensus clusters must merge 600 disjoint rows despite automatic schema conflicts, + // including the final 200 rows written after both clusters restart. @Test public void testDoubleLivingAutoConflict() throws Exception { // Double living is two clusters each with a pipe connecting to the other. @@ -95,15 +112,7 @@ public void testDoubleLivingAutoConflict() throws Exception { final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - try (Connection connection = senderEnv.getConnection()) { - for (int i = 0; i < 100; ++i) { - TestUtils.executeNonQuery( - senderEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - } - TestUtils.executeNonQuery(senderEnv, "flush", connection); - } + insertData(senderEnv, 0, 100, true); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { @@ -115,7 +124,7 @@ public void testDoubleLivingAutoConflict() throws Exception { sourceAttributes.put("source.inclusion.exclusion", ""); // Add this property to avoid making self cycle. sourceAttributes.put("source.forwarding-pipe-requests", "false"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.batch.enable", "false"); @@ -133,25 +142,9 @@ public void testDoubleLivingAutoConflict() throws Exception { TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); } - try (Connection connection = senderEnv.getConnection()) { - for (int i = 100; i < 200; ++i) { - TestUtils.executeNonQuery( - senderEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - } - TestUtils.executeNonQuery(senderEnv, "flush", connection); - } + insertData(senderEnv, 100, 200, true); - try (Connection connection = receiverEnv.getConnection()) { - for (int i = 200; i < 300; ++i) { - TestUtils.executeNonQuery( - receiverEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - } - TestUtils.executeNonQuery(receiverEnv, "flush", connection); - } + insertData(receiverEnv, 200, 300, true); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { @@ -163,7 +156,7 @@ public void testDoubleLivingAutoConflict() throws Exception { sourceAttributes.put("source.inclusion.exclusion", ""); // Add this property to avoid to make self cycle. sourceAttributes.put("source.forwarding-pipe-requests", "false"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.batch.enable", "false"); @@ -180,15 +173,7 @@ public void testDoubleLivingAutoConflict() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); } - try (Connection connection = receiverEnv.getConnection()) { - for (int i = 300; i < 400; ++i) { - TestUtils.executeNonQuery( - receiverEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - } - TestUtils.executeNonQuery(receiverEnv, "flush", connection); - } + insertData(receiverEnv, 300, 400, true); final Set expectedResSet = new HashSet<>(); for (int i = 0; i < 400; ++i) { @@ -196,48 +181,41 @@ public void testDoubleLivingAutoConflict() throws Exception { } TestUtils.assertDataEventuallyOnEnv( - senderEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); + senderEnv, + isolateDatabasePaths("select * from root.db.**"), + isolateDatabasePaths("Time,root.db.d1.s1,"), + expectedResSet); TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); - - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } + receiverEnv, + isolateDatabasePaths("select * from root.db.**"), + isolateDatabasePaths("Time,root.db.d1.s1,"), + expectedResSet); - try (Connection connection = senderEnv.getConnection()) { - for (int i = 400; i < 500; ++i) { - TestUtils.executeNonQuery( - senderEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - } - TestUtils.executeNonQuery(senderEnv, "flush", connection); - } + // Close pre-restart sessions, then overlap the required restarts. A restart failure must fail + // the test rather than skip the assertions on the final 200 rows. + closeWriteSessions(); + executeOnBothEnvs(TestUtils::restartCluster); - try (Connection connection = receiverEnv.getConnection()) { - for (int i = 500; i < 600; ++i) { - TestUtils.executeNonQuery( - receiverEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - } + insertData(senderEnv, 400, 500, true); - TestUtils.executeNonQuery(receiverEnv, "flush", connection); - } + insertData(receiverEnv, 500, 600, true); for (int i = 400; i < 600; ++i) { expectedResSet.add(i + ",1.0,"); } TestUtils.assertDataEventuallyOnEnv( - senderEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); + senderEnv, + isolateDatabasePaths("select * from root.db.**"), + isolateDatabasePaths("Time,root.db.d1.s1,"), + expectedResSet); TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); + receiverEnv, + isolateDatabasePaths("select * from root.db.**"), + isolateDatabasePaths("Time,root.db.d1.s1,"), + expectedResSet); } + // Bidirectional pipes must auto-extend a shared template for s1 and merge 400 rows on both sides. @Test public void testDoubleLivingAutoConflictTemplate() throws Exception { final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); @@ -257,7 +235,7 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); sourceAttributes.put("source.forwarding-pipe-requests", "false"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.batch.enable", "false"); @@ -284,7 +262,7 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); sourceAttributes.put("source.forwarding-pipe-requests", "false"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.batch.enable", "false"); @@ -307,8 +285,8 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { Arrays.asList( // Auto extend s1 "create schema template t1 (s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", - "create database root.db", - "set device template t1 to root.db"), + isolateDatabasePaths("create database root.db"), + isolateDatabasePaths("set device template t1 to root.db")), null); // Prevent receiver writes from creating ordinary timeseries before the template binding @@ -317,34 +295,20 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { receiverEnv, "show paths set device template t1", "Paths,", - Collections.singleton("root.db,")); - - try (Connection connection = senderEnv.getConnection()) { - for (int i = 0; i < 200; ++i) { - TestUtils.executeNonQuery( - senderEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - } - } + Collections.singleton(isolateDatabasePaths("root.db,"))); + + insertData(senderEnv, 0, 200, false); // SHOW PATHS can expose a pre-set binding before the DataNodes can use the template. // Retry the first idempotent write until the receiver can activate and extend the template. TestUtils.executeNonQueryWithRetry( - receiverEnv, "insert into root.db.d1(time, s1) values (200, 1)"); - - try (Connection connection = receiverEnv.getConnection()) { - for (int i = 201; i < 400; ++i) { - TestUtils.executeNonQuery( - receiverEnv, - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i), - connection); - } - } + receiverEnv, isolateDatabasePaths("insert into root.db.d1(time, s1) values (200, 1)")); - TestUtils.executeNonQuery(senderEnv, "flush", null); + insertData(receiverEnv, 201, 400, false); - TestUtils.executeNonQuery(receiverEnv, "flush", null); + writeSession(senderEnv).executeNonQueryStatement("flush"); + + writeSession(receiverEnv).executeNonQueryStatement("flush"); final Set expectedResSet = new HashSet<>(); for (int i = 0; i < 400; ++i) { @@ -352,11 +316,19 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { } TestUtils.assertDataEventuallyOnEnv( - senderEnv, "select s1 from root.db.d1", "Time,root.db.d1.s1,", expectedResSet); + senderEnv, + isolateDatabasePaths("select s1 from root.db.d1"), + isolateDatabasePaths("Time,root.db.d1.s1,"), + expectedResSet); TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select s1 from root.db.d1", "Time,root.db.d1.s1,", expectedResSet); + receiverEnv, + isolateDatabasePaths("select s1 from root.db.d1"), + isolateDatabasePaths("Time,root.db.d1.s1,"), + expectedResSet); } + // A manually created receiver series must acquire the sender tag and attribute when auto-created + // schema events race with the existing series. @Test public void testAutoManualCreateRace() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -371,7 +343,7 @@ public void testAutoManualCreateRace() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("source.inclusion", "all"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.ip", receiverIp); @@ -389,22 +361,28 @@ public void testAutoManualCreateRace() throws Exception { TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); TestUtils.executeNonQuery( - receiverEnv, "create timeSeries root.ln.wf01.wt01.status with datatype=BOOLEAN", null); + receiverEnv, + isolateDatabasePaths("create timeSeries root.ln.wf01.wt01.status with datatype=BOOLEAN"), + null); TestUtils.executeNonQuery( senderEnv, - "create timeSeries root.ln.wf01.wt01.status with datatype=BOOLEAN tags (tag3=v3) attributes (attr4=v4)", + isolateDatabasePaths( + "create timeSeries root.ln.wf01.wt01.status with datatype=BOOLEAN tags (tag3=v3) attributes (attr4=v4)"), null); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "show timeSeries root.ln.**", + isolateDatabasePaths("show timeSeries root.ln.**"), "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", Collections.singleton( - "root.ln.wf01.wt01.status,null,root.ln,BOOLEAN,RLE,LZ4,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,")); + isolateDatabasePaths( + "root.ln.wf01.wt01.status,null,root.ln,BOOLEAN,RLE,LZ4,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,"))); } } + // Historical sync must preserve an aligned template binding and all three activated devices, + // including a device activated by a negative timestamp and a later template extension. @Test public void testHistoricalActivationRace() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -418,15 +396,22 @@ public void testHistoricalActivationRace() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "create database root.sg_aligned", + isolateDatabasePaths("create database root.sg_aligned"), "create device template aligned_template aligned (s0 int32, s1 int64, s2 float, s3 double, s4 boolean, s5 text)", - "set device template aligned_template to root.sg_aligned.device_aligned", - "create timeseries using device template on root.sg_aligned.device_aligned.d10", - "create timeseries using device template on root.sg_aligned.device_aligned.d12", - "insert into root.sg_aligned.device_aligned.d10(time, s0, s1, s2,s3,s4,s5) values (1706659200,1706659200,10,20.245,25.24555,true,''),(1706662800,null,1706662800,20.241,25.24111,false,'2'),(1706666400,3,null,20.242,25.24222,true,'3'),(1706670000,4,40,null,35.5474,true,'4'),(1706670600,5,1706670600000,20.246,null,false,'5'),(1706671200,6,60,20.248,25.24888,null,'6'),(1706671800,7,1706671800,20.249,25.24999,false,null),(1706672400,8,80,1245.392,75.51234,false,'8'),(1706672600,9,90,2345.397,2285.58734,false,'9'),(1706673000,10,100,20.241,25.24555,false,'10'),(1706673600,11,110,3345.394,4105.544,false,'11'),(1706674200,12,1706674200,30.245,35.24555,false,'12'),(1706674800,13,130,5.39,125.51234,false,'13'),(1706675400,14,1706675400,5.39,135.51234,false,'14'),(1706676000,15,150,5.39,145.51234,false,'15'),(1706676600,16,160,5.39,155.51234,false,'16'),(1706677200,17,170,5.39,165.51234,false,'17'),(1706677600,18,180,5.39,175.51234,false,'18'),(1706677800,19,190,5.39,185.51234,false,'19'),(1706678000,20,200,5.39,195.51234,false,'20'),(1706678200,21,210,5.39,null,false,'21')", - "insert into root.sg_aligned.device_aligned.d10(time, s0, s1, s2,s3,s4,s5) values (-1,1,10,5.39,5.51234,false,'negative')", - "insert into root.sg_aligned.device_aligned.d11(time, s0, s1, s2,s3,s4,s5) values (-1,-11,-110,-5.39,-5.51234,false,'activate:1')", - "insert into root.sg_aligned.device_aligned.d10(time, s0, s1, s2,s3,s4,s5,s6) values(1706678800,1,1706678800,5.39,5.51234,false,'add:s6',32);"), + isolateDatabasePaths( + "set device template aligned_template to root.sg_aligned.device_aligned"), + isolateDatabasePaths( + "create timeseries using device template on root.sg_aligned.device_aligned.d10"), + isolateDatabasePaths( + "create timeseries using device template on root.sg_aligned.device_aligned.d12"), + isolateDatabasePaths( + "insert into root.sg_aligned.device_aligned.d10(time, s0, s1, s2,s3,s4,s5) values (1706659200,1706659200,10,20.245,25.24555,true,''),(1706662800,null,1706662800,20.241,25.24111,false,'2'),(1706666400,3,null,20.242,25.24222,true,'3'),(1706670000,4,40,null,35.5474,true,'4'),(1706670600,5,1706670600000,20.246,null,false,'5'),(1706671200,6,60,20.248,25.24888,null,'6'),(1706671800,7,1706671800,20.249,25.24999,false,null),(1706672400,8,80,1245.392,75.51234,false,'8'),(1706672600,9,90,2345.397,2285.58734,false,'9'),(1706673000,10,100,20.241,25.24555,false,'10'),(1706673600,11,110,3345.394,4105.544,false,'11'),(1706674200,12,1706674200,30.245,35.24555,false,'12'),(1706674800,13,130,5.39,125.51234,false,'13'),(1706675400,14,1706675400,5.39,135.51234,false,'14'),(1706676000,15,150,5.39,145.51234,false,'15'),(1706676600,16,160,5.39,155.51234,false,'16'),(1706677200,17,170,5.39,165.51234,false,'17'),(1706677600,18,180,5.39,175.51234,false,'18'),(1706677800,19,190,5.39,185.51234,false,'19'),(1706678000,20,200,5.39,195.51234,false,'20'),(1706678200,21,210,5.39,null,false,'21')"), + isolateDatabasePaths( + "insert into root.sg_aligned.device_aligned.d10(time, s0, s1, s2,s3,s4,s5) values (-1,1,10,5.39,5.51234,false,'negative')"), + isolateDatabasePaths( + "insert into root.sg_aligned.device_aligned.d11(time, s0, s1, s2,s3,s4,s5) values (-1,-11,-110,-5.39,-5.51234,false,'activate:1')"), + isolateDatabasePaths( + "insert into root.sg_aligned.device_aligned.d10(time, s0, s1, s2,s3,s4,s5,s6) values(1706678800,1,1706678800,5.39,5.51234,false,'add:s6',32);")), null); final Map sourceAttributes = new HashMap<>(); @@ -434,7 +419,7 @@ public void testHistoricalActivationRace() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("source.inclusion", "all"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.ip", receiverIp); @@ -455,12 +440,101 @@ public void testHistoricalActivationRace() throws Exception { receiverEnv, "show paths set device template aligned_template", "Paths,", - Collections.singleton("root.sg_aligned.device_aligned,")); + Collections.singleton(isolateDatabasePaths("root.sg_aligned.device_aligned,"))); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "count devices root.sg_aligned.**", + isolateDatabasePaths("count devices root.sg_aligned.**"), "count(devices),", Collections.singleton("3,")); } } + + private String isolateDatabasePaths(final String sql) { + // A stable short suffix isolates each method without changing device names or template scope. + return sql.replaceAll( + "root\\.(db|ln|sg_aligned)(?=[.,; ]|$)", + "$0_" + Integer.toUnsignedString(testName.getMethodName().hashCode(), 16)); + } + + private ISession writeSession(final BaseEnv env) throws Exception { + if (env == senderEnv) { + if (senderSession == null) { + senderSession = env.getSessionConnection(); + } + return senderSession; + } + if (receiverSession == null) { + receiverSession = env.getSessionConnection(); + } + return receiverSession; + } + + private void insertData(final BaseEnv env, final int start, final int end, final boolean flush) + throws Exception { + final ISession session = writeSession(env); + // Keep the point count and phase boundaries, while reducing INSERT RPCs to one tenth. + for (int batchStart = start; batchStart < end; batchStart += 10) { + final StringBuilder sql = + new StringBuilder(isolateDatabasePaths("insert into root.db.d1(time, s1) values ")); + for (int i = batchStart; i < Math.min(batchStart + 10, end); ++i) { + if (i > batchStart) { + sql.append(','); + } + sql.append('(').append(i).append(",1)"); + } + session.executeNonQueryStatement(sql.toString()); + } + if (flush) { + session.executeNonQueryStatement("flush"); + } + } + + private void closeWriteSessions() throws Exception { + try { + if (senderSession != null) { + senderSession.close(); + } + } finally { + senderSession = null; + try { + if (receiverSession != null) { + receiverSession.close(); + } + } finally { + receiverSession = null; + } + } + } + + @Override + protected void cleanEnvironment() { + try { + closeWriteSessions(); + } catch (final Exception e) { + throw new AssertionError("Failed to close conflict test sessions", e); + } finally { + // Drop pipes before databases so neither side can recreate data while cleanup runs. + super.cleanEnvironment(); + if (clustersReady) { + // Deleting databases removes bindings/activations, but template definitions survive it. + // Reset them explicitly to prevent one method's schema from changing another method. + for (final BaseEnv env : new BaseEnv[] {senderEnv, receiverEnv}) { + try (final Connection connection = env.getConnection(); + final Statement statement = connection.createStatement()) { + final List templates = new ArrayList<>(); + try (final ResultSet result = statement.executeQuery("show schema templates")) { + while (result.next()) { + templates.add(result.getString(1)); + } + } + for (final String template : templates) { + statement.execute("drop schema template `" + template.replace("`", "``") + "`"); + } + } catch (final Exception e) { + throw new AssertionError("Failed to reset conflict test templates", e); + } + } + } + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java index 19b30f628d94d..f07d96d9e3fd9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; @@ -31,16 +32,21 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -53,20 +59,73 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeAutoDropIT extends AbstractPipeDualTreeModelAutoIT { - - @Override - @Before - public void setUp() { - super.setUp(); - } +public class IoTDBPipeAutoDropIT extends AbstractPipeTreeModelSharedClusterIT { @Override protected void setupConfig() { super.setupConfig(); + // Auto-drop waits for completion reports, so collect them frequently in this IT. + senderEnv.getConfig().getCommonConfig().setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(1); senderEnv.getConfig().getConfigNodeConfig().setLeaderDistributionPolicy("HASH"); } + @Override + protected void cleanEnvironment() { + try { + // The unmatched-region scenario creates a table-model pipe while running on the shared + // tree clusters. Drop it before the shared tree cleanup so it cannot recreate the table data. + dropTablePipes(senderEnv); + dropTablePipes(receiverEnv); + super.cleanEnvironment(); + deleteTableDatabases(senderEnv); + deleteTableDatabases(receiverEnv); + } catch (Exception e) { + throw new AssertionError("Failed to reset table state in shared auto-drop clusters", e); + } + } + + private static void dropTablePipes(final BaseEnv env) throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { + final TShowPipeResp response = + client.showPipe( + new TShowPipeReq().setIsTableModel(true).setUserName(SessionConfig.DEFAULT_USER)); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + for (final TShowPipeInfo pipe : response.getPipeInfoList()) { + if (!pipe.getId().startsWith(PipeStaticMeta.SYSTEM_PIPE_PREFIX)) { + final int statusCode = + client + .dropPipeExtended(new TDropPipeReq(pipe.getId()).setIsTableModel(true)) + .getCode(); + // Automatic drop can remove a pipe after the SHOW PIPE snapshot. + if (statusCode != TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode()) { + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), statusCode); + } + } + } + } + } + + private static void deleteTableDatabases(final BaseEnv env) throws SQLException { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final List databases = new ArrayList<>(); + try (final ResultSet result = statement.executeQuery("show databases")) { + while (result.next()) { + final String database = result.getString(1); + if (!"information_schema".equals(database)) { + databases.add(database); + } + } + } + for (final String database : databases) { + TestUtils.dropDatabaseWithRetry(statement, "\"" + database.replace("\"", "\"\"") + "\""); + } + } + } + + // A query-mode pipe must transfer the historical row and remove itself after completion. @Test public void testAutoDropInHistoricalTransfer() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -88,7 +147,7 @@ public void testAutoDropInHistoricalTransfer() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("source.mode", "query"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.batch.enable", "false"); @@ -102,8 +161,8 @@ public void testAutoDropInHistoricalTransfer() throws Exception { .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + // Creation already starts finite pipes; another start can race with their auto-drop. TestUtils.assertDataEventuallyOnEnv( receiverEnv, @@ -114,8 +173,8 @@ public void testAutoDropInHistoricalTransfer() throws Exception { // Fetch a fresh ConfigNode snapshot on every poll while pipes are being removed. await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(600, TimeUnit.SECONDS) .untilAsserted( () -> { @@ -131,6 +190,8 @@ public void testAutoDropInHistoricalTransfer() throws Exception { } } + // With a fresh sender that has no DataRegions, query/history pipes must disappear while a + // realtime-only pipe stays alive. Reusing a cluster could leave regions from another scenario. @Test public void testAutoDropFinitePipesWithoutDataRegion() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -179,7 +240,8 @@ public void testAutoDropFinitePipesWithoutDataRegion() throws Exception { // their auto-drop when they have no DataRegions to process. await() .pollInSameThread() - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(600, TimeUnit.SECONDS) .untilAsserted( () -> { @@ -206,6 +268,8 @@ public void testAutoDropFinitePipesWithoutDataRegion() throws Exception { } } + // Tree and table query pipes must transfer their matching row and auto-drop despite the sender's + // other, unmatched DataRegions in both models. @Test public void testAutoDropIgnoredUnmatchedDataRegions() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -244,7 +308,7 @@ public void testAutoDropIgnoredUnmatchedDataRegions() throws Exception { treeSourceAttributes.put("source.path", "root.db.**"); treeSourceAttributes.put("source.capture.tree", "true"); treeSourceAttributes.put("source.capture.table", "false"); - treeSourceAttributes.put("user", "root"); + treeSourceAttributes.put("user", SessionConfig.DEFAULT_USER); TSStatus status = client.createPipe( @@ -253,8 +317,6 @@ public void testAutoDropIgnoredUnmatchedDataRegions() throws Exception { .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p_tree").getCode()); // Table-model pipe: only listens to table_db, which must not be blocked by the tree // DataRegions either. @@ -265,7 +327,7 @@ public void testAutoDropIgnoredUnmatchedDataRegions() throws Exception { tableSourceAttributes.put("source.capture.tree", "false"); tableSourceAttributes.put("source.capture.table", "true"); tableSourceAttributes.put("__system.sql-dialect", "table"); - tableSourceAttributes.put("user", "root"); + tableSourceAttributes.put("user", SessionConfig.DEFAULT_USER); status = client.createPipe( @@ -274,8 +336,6 @@ public void testAutoDropIgnoredUnmatchedDataRegions() throws Exception { .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p_table").getCode()); TestUtils.assertDataEventuallyOnEnv( receiverEnv, @@ -286,8 +346,8 @@ public void testAutoDropIgnoredUnmatchedDataRegions() throws Exception { await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(600, TimeUnit.SECONDS) .untilAsserted( () -> { @@ -308,6 +368,8 @@ public void testAutoDropIgnoredUnmatchedDataRegions() throws Exception { } } + // Of five historical rows, the query pipe must transfer only the three in [2000, 4000] and then + // remove itself, preserving the inclusive start/end-time boundaries. @Test public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -332,7 +394,7 @@ public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { sourceAttributes.put("source.mode", "query"); sourceAttributes.put("source.start-time", "1970-01-01T08:00:02+08:00"); sourceAttributes.put("source.end-time", "1970-01-01T08:00:04+08:00"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.batch.enable", "false"); @@ -346,8 +408,6 @@ public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); TestUtils.assertDataEventuallyOnEnv( receiverEnv, @@ -357,8 +417,8 @@ public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(600, TimeUnit.SECONDS) .untilAsserted( () -> { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java index f17f63214cf15..4c74ce765425d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java @@ -39,6 +39,7 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.PipeTestUtils; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; @@ -71,8 +72,6 @@ @Category({MultiClusterIT2DualTreeAutoEnhanced.class}) public class IoTDBPipeClusterIT extends AbstractPipeDualTreeModelAutoIT { - private static final double SYNC_LAG_DELTA = 0.001; - @Override @Before public void setUp() { @@ -85,6 +84,8 @@ public void setUp() { .getCommonConfig() .setAutoCreateSchemaEnabled(true) .setDatanodeMemoryProportion("3:3:1:1:1:0") + // Retain AUTO expansion when nodes join without scaling tiny IT databases to host CPUs. + .setDataRegionPerDataNode(2) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) @@ -93,7 +94,7 @@ public void setUp() { senderEnv .getConfig() .getDataNodeConfig() - .setMetricReporterType(Collections.singletonList("PROMETHEUS")); + .setMetricReporterType(Collections.singletonList("JMX")); senderEnv.getConfig().getConfigNodeConfig().setLeaderDistributionPolicy("HASH"); receiverEnv @@ -101,6 +102,7 @@ public void setUp() { .getCommonConfig() .setAutoCreateSchemaEnabled(true) .setDatanodeMemoryProportion("3:3:1:1:1:0") + .setDataRegionPerDataNode(2) .setDataReplicationFactor(2) .setSchemaReplicationFactor(3) .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) @@ -111,14 +113,13 @@ public void setUp() { receiverEnv .getConfig() .getDataNodeConfig() - .setMetricReporterType(Collections.singletonList("PROMETHEUS")); + .setMetricReporterType(Collections.singletonList("JMX")); // 10 min, assert that the operations will not time out senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); - senderEnv.initClusterEnvironment(3, 3, 180); - receiverEnv.initClusterEnvironment(3, 3, 180); + executeOnBothEnvs(env -> env.initClusterEnvironment(3, 3, 180)); } @Test @@ -295,6 +296,9 @@ public void testWithAllParameters(final String realtimeMode) throws Exception { } } + // Writes after a DataRegion leader restart must reach the receiver; after both clusters restart, + // a new pipe on another device must still transfer its row. Read live replication gauges on every + // replica before flushing/restarting, so cached zeros cannot hide unreplicated writes. @Test public void testPipeAfterDataRegionLeaderStop() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -354,13 +358,7 @@ public void testPipeAfterDataRegionLeaderStop() throws Exception { receiverEnv, Collections.singletonList("root.db")); } - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } + executeOnBothEnvs(TestUtils::restartCluster); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { @@ -460,7 +458,8 @@ private void waitForTreeDataRegionReplicationComplete( // IoTConsensus accepts writes on any replica, including a previous leader during failover. // Check every replica's outgoing replication before flushing or restarting the cluster. await() - .pollInterval(500, TimeUnit.MILLISECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .atMost(2, TimeUnit.MINUTES) .untilAsserted( () -> { @@ -471,21 +470,7 @@ private void waitForTreeDataRegionReplicationComplete( "No tree DataRegion found for databases " + databases, regionInfoList.isEmpty()); - for (final TRegionInfo regionInfo : regionInfoList) { - final DataNodeWrapper replicaNode = - findDataNodeWrapperByPort(env, regionInfo.getClientRpcPort()); - final String metricsUrl = - "http://" - + replicaNode.getIp() - + ":" - + replicaNode.getMetricPort() - + "/metrics"; - final String metricsContent = env.getUrlContent(metricsUrl, null); - Assert.assertNotNull( - "Failed to fetch metrics from replica DataNode at " + metricsUrl, - metricsContent); - assertSyncLagIsZero(metricsContent, buildDataRegionTag(regionInfo), metricsUrl); - } + PipeTestUtils.assertDataRegionReplicationComplete(env, regionInfoList); } }); } @@ -513,40 +498,8 @@ private List showTreeDataRegions( return showRegionResp.getRegionInfoList(); } - private DataNodeWrapper findDataNodeWrapperByPort(final BaseEnv env, final int port) { - for (final DataNodeWrapper dataNodeWrapper : env.getDataNodeWrapperList()) { - if (dataNodeWrapper.getPort() == port) { - return dataNodeWrapper; - } - } - fail("Failed to find DataNodeWrapper for client rpc port " + port); - return null; - } - - private String buildDataRegionTag(final TRegionInfo regionInfo) { - return "DataRegion[" + regionInfo.getConsensusGroupId().getId() + "]"; - } - - private void assertSyncLagIsZero( - final String metricsContent, final String dataRegionTag, final String metricsUrl) { - for (final String line : metricsContent.split("\\R")) { - if (!line.startsWith("iot_consensus{") - || !line.contains("type=\"syncLag\"") - || !line.contains("region=\"" + dataRegionTag + "\"")) { - continue; - } - final int lastSpaceIndex = line.lastIndexOf(' '); - Assert.assertTrue("Malformed syncLag metric line: " + line, lastSpaceIndex > 0); - Assert.assertEquals( - "Expected syncLag of " + dataRegionTag + " to be 0 at " + metricsUrl + " but got " + line, - 0.0, - Double.parseDouble(line.substring(lastSpaceIndex + 1)), - SYNC_LAG_DELTA); - return; - } - fail("No syncLag metric found for " + dataRegionTag + " at " + metricsUrl); - } - + // Adding a sender DataNode must preserve replication of writes through that node, and after both + // clusters restart, a new pipe must also transfer writes on a second device. @Test public void testPipeAfterRegisterNewDataNode() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -583,12 +536,7 @@ public void testPipeAfterRegisterNewDataNode() throws Exception { Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"), null); - try { - senderEnv.registerNewDataNode(true); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } + senderEnv.registerNewDataNode(true); final DataNodeWrapper newDataNode = senderEnv.getDataNodeWrapper(senderEnv.getDataNodeWrapperList().size() - 1); TestUtils.tryExecuteNonQueriesOnSpecifiedDataNodeWithRetry( @@ -602,13 +550,7 @@ public void testPipeAfterRegisterNewDataNode() throws Exception { Collections.singleton("2,")); } - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } + executeOnBothEnvs(TestUtils::restartCluster); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java index 7d0e7f71c0a87..ce96d436be441 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java @@ -26,10 +26,9 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -43,14 +42,10 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeConditionalOperationsIT extends AbstractPipeDualTreeModelAutoIT { - - @Override - @Before - public void setUp() { - super.setUp(); - } +public class IoTDBPipeConditionalOperationsIT extends AbstractPipeTreeModelSharedClusterIT { + // Repeating CREATE PIPE IF NOT EXISTS with different source attributes must preserve the + // running pipe's original configuration and creation time. @Test public void testBasicCreatePipeIfNotExists() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -60,54 +55,42 @@ public void testBasicCreatePipeIfNotExists() throws Exception { String.format( "create pipe If Not Exists a2b with source ('source'='iotdb-source', 'source.pattern'='root.test1', 'source.realtime.mode'='stream') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", receiverDataNode.getIpAndPortString()); + // Keep both clients across the repeated command and its metadata checks. try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + final Statement statement = connection.createStatement(); + final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { statement.execute(sql); - } catch (SQLException e) { - fail(e.getMessage()); - } - // show pipe - long creationTime; - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = + // show pipe + final List createdPipes = client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; - showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(1, showPipeResult.size()); + createdPipes.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(1, createdPipes.size()); // Check status - Assert.assertEquals("RUNNING", showPipeResult.get(0).state); + Assert.assertEquals("RUNNING", createdPipes.get(0).state); // Check configurations - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test1")); + Assert.assertTrue(createdPipes.get(0).pipeExtractor.contains("source=iotdb-source")); + Assert.assertTrue(createdPipes.get(0).pipeExtractor.contains("source.pattern=root.test1")); + Assert.assertTrue(createdPipes.get(0).pipeExtractor.contains("source.realtime.mode=stream")); Assert.assertTrue( - showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); - Assert.assertTrue( - showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); + createdPipes.get(0).pipeProcessor.contains("processor=do-nothing-processor")); Assert.assertTrue( - showPipeResult + createdPipes .get(0) .pipeConnector .contains(String.format("node-urls=%s", receiverDataNode.getIpAndPortString()))); // Record last creation time - creationTime = showPipeResult.get(0).creationTime; - } + final long creationTime = createdPipes.get(0).creationTime; - // Create pipe If Not Exists - sql = - String.format( - "create pipe If Not Exists a2b with source ('source'='iotdb-source', 'source.path'='root.test2.**') with sink ('node-urls'='%s')", - receiverDataNode.getIpAndPortString()); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Create pipe If Not Exists + sql = + String.format( + "create pipe If Not Exists a2b with source ('source'='iotdb-source', 'source.path'='root.test2.**') with sink ('node-urls'='%s')", + receiverDataNode.getIpAndPortString()); statement.execute(sql); - } catch (SQLException e) { - fail(e.getMessage()); - } - // show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // show pipe final List showPipeResult = client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); @@ -131,6 +114,7 @@ public void testBasicCreatePipeIfNotExists() throws Exception { } } + // DROP PIPE IF EXISTS must remove a running pipe and succeed again after the pipe is absent. @Test public void testBasicDropPipeIfExists() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -140,36 +124,24 @@ public void testBasicDropPipeIfExists() throws Exception { String.format( "create pipe If Not Exists a2b with source ('source'='iotdb-source', 'source.path'='root.test1.**') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", receiverDataNode.getIpAndPortString()); + // Reuse the SQL connection through both DROP calls and keep the intervening absence check. try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + final Statement statement = connection.createStatement(); + final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { statement.execute(sql); - } catch (SQLException e) { - fail(e.getMessage()); - } - // Drop pipe If Exists - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Drop pipe If Exists statement.execute("Drop pipe If Exists a2b"); - } catch (SQLException e) { - fail(e.getMessage()); - } - // show pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // show pipe final List showPipeResult = client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); Assert.assertEquals(0, showPipeResult.size()); - } - // Drop pipe If Exists - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { + // Drop pipe If Exists statement.execute("Drop pipe If Exists a2b"); - } catch (SQLException e) { - fail(e.getMessage()); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java index fb8945471b77e..5f5e605d38828 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java @@ -24,37 +24,51 @@ import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeIdempotentIT extends AbstractPipeDualTreeModelAutoIT { +public class IoTDBPipeIdempotentIT extends AbstractPipeTreeModelSharedClusterIT { + + @Rule public final TestName testName = new TestName(); + + private boolean clustersReady; @Override @Before public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); + super.setUp(); + clustersReady = true; + } + @Override + protected void setupConfig() { // All the schema operations must be under the same database to // be in the same region, therefore a non-idempotent operation can block the next one // and fail the IT @@ -90,11 +104,62 @@ public void setUp() { // 10 min, assert that the operations will not time out senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); + } - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); + @Override + protected void cleanEnvironment() { + super.cleanEnvironment(); + if (!clustersReady) { + return; + } + // The base drops pipes before databases, removing template bindings and activations as well. + // Templates and authorization outlive databases, so reset them before the next scenario. + for (final BaseEnv env : new BaseEnv[] {senderEnv, receiverEnv}) { + try (final Connection connection = env.getConnection(); + final Statement statement = connection.createStatement()) { + final List templates = new ArrayList<>(); + try (final ResultSet result = statement.executeQuery("show schema templates")) { + while (result.next()) { + templates.add(result.getString(1)); + } + } + for (final String template : templates) { + statement.execute("drop schema template `" + template.replace("`", "``") + "`"); + } + for (final String type : new String[] {"user", "role"}) { + final List names = new ArrayList<>(); + try (final ResultSet result = statement.executeQuery("list " + type)) { + while (result.next()) { + final String name = result.getString("user".equals(type) ? "User" : "Role"); + if (!"user".equals(type) || !SessionConfig.DEFAULT_USER.equals(name)) { + names.add(name); + } + } + } + for (final String name : names) { + statement.execute("drop " + type + " `" + name.replace("`", "``") + "`"); + } + } + } catch (final Exception e) { + throw new AssertionError("Failed to reset idempotent test metadata", e); + } + } } + private String databasePattern() { + // Count only this method's suffixed databases, including its setup and follow-up events. + return "root.*_" + Integer.toUnsignedString(testName.getMethodName().hashCode(), 16); + } + + private String isolateDatabasePaths(final String sql) { + // Keep all operations within each original database on one schema region. A short method hash + // isolates different methods without exceeding the receiver's database path length limit. + return sql.replaceAll( + "root\\.(ln|sg[0-9]*|t1)(?=[.,; ]|$)", + "$0_" + Integer.toUnsignedString(testName.getMethodName().hashCode(), 16)); + } + + // Creating an existing BOOLEAN series must allow the following series to sync (two total). @Test public void testCreateTimeSeriesIdempotent() throws Exception { testIdempotent( @@ -106,6 +171,7 @@ public void testCreateTimeSeriesIdempotent() throws Exception { Collections.singleton("2,")); } + // Recreating an aligned pair must allow a later scalar series to sync (three total). @Test public void testCreateAlignedTimeSeriesIdempotent() throws Exception { testIdempotent( @@ -117,6 +183,7 @@ public void testCreateAlignedTimeSeriesIdempotent() throws Exception { Collections.singleton("3,")); } + // Recreating a series with the same alias must allow a second aliased series to sync. @Test public void testCreateTimeSeriesWithAliasIdempotent() throws Exception { testIdempotent( @@ -128,6 +195,7 @@ public void testCreateTimeSeriesWithAliasIdempotent() throws Exception { Collections.singleton("2,")); } + // An insert that auto-creates an existing series must not block later schema replication. @Test public void testInternalCreateTimeSeriesIdempotent() throws Exception { testIdempotent( @@ -139,6 +207,7 @@ public void testInternalCreateTimeSeriesIdempotent() throws Exception { Collections.singleton("2,")); } + // Adding a tag already present on the receiver must not block the following series creation. @Test public void testAlterTimeSeriesAddTagIdempotent() throws Exception { testIdempotent( @@ -151,6 +220,7 @@ public void testAlterTimeSeriesAddTagIdempotent() throws Exception { Collections.singleton("2,")); } + // Adding an existing attribute must allow the following series to sync (two total). @Test public void testAlterTimeSeriesAddAttrIdempotent() throws Exception { testIdempotent( @@ -163,6 +233,7 @@ public void testAlterTimeSeriesAddAttrIdempotent() throws Exception { Collections.singleton("2,")); } + // Renaming an attribute already renamed on the receiver must allow later schema to sync. @Test public void testAlterTimeSeriesRenameIdempotent() throws Exception { testIdempotent( @@ -176,6 +247,7 @@ public void testAlterTimeSeriesRenameIdempotent() throws Exception { Collections.singleton("2,")); } + // Deleting a series already absent on the receiver must allow the next database to sync. @Test public void testDeleteTimeSeriesIdempotent() throws Exception { testIdempotent( @@ -183,22 +255,24 @@ public void testDeleteTimeSeriesIdempotent() throws Exception { "create timeSeries root.ln.wf01.wt01.status0(status0) with datatype=BOOLEAN,encoding=PLAIN"), "delete timeSeries root.ln.wf01.wt01.status0", "create database root.sg", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("2,")); } + // Creating an existing template must allow the following database creation to sync. @Test public void testCreateTemplateIdempotent() throws Exception { testIdempotent( Collections.emptyList(), "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)", "create database root.sg1", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("1,")); } + // Extending a template with existing measurements must allow later database events to sync. @Test public void testExtendTemplateIdempotent() throws Exception { testIdempotent( @@ -206,11 +280,12 @@ public void testExtendTemplateIdempotent() throws Exception { "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)"), "alter schema template t1 add (rest FLOAT encoding=RLE, FLOAT2 TEXT encoding=PLAIN compression=SNAPPY)", "create database root.sg1", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("1,")); } + // Dropping an already absent template must allow the following database to sync. @Test public void testDropTemplateIdempotent() throws Exception { testIdempotent( @@ -218,11 +293,12 @@ public void testDropTemplateIdempotent() throws Exception { "create schema template t1 (s1 INT64 encoding=RLE, s2 INT64 encoding=RLE, s3 INT64 encoding=RLE compression=SNAPPY)"), "drop schema template t1", "create database root.sg1", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("1,")); } + // Binding a template already bound on the receiver must allow a second database to sync. @Test public void testSetTemplateIdempotent() throws Exception { testIdempotent( @@ -231,11 +307,12 @@ public void testSetTemplateIdempotent() throws Exception { "create database root.sg1"), "set schema template t1 to root.sg1", "create database root.sg2", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("2,")); } + // Unbinding an already unbound template must allow a second database to sync. @Test public void testUnsetTemplateIdempotent() throws Exception { testIdempotent( @@ -245,11 +322,12 @@ public void testUnsetTemplateIdempotent() throws Exception { "set schema template t1 to root.sg1"), "unset schema template t1 from root.sg1", "create database root.sg2", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("2,")); } + // Activating an already active device must allow a second device activation (six series total). @Test public void testActivateTemplateIdempotent() throws Exception { testIdempotent( @@ -264,6 +342,7 @@ public void testActivateTemplateIdempotent() throws Exception { Collections.singleton("6,")); } + // Deactivating template devices already inactive on the receiver must allow later database sync. @Test public void testDeactivateTemplateIdempotent() throws Exception { testIdempotent( @@ -275,133 +354,144 @@ public void testDeactivateTemplateIdempotent() throws Exception { "create timeSeries using device template on root.sg1.d2"), "delete timeSeries of schema template t1 from root.sg1.*", "create database root.sg2", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("2,")); } + // Creating an existing database must allow the second database to sync (two total). @Test public void testCreateDatabaseIdempotent() throws Exception { testIdempotent( Collections.emptyList(), "create database root.sg1", "create database root.sg2", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("2,")); } + // Reapplying schema/data region limits must allow a second database to sync. @Test public void testAlterDatabaseIdempotent() throws Exception { testIdempotent( Collections.singletonList("create database root.sg1"), "ALTER DATABASE root.sg1 WITH MAX_SCHEMA_REGION_GROUP_NUM=2, MAX_DATA_REGION_GROUP_NUM=3;", "create database root.sg2", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("2,")); } + // Dropping an already absent database must allow the next creation to sync (one remaining). @Test public void testDropDatabaseIdempotent() throws Exception { testIdempotent( Collections.singletonList("create database root.sg1"), "drop database root.sg1", "create database root.sg2", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("1,")); } + // Creating an existing user must not block the subsequent database event. @Test public void testCreateUserIdempotent() throws Exception { testIdempotent( Collections.emptyList(), - "create user `ln_write_user` 'write_pwd123456'", + "create user `ln_write_user` 'Write_pwd!123456'", "create database root.sg1", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("1,")); } + // Creating an existing role must not block the subsequent database event. @Test public void testCreateRoleIdempotent() throws Exception { testIdempotent( Collections.emptyList(), "create role `test`", "create database root.sg1", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("1,")); } + // Granting an already granted role membership must allow later database events to sync. @Test public void testGrantRoleToUserIdempotent() throws Exception { testIdempotent( - Arrays.asList("create user `ln_write_user` 'write_pwd123456'", "create role `test`"), + Arrays.asList("create user `ln_write_user` 'Write_pwd!123456'", "create role `test`"), "grant role test to ln_write_user", "create database root.sg1", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("1,")); } + // Revoking already absent user path privileges must allow the next database event to sync. @Test public void testRevokeUserIdempotent() throws Exception { testIdempotent( Arrays.asList( - "create user `ln_write_user` 'write_pwd123456'", + "create user `ln_write_user` 'Write_pwd!123456'", "GRANT READ_DATA, WRITE_DATA ON root.t1.** TO USER ln_write_user;"), "REVOKE READ_DATA, WRITE_DATA ON root.t1.** FROM USER ln_write_user;", "create database root.sg1", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("1,")); } + // Revoking already absent role privileges must allow the next database event to sync. @Test public void testRevokeRoleIdempotent() throws Exception { testIdempotent( Arrays.asList("create role `test`", "GRANT READ ON root.** TO ROLE test;"), "REVOKE READ ON root.** FROM ROLE test;", "create database root.sg1", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("1,")); } + // Revoking an already absent role membership must allow later database events to sync. @Test public void testRevokeRoleFromUserIdempotent() throws Exception { testIdempotent( Arrays.asList( - "create user `ln_write_user` 'write_pwd123456'", + "create user `ln_write_user` 'Write_pwd!123456'", "create role `test`", "grant role test to ln_write_user"), "revoke role test from ln_write_user", "create database root.sg1", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("1,")); } + // Dropping an already absent user must not block the subsequent database event. @Test public void testDropUserIdempotent() throws Exception { testIdempotent( - Collections.singletonList("create user `ln_write_user` 'write_pwd123456'"), + Collections.singletonList("create user `ln_write_user` 'Write_pwd!123456'"), "drop user `ln_write_user`", "create database root.sg1", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("1,")); } + // Dropping an already absent role must not block the subsequent database event. @Test public void testDropRoleIdempotent() throws Exception { testIdempotent( Collections.singletonList("create role `test`"), "drop role test", "create database root.sg1", - "count databases", + "count databases " + databasePattern(), "count,", Collections.singleton("1,")); } @@ -428,7 +518,7 @@ private void testIdempotent( sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); sourceAttributes.put("source.forwarding-pipe-requests", "false"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.ip", receiverIp); @@ -449,14 +539,31 @@ private void testIdempotent( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); } - TestUtils.executeNonQueries(senderEnv, beforeSqlList, null); - - TestUtils.tryExecuteNonQuery(receiverEnv, testSql, null); - - // Create an idempotent conflict, after sql shall be executed on the same region as testSql - TestUtils.executeNonQueries(senderEnv, Arrays.asList(testSql, afterSql), null); + try (final Connection senderConnection = senderEnv.getConnection(); + final Connection receiverConnection = receiverEnv.getConnection()) { + TestUtils.executeNonQueries( + senderEnv, + beforeSqlList.stream().map(this::isolateDatabasePaths).collect(Collectors.toList()), + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + senderConnection); + + TestUtils.tryExecuteNonQuery(receiverEnv, isolateDatabasePaths(testSql), receiverConnection); + + // Create an idempotent conflict; afterSql must run on the same region as testSql. + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList(isolateDatabasePaths(testSql), isolateDatabasePaths(afterSql)), + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + senderConnection); + } // Assume that the afterSql is executed on receiverEnv - TestUtils.assertDataEventuallyOnEnv(receiverEnv, afterSqlQuery, expectedHeader, expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + isolateDatabasePaths(afterSqlQuery), + isolateDatabasePaths(expectedHeader), + expectedResSet); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java index e3f6b3a957793..5fe4344550a80 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java @@ -27,9 +27,7 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; @@ -38,22 +36,24 @@ import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; -import java.util.function.Consumer; - -import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeNullValueIT extends AbstractPipeDualTreeModelAutoIT { +public class IoTDBPipeNullValueIT extends AbstractPipeTreeModelSharedClusterIT { + + @Rule public final TestName testName = new TestName(); // Test dimensions: // 1. is or not aligned @@ -69,8 +69,6 @@ private enum InsertType { SQL_INSERT, } - private static final Map> INSERT_NULL_VALUE_MAP = new HashMap<>(); - private static final List CREATE_TIMESERIES_SQL = Arrays.asList( "create timeseries root.sg.d1.s0 with datatype=float", @@ -79,7 +77,7 @@ private enum InsertType { private static final List CREATE_ALIGNED_TIMESERIES_SQL = Collections.singletonList("create aligned timeseries root.sg.d1(s0 float, s1 float)"); - private final String deviceId = "root.sg.d1"; + private String deviceId; private final List measurements = Arrays.asList("s0", "s1"); private final List types = Arrays.asList(TSDataType.FLOAT, TSDataType.FLOAT); @@ -90,6 +88,7 @@ private enum InsertType { private Tablet allNullTablet; private void constructTablet() { + deviceId = databasePath() + ".d1"; final MeasurementSchema[] schemas = new MeasurementSchema[2]; for (int i = 0; i < schemas.length; i++) { schemas[i] = new MeasurementSchema(measurements.get(i), types.get(i)); @@ -137,72 +136,46 @@ private void constructTablet() { @Before public void setUp() { super.setUp(); - constructTablet(); + } - // Initialize INSERT_NULL_VALUE_MAP - INSERT_NULL_VALUE_MAP.put( - InsertType.SESSION_INSERT_RECORD, - (isAligned) -> { - try { - try (final ISession session = senderEnv.getSessionConnection()) { - if (isAligned) { - session.insertAlignedRecord(deviceId, 3, measurements, types, partialNullValues); - session.insertAlignedRecord(deviceId, 4, measurements, types, allNullValues); - } else { - session.insertRecord(deviceId, 3, measurements, types, partialNullValues); - session.insertRecord(deviceId, 4, measurements, types, allNullValues); - } - } catch (StatementExecutionException e) { - fail(e.getMessage()); - } - } catch (IoTDBConnectionException e) { - fail(e.getMessage()); - } - }); - - INSERT_NULL_VALUE_MAP.put( - InsertType.SESSION_INSERT_TABLET, - (isAligned) -> { - try { - try (final ISession session = senderEnv.getSessionConnection()) { - if (isAligned) { - session.insertAlignedTablet(partialNullTablet); - session.insertAlignedTablet(allNullTablet); - } else { - session.insertTablet(partialNullTablet); - session.insertTablet(allNullTablet); - } - } catch (StatementExecutionException e) { - fail(e.getMessage()); + private void insertNullValues(final InsertType insertType, final boolean isAligned) + throws Exception { + switch (insertType) { + case SQL_INSERT -> { + final String alignment = isAligned ? " aligned" : ""; + // Both original rows and the final flush use one JDBC connection, preserving the SQL API. + TestUtils.executeNonQueries( + senderEnv, + isolateDatabasePaths( + Arrays.asList( + "insert into root.sg.d1(time, s0, s1)" + alignment + " values (3, null, 25.34)", + "insert into root.sg.d1(time, s0, s1)" + alignment + " values (4, null, null)", + "flush")), + null); + } + case SESSION_INSERT_RECORD, SESSION_INSERT_TABLET -> { + // Reuse the write Session for flush instead of opening an additional JDBC connection. + try (final ISession session = senderEnv.getSessionConnection()) { + if (insertType == InsertType.SESSION_INSERT_RECORD) { + if (isAligned) { + session.insertAlignedRecord(deviceId, 3, measurements, types, partialNullValues); + session.insertAlignedRecord(deviceId, 4, measurements, types, allNullValues); + } else { + session.insertRecord(deviceId, 3, measurements, types, partialNullValues); + session.insertRecord(deviceId, 4, measurements, types, allNullValues); } - } catch (IoTDBConnectionException e) { - fail(e.getMessage()); + } else if (isAligned) { + session.insertAlignedTablet(partialNullTablet); + session.insertAlignedTablet(allNullTablet); + } else { + session.insertTablet(partialNullTablet); + session.insertTablet(allNullTablet); } - }); - - INSERT_NULL_VALUE_MAP.put( - InsertType.SQL_INSERT, - (isAligned) -> { - // Partial null - TestUtils.executeNonQueries( - senderEnv, - isAligned - ? Collections.singletonList( - "insert into root.sg.d1(time, s0, s1) aligned values (3, null, 25.34)") - : Collections.singletonList( - "insert into root.sg.d1(time, s0, s1) values (3, null, 25.34)"), - null); - // All null - TestUtils.executeNonQueries( - senderEnv, - isAligned - ? Collections.singletonList( - "insert into root.sg.d1(time, s0, s1) aligned values (4, null, null)") - : Collections.singletonList( - "insert into root.sg.d1(time, s0, s1) values (4, null, null)"), - null); - }); + session.executeNonQueryStatement("flush"); + } + } + } } private void testInsertNullValueTemplate( @@ -228,7 +201,7 @@ private void testInsertNullValueTemplate( if (withParsing) { sourceAttributes.put("start-time", "1970-01-01T08:00:00.000+08:00"); sourceAttributes.put("end-time", "1970-01-01T09:00:00.000+08:00"); - sourceAttributes.put("source.pattern", "root.sg.d1"); + sourceAttributes.put("source.pattern", databasePath() + ".d1"); } final TSStatus status = @@ -240,39 +213,58 @@ private void testInsertNullValueTemplate( } TestUtils.executeNonQueries( - receiverEnv, isAligned ? CREATE_ALIGNED_TIMESERIES_SQL : CREATE_TIMESERIES_SQL, null); + receiverEnv, + isolateDatabasePaths(isAligned ? CREATE_ALIGNED_TIMESERIES_SQL : CREATE_TIMESERIES_SQL), + null); TestUtils.executeNonQueries( - senderEnv, isAligned ? CREATE_ALIGNED_TIMESERIES_SQL : CREATE_TIMESERIES_SQL, null); + senderEnv, + isolateDatabasePaths(isAligned ? CREATE_ALIGNED_TIMESERIES_SQL : CREATE_TIMESERIES_SQL), + null); - INSERT_NULL_VALUE_MAP.get(insertType).accept(isAligned); - TestUtils.executeNonQuery(senderEnv, "flush", null); + insertNullValues(insertType, isAligned); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.sg.**", - "count(root.sg.d1.s0),count(root.sg.d1.s1),", + isolateDatabasePaths("select count(*) from root.sg.**"), + isolateDatabasePaths("count(root.sg.d1.s0),count(root.sg.d1.s1),"), Collections.singleton("0,1,")); } + private String databasePath() { + return "root.sg_" + testName.getMethodName().toLowerCase(Locale.ROOT); + } + + private String isolateDatabasePaths(final String value) { + return value.replace("root.sg", databasePath()); + } + + private List isolateDatabasePaths(final List values) { + return values.stream().map(this::isolateDatabasePaths).toList(); + } + // ---------------------- // // Scenario 1: SQL Insert // // ---------------------- // + // Filtered non-aligned SQL rows with partial/all nulls must yield s0=0, s1=1. @Test public void testSQLInsertWithParsing() throws Exception { testInsertNullValueTemplate(InsertType.SQL_INSERT, false, true); } + // Unfiltered non-aligned SQL rows with partial/all nulls must yield s0=0, s1=1. @Test public void testSQLInsertWithoutParsing() throws Exception { testInsertNullValueTemplate(InsertType.SQL_INSERT, false, false); } + // Filtered aligned SQL rows with partial/all nulls must yield s0=0, s1=1. @Test public void testSQLInsertAlignedWithParsing() throws Exception { testInsertNullValueTemplate(InsertType.SQL_INSERT, true, true); } + // Unfiltered aligned SQL rows with partial/all nulls must yield s0=0, s1=1. @Test public void testSQLInsertAlignedWithoutParsing() throws Exception { testInsertNullValueTemplate(InsertType.SQL_INSERT, true, false); @@ -281,21 +273,25 @@ public void testSQLInsertAlignedWithoutParsing() throws Exception { // --------------------------------- // // Scenario 2: Session Insert Record // // --------------------------------- // + // Filtered non-aligned insertRecord rows with partial/all nulls must yield s0=0, s1=1. @Test public void testSessionInsertRecordWithParsing() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_RECORD, false, true); } + // Unfiltered non-aligned insertRecord rows with partial/all nulls must yield s0=0, s1=1. @Test public void testSessionInsertRecordWithoutParsing() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_RECORD, false, false); } + // Filtered aligned insertRecord rows with partial/all nulls must yield s0=0, s1=1. @Test public void testSessionInsertRecordAlignedWithParsing() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_RECORD, true, true); } + // Unfiltered aligned insertRecord rows with partial/all nulls must yield s0=0, s1=1. @Test public void testSessionInsertRecordAlignedWithoutParsing() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_RECORD, true, false); @@ -304,21 +300,25 @@ public void testSessionInsertRecordAlignedWithoutParsing() throws Exception { // --------------------------------- // // Scenario 3: Session Insert Tablet // // --------------------------------- // + // Filtered non-aligned Tablet rows with partial/all nulls must yield s0=0, s1=1. @Test public void testSessionInsertTabletWithParsing() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, true); } + // Unfiltered non-aligned Tablet rows with partial/all nulls must yield s0=0, s1=1. @Test public void testSessionInsertTabletWithoutParsing() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, false); } + // Filtered aligned Tablet rows with partial/all nulls must yield s0=0, s1=1. @Test public void testSessionInsertTabletAlignedWithParsing() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, true); } + // Unfiltered aligned Tablet rows with partial/all nulls must yield s0=0, s1=1. @Test public void testSessionInsertTabletAlignedWithoutParsing() throws Exception { testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, false); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeSinkCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeSinkCompressionIT.java index b5f4b55122d0f..e74f6df2974fa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeSinkCompressionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeSinkCompressionIT.java @@ -27,18 +27,19 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionConfig; -import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; -import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.sql.Connection; @@ -48,6 +49,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Consumer; @@ -55,15 +57,16 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeSinkCompressionIT extends AbstractPipeDualTreeModelAutoIT { +public class IoTDBPipeSinkCompressionIT extends AbstractPipeTreeModelSharedClusterIT { + + @Rule public final TestName testName = new TestName(); @Override - @Before - public void setUp() { - // Override to enable air-gap - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); + protected void setupConfig() { + // One LZMA2 encoder needs about 100 MB in addition to the node and in-flight callbacks. + // Give the sender headroom above the default 256 MB while bounding sink concurrency below. + senderEnv.getConfig().getDataNodeJVMConfig().setMaxHeapSize(512); + // Retain the compression scenarios' memory settings and receiver air-gap support. senderEnv .getConfig() @@ -91,36 +94,39 @@ public void setUp() { // 10 min, assert that the operations will not time out senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); } + // Async Thrift with Snappy and batching must transfer both historical and realtime data. @Test public void testCompression1() throws Exception { doTest("iotdb-thrift-sink", "stream", true, "snappy"); } + // Async Thrift batch mode must transfer data compressed by Snappy followed by LZMA2. @Test public void testCompression2() throws Exception { doTest("iotdb-thrift-sink", "batch", true, "snappy, lzma2"); } + // Sync Thrift stream mode without batching must decode two successive Snappy layers. @Test public void testCompression3() throws Exception { doTest("iotdb-thrift-sync-sink", "stream", false, "snappy, snappy"); } + // Sync Thrift batch mode must decode Gzip followed by Zstd for both write phases. @Test public void testCompression4() throws Exception { doTest("iotdb-thrift-sync-sink", "batch", true, "gzip, zstd"); } + // Air-gap stream mode must decode LZMA2 followed by LZ4 without batching. @Test public void testCompression5() throws Exception { doTest("iotdb-air-gap-sink", "stream", false, "lzma2, lz4"); } + // Air-gap batch mode must transfer historical and realtime data compressed by LZMA2. @Test public void testCompression6() throws Exception { doTest("iotdb-air-gap-sink", "batch", true, "lzma2"); @@ -137,22 +143,16 @@ private void doTest( ? receiverDataNode.getPipeAirGapReceiverPort() : receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - final Consumer senderOnlyFailure = - o -> TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (final RetryFlushHandler handleFailure = new RetryFlushHandler(); + final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { TestUtils.executeNonQueries( senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s1) values (2010-01-01T10:00:00+08:00, 1)", - "insert into root.db.d1(time, s1) values (2010-01-02T10:00:00+08:00, 2)", - "flush"), + isolateDatabasePaths( + Arrays.asList( + "insert into root.db.d1(time, s1) values (2010-01-01T10:00:00+08:00, 1)", + "insert into root.db.d1(time, s1) values (2010-01-02T10:00:00+08:00, 2)", + "flush")), null); final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); @@ -160,16 +160,21 @@ private void doTest( sourceAttributes.put("source", "iotdb-source"); sourceAttributes.put("source.realtime.mode", realtimeMode); - sourceAttributes.put("user", "root"); + // The default pattern format matches literal prefixes; a wildcard would exclude the data. + sourceAttributes.put("source.pattern", databasePath() + "."); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); processorAttributes.put("processor", "do-nothing-processor"); sinkAttributes.put("sink", sinkType); sinkAttributes.put("sink.batch.enable", useBatchMode ? "true" : "false"); + // LZMA2 allocates a large dictionary per task. Bound its peak memory for these functional + // round trips while retaining every codec and batching combination. + sinkAttributes.put("sink.parallel.tasks", "1"); sinkAttributes.put("sink.ip", receiverIp); sinkAttributes.put("sink.port", Integer.toString(receiverPort)); - sinkAttributes.put("sink.user", "root"); - sinkAttributes.put("sink.password", "root"); + sinkAttributes.put("sink.user", SessionConfig.DEFAULT_USER); + sinkAttributes.put("sink.password", SessionConfig.DEFAULT_PASSWORD); sinkAttributes.put("sink.compressor", compressionTypes); final TSStatus status = @@ -184,32 +189,35 @@ private void doTest( TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.s1),", + isolateDatabasePaths("select count(*) from root.db.**"), + isolateDatabasePaths("count(root.db.d1.s1),"), Collections.singleton("2,"), handleFailure); TestUtils.executeNonQueries( senderEnv, - Arrays.asList( - "insert into root.db.d1(time, s1) values (3, 3)", - "insert into root.db.d1(time, s1) values (4, 4)", - "insert into root.db.d1(time, s1) values (5, 5)", - "insert into root.db.d1(time, s1) values (6, 6)", - "insert into root.db.d1(time, s1) values (7, 7)", - "insert into root.db.d1(time, s1) values (8, 8)", - "flush"), + isolateDatabasePaths( + Arrays.asList( + "insert into root.db.d1(time, s1) values (3, 3)", + "insert into root.db.d1(time, s1) values (4, 4)", + "insert into root.db.d1(time, s1) values (5, 5)", + "insert into root.db.d1(time, s1) values (6, 6)", + "insert into root.db.d1(time, s1) values (7, 7)", + "insert into root.db.d1(time, s1) values (8, 8)", + "flush")), null); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "select count(*) from root.db.**", - "count(root.db.d1.s1),", + isolateDatabasePaths("select count(*) from root.db.**"), + isolateDatabasePaths("count(root.db.d1.s1),"), Collections.singleton("8,"), - senderOnlyFailure); + handleFailure::flushSender); } } + // Zstd levels 3, 22 and -131072 must each sync; -131073 and 23 must fail validation while the + // shared SQL connection remains usable and only the three valid pipes remain registered. @Test public void testZstdCompressorLevel() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -217,25 +225,46 @@ public void testZstdCompressorLevel() throws Exception { final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement(); + final RetryFlushHandler handleFailure = new RetryFlushHandler(); + final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { // Create legal zstd level pipes one by one, so the assertion identifies the exact level // that fails and avoids concurrent historical TsFile splitting for this level test. + // Retain the writer for inserts, flushes and all CREATEs, including rejected levels. createZstdPipeAndAssertData( - "p1", "root.db.d1.s1", "3", receiverIp, receiverPort, "s1", handleFailure); + statement, + "p1", + databasePath() + ".d1.s1", + "3", + receiverIp, + receiverPort, + "s1", + handleFailure); createZstdPipeAndAssertData( - "p2", "root.db.d1.s2", "22", receiverIp, receiverPort, "s2", handleFailure); + statement, + "p2", + databasePath() + ".d1.s2", + "22", + receiverIp, + receiverPort, + "s2", + handleFailure); createZstdPipeAndAssertData( - "p3", "root.db.d1.s3", "-131072", receiverIp, receiverPort, "s3", handleFailure); + statement, + "p3", + databasePath() + ".d1.s3", + "-131072", + receiverIp, + receiverPort, + "s3", + handleFailure); - assertCreateZstdPipeFailed("p4", "root.db.d1.s4", "-131073", receiverIp, receiverPort); - assertCreateZstdPipeFailed("p5", "root.db.d1.s5", "23", receiverIp, receiverPort); + assertCreateZstdPipeFailed( + statement, "p4", databasePath() + ".d1.s4", "-131073", receiverIp, receiverPort); + assertCreateZstdPipeFailed( + statement, "p5", databasePath() + ".d1.s5", "23", receiverIp, receiverPort); final List showPipeResult = client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList; @@ -248,43 +277,81 @@ public void testZstdCompressorLevel() throws Exception { } } + private final class RetryFlushHandler implements Consumer, AutoCloseable { + // Index 0 owns the sender Session and index 1 owns the receiver Session. Open lazily so a + // successful first assertion poll needs no retry clients. + private final ISession[] sessions = new ISession[2]; + + @Override + public void accept(final String ignored) { + flush(0); + flush(1); + } + + private void flushSender(final String ignored) { + flush(0); + } + + private void flush(final int index) { + // Assertions poll every 100 ms; keep connections across retries instead of opening JDBC + // connections to every DataNode on each failed poll. + try { + if (sessions[index] == null) { + sessions[index] = (index == 0 ? senderEnv : receiverEnv).getSessionConnection(); + } + sessions[index].executeNonQueryStatement("flush"); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + @Override + public void close() throws Exception { + try { + if (sessions[0] != null) { + sessions[0].close(); + } + } finally { + if (sessions[1] != null) { + sessions[1].close(); + } + } + } + } + private void createZstdPipeAndAssertData( + final Statement statement, final String pipeName, final String sourcePattern, final String zstdLevel, final String receiverIp, final int receiverPort, final String measurement, - final Consumer handleFailure) { - TestUtils.executeNonQueries( - senderEnv, - Arrays.asList( - String.format("insert into root.db.d1(time, %s) values (1, 1)", measurement), "flush"), - null); - - try { - createZstdPipe(pipeName, sourcePattern, zstdLevel, receiverIp, receiverPort); - } catch (final SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } + final Consumer handleFailure) + throws SQLException { + statement.execute( + isolateDatabasePaths( + String.format("insert into root.db.d1(time, %s) values (1, 1)", measurement))); + statement.execute("flush"); + createZstdPipe(statement, pipeName, sourcePattern, zstdLevel, receiverIp, receiverPort); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - String.format("select count(%s) from root.db.d1", measurement), - String.format("count(root.db.d1.%s),", measurement), + isolateDatabasePaths(String.format("select count(%s) from root.db.d1", measurement)), + isolateDatabasePaths(String.format("count(root.db.d1.%s),", measurement)), Collections.singleton("1,"), handleFailure); } private void assertCreateZstdPipeFailed( + final Statement statement, final String pipeName, final String sourcePattern, final String zstdLevel, final String receiverIp, final int receiverPort) { try { - createZstdPipe(pipeName, sourcePattern, zstdLevel, receiverIp, receiverPort); + createZstdPipe(statement, pipeName, sourcePattern, zstdLevel, receiverIp, receiverPort); fail(); } catch (final SQLException e) { Assert.assertTrue(e.getMessage().contains("Zstd compression level should be in the range")); @@ -292,24 +359,34 @@ private void assertCreateZstdPipeFailed( } private void createZstdPipe( + final Statement statement, final String pipeName, final String sourcePattern, final String zstdLevel, final String receiverIp, final int receiverPort) throws SQLException { - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - String.format( - "create pipe %s" - + " with source ('source.pattern'='%s')" - + " with sink (" - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.compressor'='zstd, zstd'," - + "'sink.compressor.zstd.level'='%s')", - pipeName, sourcePattern, receiverIp, receiverPort, zstdLevel)); - } + statement.execute( + String.format( + "create pipe %s" + + " with source ('source.pattern'='%s')" + + " with sink (" + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.compressor'='zstd, zstd'," + + "'sink.compressor.zstd.level'='%s')", + pipeName, sourcePattern, receiverIp, receiverPort, zstdLevel)); + } + + private String databasePath() { + return "root.db_" + testName.getMethodName().toLowerCase(Locale.ROOT); + } + + private String isolateDatabasePaths(final String value) { + return value.replace("root.db", databasePath()); + } + + private List isolateDatabasePaths(final List values) { + return values.stream().map(this::isolateDatabasePaths).toList(); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeTypeConversionSemanticIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeTypeConversionSemanticIT.java index b221800337988..cb4b0cd1e514f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeTypeConversionSemanticIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeTypeConversionSemanticIT.java @@ -20,14 +20,12 @@ package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.TypeConversionSemanticCase; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -35,10 +33,11 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.concurrent.CompletableFuture; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeTypeConversionSemanticIT extends AbstractPipeDualTreeModelAutoIT { +public class IoTDBPipeTypeConversionSemanticIT extends AbstractPipeTreeModelSharedClusterIT { private static final String DEVICE = "root.pipe_type_conversion.d"; private static final String ALIGNED_DEVICE = "root.pipe_type_conversion.aligned_d"; @@ -55,17 +54,6 @@ public class IoTDBPipeTypeConversionSemanticIT extends AbstractPipeDualTreeModel "int32_to_timestamp", "int32_to_date"); - @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - setupConfig(); - senderEnv.initClusterEnvironment(1, 1); - receiverEnv.initClusterEnvironment(1, 1); - } - @Override protected void setupConfig() { super.setupConfig(); @@ -81,6 +69,13 @@ protected void setupConfig() { .setSchemaReplicationFactor(1); } + @Override + protected CompletableFuture initializeClusters() { + return CompletableFuture.allOf( + CompletableFuture.runAsync(() -> senderEnv.initClusterEnvironment(1, 1)), + CompletableFuture.runAsync(() -> receiverEnv.initClusterEnvironment(1, 1))); + } + @Test public void testPipeReceiverTypeConversionSemantics() { createTimeseries(senderEnv, DEVICE, TypeConversionSemanticCase.CASES, true); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java index 0a5e3b0da7452..22aba74f202b5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java @@ -41,6 +41,9 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoEnhanced.class}) @@ -81,8 +84,7 @@ public void setUp() { senderEnv.getConfig().getConfigNodeConfig().setLeaderDistributionPolicy("HASH"); - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); + executeOnBothEnvs(env -> env.initClusterEnvironment()); } /** @@ -124,11 +126,23 @@ public void testReceiverNotLoadDeletedTimeseries() throws Exception { "insert into root.db.d4 (time, s1) values (1, 1), (3, 3)", "flush"), null); - // adding new devices may create a new region, and thus trigger leader re-balancing - // the old leader may not have synced data to the new leader, and the result is that - // the following deletions performed on the new leader can not delete anything - // wait for a while to increase the chance that data has been synced to the new leader - Thread.sleep(5000); + // Adding devices may create a new region and trigger leader re-balancing. Wait for the + // committed schema to be visible through the current leader for a short quiet period rather + // than sleeping a fixed five seconds before issuing deletions. + await() + .pollInSameThread() + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) + .during(500L, TimeUnit.MILLISECONDS) + .atMost(5L, TimeUnit.SECONDS) + .untilAsserted( + () -> + TestUtils.assertDataEventuallyOnEnv( + senderEnv, + "count timeseries root.db.**", + "count(timeseries),", + Collections.singleton("6,"), + 1)); TestUtils.executeNonQueries( senderEnv, Arrays.asList( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java index f2735938e16fe..bde1cb7d48a79 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java @@ -30,11 +30,10 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeTreeModelSharedClusterIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -50,14 +49,10 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class PipeNowFunctionIT extends AbstractPipeDualTreeModelAutoIT { - - @Override - @Before - public void setUp() { - super.setUp(); - } +public class PipeNowFunctionIT extends AbstractPipeTreeModelSharedClusterIT { + // CREATE/ALTER PIPE through RPC accepts "now" for prefixed and unprefixed history/source time + // bounds; each created pipe remains visible and RUNNING until explicitly stopped. @Test public void testPipeNowFunction() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -76,7 +71,7 @@ public void testPipeNowFunction() throws Exception { sourceAttributes.put("source.history.start-time", "now"); sourceAttributes.put("source.history.end-time", "now"); sourceAttributes.put("source.history.enable", "true"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.batch.enable", "false"); @@ -96,7 +91,7 @@ public void testPipeNowFunction() throws Exception { sourceAttributes.put("history.start-time", "now"); sourceAttributes.put("history.end-time", "now"); sourceAttributes.put("history.enable", "true"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); status = client.createPipe( @@ -111,7 +106,7 @@ public void testPipeNowFunction() throws Exception { sourceAttributes.put("source.history.start-time", "now"); sourceAttributes.put("source.history.end-time", "now"); sourceAttributes.put("history.enable", "true"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); status = client.createPipe( @@ -140,7 +135,7 @@ public void testPipeNowFunction() throws Exception { sourceAttributes.put("source.end-time", "now"); sourceAttributes.put("source.history.start-time", "now"); sourceAttributes.put("source.history.end-time", "now"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); client.alterPipe( new TAlterPipeReq() .setPipeName("p1") @@ -162,7 +157,7 @@ public void testPipeNowFunction() throws Exception { sourceAttributes.put("end-time", "now"); sourceAttributes.put("history.start-time", "now"); sourceAttributes.put("history.end-time", "now"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); client.alterPipe( new TAlterPipeReq() .setPipeName("p1") @@ -183,6 +178,8 @@ public void testPipeNowFunction() throws Exception { } } + // Tree-model CREATE/ALTER PIPE must accept now in prefixed and unprefixed history/source time + // attributes; execute all six variants in order on the same connection. @Test public void testTreeModeSQLSupportNowFunc() { doTest(BaseEnv.TREE_SQL_DIALECT); @@ -212,94 +209,69 @@ private void doTest(String dialect) { try (final Connection connection = senderEnv.getConnection(dialect); final Statement statement = connection.createStatement()) { statement.execute(p1); - } catch (final SQLException e) { - fail(e.getMessage()); - } - final String p2 = - String.format( - "create pipe p2" - + " with source (" - + "'source.history.enable'='true'," - + "'start-time'='now'," - + "'end-time'='now'," - + "'history.start-time'='now'," - + "'history.end-time'='now')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(dialect); - final Statement statement = connection.createStatement()) { + final String p2 = + String.format( + "create pipe p2" + + " with source (" + + "'source.history.enable'='true'," + + "'start-time'='now'," + + "'end-time'='now'," + + "'history.start-time'='now'," + + "'history.end-time'='now')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p2); - } catch (final SQLException e) { - fail(e.getMessage()); - } - final String p3 = - String.format( - "create pipe p3" - + " with source (" - + "'source.history.enable'='true'," - + "'source.start-time'='now'," - + "'source.end-time'='now'," - + "'source.history.start-time'='now'," - + "'source.history.end-time'='now')" - + " with sink (" - + "'sink'='iotdb-thrift-sink'," - + "'sink.ip'='%s'," - + "'sink.port'='%s'," - + "'sink.batch.enable'='false')", - receiverIp, receiverPort); - try (final Connection connection = senderEnv.getConnection(dialect); - final Statement statement = connection.createStatement()) { + final String p3 = + String.format( + "create pipe p3" + + " with source (" + + "'source.history.enable'='true'," + + "'source.start-time'='now'," + + "'source.end-time'='now'," + + "'source.history.start-time'='now'," + + "'source.history.end-time'='now')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", + receiverIp, receiverPort); statement.execute(p3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - String alterP3 = - "alter pipe p3" - + " modify source (" - + "'history.enable'='true'," - + "'start-time'='now'," - + "'end-time'='now'," - + "'history.start-time'='now'," - + "'history.end-time'='now')"; - try (final Connection connection = senderEnv.getConnection(dialect); - final Statement statement = connection.createStatement()) { + String alterP3 = + "alter pipe p3" + + " modify source (" + + "'history.enable'='true'," + + "'start-time'='now'," + + "'end-time'='now'," + + "'history.start-time'='now'," + + "'history.end-time'='now')"; statement.execute(alterP3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - alterP3 = - "alter pipe p3" - + " modify source (" - + "'source.history.enable'='true'," - + "'source.start-time'='now'," - + "'source.end-time'='now'," - + "'source.history.start-time'='now'," - + "'source.history.end-time'='now')"; - try (final Connection connection = senderEnv.getConnection(dialect); - final Statement statement = connection.createStatement()) { + alterP3 = + "alter pipe p3" + + " modify source (" + + "'source.history.enable'='true'," + + "'source.start-time'='now'," + + "'source.end-time'='now'," + + "'source.history.start-time'='now'," + + "'source.history.end-time'='now')"; statement.execute(alterP3); - } catch (final SQLException e) { - fail(e.getMessage()); - } - alterP3 = - "alter pipe p3" - + " modify source (" - + "'source.history.enable'='true'," - + "'source.start-time'='now'," - + "'source.end-time'='now'," - + "'source.history.start-time'='now'," - + "'source.history.end-time'='now')"; - try (final Connection connection = senderEnv.getConnection(dialect); - final Statement statement = connection.createStatement()) { + alterP3 = + "alter pipe p3" + + " modify source (" + + "'source.history.enable'='true'," + + "'source.start-time'='now'," + + "'source.end-time'='now'," + + "'source.history.start-time'='now'," + + "'source.history.end-time'='now')"; statement.execute(alterP3); } catch (final SQLException e) { fail(e.getMessage()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java index 0fec803c2d3e4..d6f7c387f29d3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.itbase.env.BaseEnv; @@ -31,8 +30,13 @@ import org.junit.Before; import java.io.File; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.util.Objects; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; public abstract class AbstractPipeDualTreeModelManualIT { @@ -45,8 +49,16 @@ public void setUp() { senderEnv = MultiEnvFactory.getEnv(0); receiverEnv = MultiEnvFactory.getEnv(1); setupConfig(); - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); + executeOnBothEnvs(BaseEnv::initClusterEnvironment); + } + + protected void executeOnBothEnvs(final Consumer action) { + // The clusters own separate node directories and ports. Wait for both tasks, including when one + // fails, so teardown cannot race with a cluster that is still starting or restarting. + CompletableFuture.allOf( + CompletableFuture.runAsync(() -> action.accept(senderEnv)), + CompletableFuture.runAsync(() -> action.accept(receiverEnv))) + .join(); } protected void setupConfig() { @@ -80,33 +92,41 @@ protected void setupConfig() { @After public final void tearDown() { + cleanEnvironment(); + } + + protected void cleanEnvironment() { senderEnv.cleanClusterEnvironment(); receiverEnv.cleanClusterEnvironment(); } - protected static void awaitUntilFlush(BaseEnv env) { - Awaitility.await() - .atMost(1, TimeUnit.MINUTES) - .pollDelay(2, TimeUnit.SECONDS) - .pollInterval(2, TimeUnit.SECONDS) - .until( - () -> { - TestUtils.executeNonQuery(env, "flush", null); - return env.getDataNodeWrapperList().stream() - .anyMatch( - wrapper -> { - int fileNum = 0; - File sequence = new File(buildDataPath(wrapper, true)); - File unsequence = new File(buildDataPath(wrapper, false)); - if (sequence.exists() && sequence.listFiles() != null) { - fileNum += Objects.requireNonNull(sequence.listFiles()).length; - } - if (unsequence.exists() && unsequence.listFiles() != null) { - fileNum += Objects.requireNonNull(unsequence.listFiles()).length; - } - return fileNum > 0; - }); - }); + protected static void awaitUntilFlush(BaseEnv env) throws SQLException { + // Reuse the connection across immediate, bounded checks instead of reopening it on every poll. + try (Connection connection = env.getConnection(); + Statement statement = connection.createStatement()) { + Awaitility.await() + .atMost(1, TimeUnit.MINUTES) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .until( + () -> { + statement.execute("flush"); + return env.getDataNodeWrapperList().stream() + .anyMatch( + wrapper -> { + int fileNum = 0; + File sequence = new File(buildDataPath(wrapper, true)); + File unsequence = new File(buildDataPath(wrapper, false)); + if (sequence.exists() && sequence.listFiles() != null) { + fileNum += Objects.requireNonNull(sequence.listFiles()).length; + } + if (unsequence.exists() && unsequence.listFiles() != null) { + fileNum += Objects.requireNonNull(unsequence.listFiles()).length; + } + return fileNum > 0; + }); + }); + } } private static String buildDataPath(DataNodeWrapper wrapper, boolean isSequence) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeTreeModelManualSharedClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeTreeModelManualSharedClusterIT.java new file mode 100644 index 0000000000000..b58e606699967 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeTreeModelManualSharedClusterIT.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.manual; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.ExternalResource; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.junit.Assert.assertEquals; + +/** Shares fixed-topology clusters and resets tree data and authorization state between methods. */ +public abstract class AbstractPipeTreeModelManualSharedClusterIT + extends AbstractPipeDualTreeModelManualIT { + + private static BaseEnv sharedSender; + private static BaseEnv sharedReceiver; + private static CompletableFuture startup; + + @ClassRule + public static final ExternalResource CLUSTERS = + new ExternalResource() { + @Override + protected void after() { + // Release both clusters even if startup or the first cleanup fails. + try { + if (sharedSender != null) { + sharedSender.cleanClusterEnvironment(); + } + } finally { + try { + if (sharedReceiver != null) { + sharedReceiver.cleanClusterEnvironment(); + } + } finally { + sharedSender = null; + sharedReceiver = null; + startup = null; + } + } + } + }; + + @Override + @Before + public void setUp() { + if (sharedSender == null) { + MultiEnvFactory.createEnv(2); + sharedSender = MultiEnvFactory.getEnv(0); + sharedReceiver = MultiEnvFactory.getEnv(1); + senderEnv = sharedSender; + receiverEnv = sharedReceiver; + setupConfig(); + // Keep the future so failed starts cannot be mistaken for a usable shared environment. + startup = initializeClusters(); + } + senderEnv = sharedSender; + receiverEnv = sharedReceiver; + startup.join(); + } + + protected CompletableFuture initializeClusters() { + return CompletableFuture.allOf( + CompletableFuture.runAsync(senderEnv::initClusterEnvironment), + CompletableFuture.runAsync(receiverEnv::initClusterEnvironment)); + } + + @Override + protected void cleanEnvironment() { + if (startup == null || startup.isCompletedExceptionally()) { + return; + } + try { + // Stop replication on both sides before dropping data or users; a live pipe can recreate + // them. + dropUserPipes(senderEnv); + dropUserPipes(receiverEnv); + deleteTreeDatabases(senderEnv); + deleteTreeDatabases(receiverEnv); + deleteSchemaTemplates(senderEnv); + deleteSchemaTemplates(receiverEnv); + deleteUsersAndRoles(senderEnv); + deleteUsersAndRoles(receiverEnv); + } catch (Exception e) { + throw new AssertionError("Failed to reset shared tree pipe test clusters", e); + } + } + + private static void dropUserPipes(final BaseEnv env) throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { + final TShowPipeResp response = + client.showPipe( + new TShowPipeReq().setIsTableModel(false).setUserName(SessionConfig.DEFAULT_USER)); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + for (final TShowPipeInfo pipe : response.getPipeInfoList()) { + if (!pipe.getId().startsWith(PipeStaticMeta.SYSTEM_PIPE_PREFIX)) { + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(pipe.getId()).setIsTableModel(false)) + .getCode()); + } + } + } + } + + private static void deleteTreeDatabases(final BaseEnv env) throws SQLException { + try (final Connection connection = env.getConnection(); + final Statement statement = connection.createStatement()) { + final List databases = new ArrayList<>(); + try (final ResultSet result = statement.executeQuery("show databases")) { + while (result.next()) { + databases.add(result.getString(1)); + } + } + for (final String database : databases) { + statement.execute("delete database " + database); + } + } + } + + private static void deleteUsersAndRoles(final BaseEnv env) throws SQLException { + // Authorization survives database deletion. Dropping test users/roles clears grants, membership + // and login locks while retaining the administrator required by the next method. + try (final Connection connection = env.getConnection(); + final Statement statement = connection.createStatement()) { + for (final String type : new String[] {"user", "role"}) { + final List names = new ArrayList<>(); + try (final ResultSet result = statement.executeQuery("list " + type)) { + while (result.next()) { + final String name = result.getString("user".equals(type) ? "User" : "Role"); + if (!"user".equals(type) || !SessionConfig.DEFAULT_USER.equals(name)) { + names.add(name); + } + } + } + for (final String name : names) { + statement.execute("drop " + type + " `" + name.replace("`", "``") + "`"); + } + } + } + } + + private static void deleteSchemaTemplates(final BaseEnv env) throws SQLException { + try (final Connection connection = env.getConnection(); + final Statement statement = connection.createStatement()) { + final List templates = new ArrayList<>(); + try (final ResultSet result = statement.executeQuery("show schema templates")) { + while (result.next()) { + templates.add(result.getString(1)); + } + } + for (final String template : templates) { + statement.execute("drop schema template `" + template.replace("`", "``") + "`"); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java index 5dc83ca9227a1..5264b47b520ab 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java @@ -42,7 +42,7 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeInclusionIT extends AbstractPipeDualTreeModelManualIT { +public class IoTDBPipeInclusionIT extends AbstractPipeTreeModelManualSharedClusterIT { @Test public void testPureSchemaInclusion() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -269,6 +269,8 @@ public void testPureSchemaInclusionWithExclusionPattern() throws Exception { } } + // A pipe that includes all events except auth must leave a newly created sender user absent on + // the receiver throughout the observation period. @Test public void testAuthExclusion() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -301,7 +303,7 @@ public void testAuthExclusion() throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - TestUtils.executeNonQuery(senderEnv, "create user `ln_write_user` 'write_pwd123456'", null); + TestUtils.executeNonQuery(senderEnv, "create user `ln_write_user` 'Write_pwd!123456'", null); TestUtils.assertDataAlwaysOnEnv( receiverEnv, @@ -311,6 +313,8 @@ public void testAuthExclusion() throws Exception { } } + // An auth-only pipe with a root.ln path filter must replicate the user and retain global grants, + // while limiting the replicated data privileges to root.ln.**. @Test public void testAuthInclusionWithPattern() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -346,7 +350,7 @@ public void testAuthInclusionWithPattern() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "create user `ln_write_user` 'write_pwd123456'", + "create user `ln_write_user` 'Write_pwd!123456'", "grant system,security on root.** to USER ln_write_user with grant option", "GRANT READ_DATA, WRITE_DATA ON root.** TO USER ln_write_user;"), null); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java index 6df47f39f2ac4..00c2eaca1a98b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java @@ -25,14 +25,12 @@ import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -42,26 +40,19 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.concurrent.CompletableFuture; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeMetaHistoricalIT extends AbstractPipeDualTreeModelManualIT { +public class IoTDBPipeMetaHistoricalIT extends AbstractPipeTreeModelManualSharedClusterIT { @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - + protected void setupConfig() { + super.setupConfig(); senderEnv .getConfig() .getCommonConfig() - .setAutoCreateSchemaEnabled(false) - .setDatanodeMemoryProportion("3:3:1:1:1:0") .setDefaultSchemaRegionGroupNumPerDatabase(1) .setTimestampPrecision("ms") - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setDnConnectionTimeoutMs(600000) .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); @@ -69,22 +60,25 @@ public void setUp() { receiverEnv .getConfig() .getCommonConfig() - .setAutoCreateSchemaEnabled(false) - .setDatanodeMemoryProportion("3:3:1:1:1:0") .setTimestampPrecision("ms") - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) .setSchemaReplicationFactor(3) .setDataReplicationFactor(2) .setDnConnectionTimeoutMs(600000) .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); + } - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(3, 3); + @Override + protected CompletableFuture initializeClusters() { + // Preserve the receiver's 3x3 topology while starting it alongside the 1x1 sender. + return CompletableFuture.allOf( + CompletableFuture.runAsync(senderEnv::initClusterEnvironment), + CompletableFuture.runAsync(() -> receiverEnv.initClusterEnvironment(3, 3))); } + // Historical/realtime template events under root.ln must replicate to the 3C3D receiver with its + // own replication factors, while ordinary schema and authorization remain excluded. @Test public void testTemplateInclusion() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -103,7 +97,7 @@ public void testTemplateInclusion() throws Exception { "create database root.ln", "create database root.db", "set ttl to root.ln 3600000", - "create user `thulab` 'passwd123456'", + "create user `thulab` 'IoTDB!Test2024'", "create role `admin`", "grant role `admin` to `thulab`", "grant read on root.** to role `admin`", @@ -176,6 +170,8 @@ public void testTemplateInclusion() throws Exception { } } + // An auth-only historical pipe must replicate existing users, roles and their grants, while + // excluding the sender's databases, timeseries and data. @Test public void testAuthInclusion() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -193,7 +189,7 @@ public void testAuthInclusion() throws Exception { Arrays.asList( "create database root.ln", "set ttl to root.ln 3600000", - "create user `thulab` 'passwd123456'", + "create user `thulab` 'IoTDB!Test2024'", "create role `admin`", "grant role `admin` to `thulab`", "grant read on root.** to role `admin`", diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java index 4df96a8ca717e..5c1745519fc19 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java @@ -24,14 +24,16 @@ import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.cluster.env.AbstractEnv; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; +import org.awaitility.Awaitility; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -39,17 +41,21 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeMetaLeaderChangeIT extends AbstractPipeDualTreeModelManualIT { - @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); +public class IoTDBPipeMetaLeaderChangeIT extends AbstractPipeTreeModelManualSharedClusterIT { + + private static final String CONFIG_DATABASE_PREFIX = "root.meta_leader_config"; + private static final String SCHEMA_DATABASE = "root.meta_leader_schema"; + + private int stoppedConfigNodeIndex = -1; + private int stoppedDataNodeIndex = -1; + @Override + protected void setupConfig() { senderEnv .getConfig() .getCommonConfig() @@ -70,11 +76,38 @@ public void setUp() { .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false) .setDnConnectionTimeoutMs(600000); + } - senderEnv.initClusterEnvironment(3, 3, 180); - receiverEnv.initClusterEnvironment(); + @Override + protected CompletableFuture initializeClusters() { + // The scenarios need three sender replicas for leader failover, with the original receiver + // topology. Only their initial startup is independent; restore stopped nodes between methods. + return CompletableFuture.allOf( + CompletableFuture.runAsync(() -> senderEnv.initClusterEnvironment(3, 3, 180)), + CompletableFuture.runAsync(receiverEnv::initClusterEnvironment)); + } + + @Override + protected void cleanEnvironment() { + // Restore the failed node before resetting pipe/data state: cleanup must reach every replica, + // and the next scenario must start with the full topology even if this test's assertions fail. + if (stoppedConfigNodeIndex >= 0) { + if (!senderEnv.getConfigNodeWrapper(stoppedConfigNodeIndex).isAlive()) { + senderEnv.startConfigNode(stoppedConfigNodeIndex); + } + ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); + } + if (stoppedDataNodeIndex >= 0) { + if (!senderEnv.getDataNodeWrapper(stoppedDataNodeIndex).isAlive()) { + senderEnv.startDataNode(stoppedDataNodeIndex); + } + ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); + } + super.cleanEnvironment(); } + // Create ten databases, stop the ConfigNode leader, then create ten more through the new leader; + // the receiver must contain all twenty under this scenario's database prefix. @Test public void testConfigNodeLeaderChange() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -91,7 +124,7 @@ public void testConfigNodeLeaderChange() throws Exception { sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); sourceAttributes.put("source.forwarding-pipe-requests", "false"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.ip", receiverIp); @@ -111,25 +144,24 @@ public void testConfigNodeLeaderChange() throws Exception { TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); } - for (int i = 0; i < 10; ++i) { - TestUtils.executeNonQuery(senderEnv, String.format("create database root.ln%s", i), null); - } + createDatabases(0, 10); - try { - senderEnv.shutdownConfigNode(senderEnv.getLeaderConfigNodeIndex()); - } catch (final Exception e) { - e.printStackTrace(); - return; - } + stoppedConfigNodeIndex = senderEnv.getLeaderConfigNodeIndex(); + senderEnv.shutdownConfigNode(stoppedConfigNodeIndex); + Assert.assertFalse(senderEnv.getConfigNodeWrapper(stoppedConfigNodeIndex).isAlive()); + Assert.assertNotEquals(stoppedConfigNodeIndex, senderEnv.getLeaderConfigNodeIndex()); - for (int i = 10; i < 20; ++i) { - TestUtils.executeNonQuery(senderEnv, String.format("create database root.ln%s", i), null); - } + createDatabases(10, 20); TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "count databases root.ln*", "count,", Collections.singleton("20,")); + receiverEnv, + "count databases " + CONFIG_DATABASE_PREFIX + "*", + "count,", + Collections.singleton("20,")); } + // Stop the SchemaRegion leader after ten timeseries, then alter the first and create ten more. + // Failover must preserve all twenty timeseries and merge the alias, tags and attributes remotely. @Test public void testSchemaRegionLeaderChange() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -146,7 +178,7 @@ public void testSchemaRegionLeaderChange() throws Exception { sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); sourceAttributes.put("source.forwarding-pipe-requests", "false"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.ip", receiverIp); @@ -166,51 +198,76 @@ public void testSchemaRegionLeaderChange() throws Exception { TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); } - for (int i = 0; i < 10; ++i) { - TestUtils.executeNonQuery( - senderEnv, - String.format( - "create timeSeries root.ln.wf01.GPS.status%s with datatype=BOOLEAN,encoding=PLAIN", - i), - null); + // Close this phase's session before stopping a node; client reconnection is not under test. + try (final ISession session = senderEnv.getSessionConnection()) { + for (int i = 0; i < 10; ++i) { + session.executeNonQueryStatement(createTimeseriesStatement(i)); + } } - final int index; - try { - index = senderEnv.getFirstLeaderSchemaRegionDataNodeIndex(); - senderEnv.shutdownDataNode(index); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } + stoppedDataNodeIndex = senderEnv.getFirstLeaderSchemaRegionDataNodeIndex(); + Assert.assertTrue(stoppedDataNodeIndex >= 0); + senderEnv.shutdownDataNode(stoppedDataNodeIndex); + Assert.assertFalse(senderEnv.getDataNodeWrapper(stoppedDataNodeIndex).isAlive()); + // Observe a replacement leader before sending mutations, avoiding a fixed election sleep. + Awaitility.await() + .pollInSameThread() + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .atMost(180, TimeUnit.SECONDS) + .until( + () -> { + final int leaderIndex = senderEnv.getFirstLeaderSchemaRegionDataNodeIndex(); + return leaderIndex >= 0 && leaderIndex != stoppedDataNodeIndex; + }); // Include "alter" in historical transfer for new leader as possible // in order to test the "withMerge" creation in receiver side - TestUtils.tryExecuteNonQueryOnSpecifiedDataNodeWithRetry( - senderEnv, - senderEnv.getDataNodeWrapper(index == 0 ? 1 : 0), - "ALTER timeSeries root.ln.wf01.GPS.status0 UPSERT ALIAS=newAlias TAGS(tag3=v3) ATTRIBUTES(attr4=v4)"); - - for (int i = 10; i < 20; ++i) { - TestUtils.tryExecuteNonQueryOnSpecifiedDataNodeWithRetry( - senderEnv, - senderEnv.getDataNodeWrapper(index == 0 ? 1 : 0), - String.format( - "create timeSeries root.ln.wf01.GPS.status%s with datatype=BOOLEAN,encoding=PLAIN", - i)); + // Use one session on a surviving node after election. The session's RPC retries apply only to + // the current statement, so an error cannot replay earlier successful CREATE statements. + final DataNodeWrapper survivingDataNode = + senderEnv.getDataNodeWrapper(stoppedDataNodeIndex == 0 ? 1 : 0); + try (final ISession session = + senderEnv.getSessionConnection( + Collections.singletonList(survivingDataNode.getIpAndPortString()))) { + session.executeNonQueryStatement( + "ALTER timeSeries " + + SCHEMA_DATABASE + + ".wf01.GPS.status0 UPSERT ALIAS=newAlias TAGS(tag3=v3) ATTRIBUTES(attr4=v4)"); + for (int i = 10; i < 20; ++i) { + session.executeNonQueryStatement(createTimeseriesStatement(i)); + } } TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "show timeSeries root.ln.wf01.GPS.status0", + "show timeSeries " + SCHEMA_DATABASE + ".wf01.GPS.status0", "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", Collections.singleton( - "root.ln.wf01.GPS.status0,newAlias,root.ln,BOOLEAN,PLAIN,LZ4,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,")); + SCHEMA_DATABASE + + ".wf01.GPS.status0,newAlias," + + SCHEMA_DATABASE + + ",BOOLEAN,PLAIN,LZ4,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,")); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "count timeSeries root.ln.**", + "count timeSeries " + SCHEMA_DATABASE + ".**", "count(timeseries),", Collections.singleton("20,")); } + + private void createDatabases(final int startInclusive, final int endExclusive) throws Exception { + // Reuse one session per phase, closing it before ConfigNode failover. + try (final ISession session = senderEnv.getSessionConnection()) { + for (int i = startInclusive; i < endExclusive; ++i) { + session.executeNonQueryStatement("create database " + CONFIG_DATABASE_PREFIX + i); + } + } + } + + private static String createTimeseriesStatement(final int index) { + return String.format( + "create timeSeries %s.wf01.GPS.status%d with datatype=BOOLEAN,encoding=PLAIN", + SCHEMA_DATABASE, index); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java index 053fdee0731ac..47ca39ed1ba56 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java @@ -23,6 +23,8 @@ import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; @@ -33,14 +35,19 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; -import java.sql.Connection; import java.util.Collections; import java.util.HashMap; import java.util.Map; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeMetaRestartIT extends AbstractPipeDualTreeModelManualIT { +public class IoTDBPipeMetaRestartIT extends AbstractPipeTreeModelManualSharedClusterIT { + + private static final String SCHEMA_DATABASE = "root.meta_restart_schema"; + private static final String CONFIG_DATABASE_PREFIX = "root.meta_restart_config"; + + // Create ten timeseries before restarting both clusters and ten afterward; the recovered schema + // task must replicate all twenty into this scenario's database. @Test public void testAutoRestartSchemaTask() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -57,7 +64,7 @@ public void testAutoRestartSchemaTask() throws Exception { sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); sourceAttributes.put("source.forwarding-pipe-requests", "false"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.ip", receiverIp); @@ -77,39 +84,19 @@ public void testAutoRestartSchemaTask() throws Exception { TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); } - for (int i = 0; i < 10; ++i) { - TestUtils.executeNonQuery( - senderEnv, - String.format( - "create timeseries root.ln.wf01.GPS.status%s with datatype=BOOLEAN,encoding=PLAIN", - i), - null); - } - - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (Throwable e) { - e.printStackTrace(); - return; - } - - for (int i = 10; i < 20; ++i) { - TestUtils.executeNonQuery( - senderEnv, - String.format( - "create timeseries root.ln.wf01.GPS.status%s with datatype=BOOLEAN,encoding=PLAIN", - i), - null); - } + createTimeseries(0, 10); + restartClusters(); + createTimeseries(10, 20); TestUtils.assertDataEventuallyOnEnv( receiverEnv, - "count timeseries root.ln.**", + "count timeseries " + SCHEMA_DATABASE + ".**", "count(timeseries),", Collections.singleton("20,")); } + // Create ten databases before restarting both clusters and ten afterward; the recovered config + // task must replicate all twenty, using a prefix distinct from the schema-restart scenario. @Test public void testAutoRestartConfigTask() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -126,7 +113,7 @@ public void testAutoRestartConfigTask() throws Exception { sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); sourceAttributes.put("source.forwarding-pipe-requests", "false"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.ip", receiverIp); @@ -146,28 +133,42 @@ public void testAutoRestartConfigTask() throws Exception { TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); } - try (Connection connection = senderEnv.getConnection()) { - for (int i = 0; i < 10; ++i) { - TestUtils.executeNonQuery( - senderEnv, String.format("create database root.ln%s", i), connection); - } - } + createDatabases(0, 10); + restartClusters(); + createDatabases(10, 20); - try { - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "count databases " + CONFIG_DATABASE_PREFIX + "*", + "count,", + Collections.singleton("20,")); + } + + private void createTimeseries(final int startInclusive, final int endExclusive) throws Exception { + // Reuse one connection per phase, closing it before the cluster restart. + try (final ISession session = senderEnv.getSessionConnection()) { + for (int i = startInclusive; i < endExclusive; ++i) { + session.executeNonQueryStatement( + String.format( + "create timeseries %s.wf01.GPS.status%d with datatype=BOOLEAN,encoding=PLAIN", + SCHEMA_DATABASE, i)); + } } + } - try (Connection connection = senderEnv.getConnection()) { - for (int i = 10; i < 20; ++i) { - TestUtils.executeNonQuery(senderEnv, String.format("create database root.ln%s", i), null); + private void createDatabases(final int startInclusive, final int endExclusive) throws Exception { + try (final ISession session = senderEnv.getSessionConnection()) { + for (int i = startInclusive; i < endExclusive; ++i) { + session.executeNonQueryStatement("create database " + CONFIG_DATABASE_PREFIX + i); } } + } - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "count databases root.ln*", "count,", Collections.singleton("20,")); + private void restartClusters() { + // Keep the receiver available while the sender restores its pipe tasks: restarting both at + // once can fail the recovery handshake and wait for the next periodic metadata sync. + // Propagate restart failures so an unsuccessful recovery cannot pass without the assertions. + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java index f5677edabb7b0..287ac0d2f246c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java @@ -25,7 +25,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; @@ -33,7 +33,6 @@ import org.awaitility.Awaitility; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -48,20 +47,17 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipePermissionIT extends AbstractPipeDualTreeModelManualIT { +public class IoTDBPipePermissionIT extends AbstractPipeTreeModelManualSharedClusterIT { @Override - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - + protected void setupConfig() { + super.setupConfig(); senderEnv .getConfig() .getCommonConfig() @@ -91,17 +87,22 @@ public void setUp() { .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false) .setPipeAutoSplitFullEnabled(false); + } - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(3, 3); + @Override + protected CompletableFuture initializeClusters() { + return CompletableFuture.allOf( + CompletableFuture.runAsync(senderEnv::initClusterEnvironment), + CompletableFuture.runAsync(() -> receiverEnv.initClusterEnvironment(3, 3))); } + // Both thrift sinks must authenticate as the receiver user and replicate schema, rows and users. @Test public void testWithSyncAndAsyncSink() throws Exception { TestUtils.executeNonQueries( receiverEnv, Arrays.asList( - "create user `thulab` 'passwd123456'", + "create user `thulab` 'IoTDB!Test2024'", "create role `admin`", "grant role `admin` to `thulab`", "grant WRITE, READ, SYSTEM, SECURITY on root.** to role `admin`"), @@ -125,7 +126,7 @@ private void testWithSink(final String sink, final String suffix) throws Excepti TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "create user " + sourceUser + " 'passwd123456'", + "create user " + sourceUser + " 'IoTDB!Test2024'", "create timeseries " + device + ".temperature with datatype=INT64,encoding=PLAIN", "create timeseries " + device + ".status with datatype=BOOLEAN,encoding=PLAIN", "insert into " @@ -139,13 +140,13 @@ private void testWithSink(final String sink, final String suffix) throws Excepti final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("source.inclusion", "all"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", sink); sinkAttributes.put("sink.ip", receiverIp); sinkAttributes.put("sink.port", Integer.toString(receiverPort)); sinkAttributes.put("sink.username", "thulab"); - sinkAttributes.put("sink.password", "passwd123456"); + sinkAttributes.put("sink.password", "IoTDB!Test2024"); final TSStatus status = client.createPipe( @@ -202,17 +203,34 @@ private void assertUserEventuallyExists(final String username) { }); } + // A receiver user with read-only privileges must receive neither source users nor timeseries. @Test public void testNoPermission() throws Exception { TestUtils.executeNonQueries( receiverEnv, Arrays.asList( - "create user `thulab` 'passwd123456'", + "create user `thulab` 'IoTDB!Test2024'", "create role `admin`", "grant role `admin` to `thulab`", "grant READ on root.ln.** to role `admin`"), null); + // User IDs keep increasing after users are dropped in a reused cluster. Verify the initial + // names, then retain their actual IDs to detect any unauthorized user replication. + final Set expectedUserRows = new HashSet<>(); + final Set expectedUserNames = new HashSet<>(); + try (final Connection connection = receiverEnv.getConnection(); + final Statement statement = connection.createStatement(); + final ResultSet result = statement.executeQuery("list user")) { + while (result.next()) { + final String username = result.getString("User"); + expectedUserNames.add(username); + expectedUserRows.add(result.getString("UserId") + "," + username + ","); + } + } + Assert.assertEquals( + new HashSet<>(Arrays.asList(SessionConfig.DEFAULT_USER, "thulab")), expectedUserNames); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); @@ -222,7 +240,7 @@ public void testNoPermission() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "create user someUser 'passwd'", + "create user someUser 'IoTDB!Test2024'", "create timeseries root.noPermission.wf02.wt01.status with datatype=BOOLEAN,encoding=PLAIN"), null); @@ -231,13 +249,13 @@ public void testNoPermission() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("source.inclusion", "all"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "iotdb-thrift-async-sink"); sinkAttributes.put("sink.ip", receiverIp); sinkAttributes.put("sink.port", Integer.toString(receiverPort)); sinkAttributes.put("sink.username", "thulab"); - sinkAttributes.put("sink.password", "passwd123456"); + sinkAttributes.put("sink.password", "IoTDB!Test2024"); final TSStatus status = client.createPipe( @@ -258,10 +276,11 @@ public void testNoPermission() throws Exception { receiverEnv, "list user", ColumnHeaderConstant.USER_ID + "," + ColumnHeaderConstant.USER + ",", - new HashSet<>(Arrays.asList("0,root,", "10000,thulab,"))); + expectedUserRows); } } + // Granting source READ must unblock aligned rows; the two-stage processor uses that source user. @Test public void testSourcePermissionForDataAndTwoStageProcessor() throws Exception { try (final Connection connection = senderEnv.getConnection(); @@ -344,8 +363,8 @@ private void testCountPointProcessorUsesSourceCredentials() throws Exception { sinkAttributes.put("sink.batch.enable", "false"); sinkAttributes.put("sink.ip", receiverDataNode.getIp()); sinkAttributes.put("sink.port", Integer.toString(receiverDataNode.getPort())); - sinkAttributes.put("sink.user", "root"); - sinkAttributes.put("sink.password", "root"); + sinkAttributes.put("sink.user", SessionConfig.DEFAULT_USER); + sinkAttributes.put("sink.password", SessionConfig.DEFAULT_PASSWORD); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { @@ -373,6 +392,7 @@ private void testCountPointProcessorUsesSourceCredentials() throws Exception { Collections.singleton("3,")); } + // Missing/wrong source credentials fail; schema and rows stay filtered until grants permit them. @Test public void testSourcePermission() { TestUtils.executeNonQuery(senderEnv, "create user `thulab` 'passwD@123456'", null); @@ -604,6 +624,8 @@ public void testSourcePermission() { } } + // Wrong passwords fail creation. After each sender restart, correcting the source password + // or restoring the user's password must resume replication of all three rows. @Test public void testIllegalPassword() throws Exception { TestUtils.executeNonQueries( @@ -622,116 +644,108 @@ public void testIllegalPassword() throws Exception { receiverEnv, "create aligned timeSeries root.vehicle.plane(temperature DOUBLE, pressure INT32)"); - Connection connection = senderEnv.getConnection(); - Statement statement = connection.createStatement(); - try { + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + try { + statement.execute( + String.format( + "create pipe a2b" + + " with source (" + + "'user'='thulab'" + + ", 'password'='passwd')" + + " with sink (" + + "'node-urls'='%s')", + receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); + fail(); + } catch (final Exception e) { + Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); + } + + try { + statement.execute( + "create pipe a2b ('sink'='write-back-sink', 'user'='thulab', 'password'='passwd')"); + fail(); + } catch (final Exception e) { + Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); + } + statement.execute( String.format( "create pipe a2b" + " with source (" + "'user'='thulab'" - + ", 'password'='passwd')" + + ", 'password'='ST@ongpasswd123456')" + " with sink (" + "'node-urls'='%s')", receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); - fail(); - } catch (final Exception e) { - Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); - } - try { - statement.execute( - "create pipe a2b ('sink'='write-back-sink', 'user'='thulab', 'password'='passwd')"); - fail(); - } catch (final Exception e) { - Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); - } - - statement.execute( - String.format( - "create pipe a2b" - + " with source (" - + "'user'='thulab'" - + ", 'password'='ST@ongpasswd123456')" - + " with sink (" - + "'node-urls'='%s')", - receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); + TestUtils.executeNonQuery( + senderEnv, "insert into root.vehicle.plane(temperature, pressure) values (36.5, 1103)"); - TestUtils.executeNonQuery( - senderEnv, "insert into root.vehicle.plane(temperature, pressure) values (36.5, 1103)"); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(pressure) from root.vehicle.plane", - "count(root.vehicle.plane.pressure),", - Collections.singleton("1,")); - - statement.execute("alter user thulab set password 'newST@ongPassword'"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(pressure) from root.vehicle.plane", + "count(root.vehicle.plane.pressure),", + Collections.singleton("1,")); - try { - TestUtils.restartCluster(senderEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; + statement.execute("alter user thulab set password 'newST@ongPassword1'"); } - connection = senderEnv.getConnection(); - statement = connection.createStatement(); - TestUtils.executeNonQuery( - senderEnv, "insert into root.vehicle.plane(temperature, pressure) values (36.5, 1103)"); + // Close clients before restarting so the shared cluster does not retain stale connections. + // Restart failures must fail the test, otherwise recovery coverage would silently be skipped. + TestUtils.restartCluster(senderEnv); - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "select count(pressure) from root.vehicle.plane", - "count(root.vehicle.plane.pressure),", - Collections.singleton("1,")); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + TestUtils.executeNonQuery( + senderEnv, "insert into root.vehicle.plane(temperature, pressure) values (36.5, 1103)"); - // After restart, the pipe keeps retrying with the stale password and may trigger login lock. - statement.execute("alter user thulab account unlock"); + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "select count(pressure) from root.vehicle.plane", + "count(root.vehicle.plane.pressure),", + Collections.singleton("1,")); - try { - statement.execute("alter pipe a2b modify source ('password'='fake')"); - fail(); - } catch (final SQLException e) { - Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); - } + // After restart, the pipe keeps retrying with the stale password and may trigger login lock. + statement.execute("alter user thulab account unlock"); - statement.execute("alter user thulab account unlock"); - statement.execute("alter pipe a2b modify source ('password'='newST@ongPassword')"); + try { + statement.execute("alter pipe a2b modify source ('password'='fake')"); + fail(); + } catch (final SQLException e) { + Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); + } - // Test empty alter - statement.execute("alter pipe a2b"); + statement.execute("alter user thulab account unlock"); + statement.execute("alter pipe a2b modify source ('password'='newST@ongPassword1')"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(pressure) from root.vehicle.plane", - "count(root.vehicle.plane.pressure),", - Collections.singleton("2,")); + // Test empty alter + statement.execute("alter pipe a2b"); - statement.execute("alter user thulab set password 'anotherST@ongPassword'"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(pressure) from root.vehicle.plane", + "count(root.vehicle.plane.pressure),", + Collections.singleton("2,")); - try { - TestUtils.restartCluster(senderEnv); - } catch (final Throwable e) { - e.printStackTrace(); - return; + statement.execute("alter user thulab set password 'anotherST@ongPassword1'"); } - connection = senderEnv.getConnection(); - statement = connection.createStatement(); - TestUtils.executeNonQuery( - senderEnv, "insert into root.vehicle.plane(temperature, pressure) values (36.5, 1103)"); - statement.execute("alter user thulab account unlock"); - statement.execute("alter user thulab set password 'newST@ongPassword'"); - statement.execute("alter pipe a2b"); + TestUtils.restartCluster(senderEnv); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(pressure) from root.vehicle.plane", - "count(root.vehicle.plane.pressure),", - Collections.singleton("3,")); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + TestUtils.executeNonQuery( + senderEnv, "insert into root.vehicle.plane(temperature, pressure) values (36.5, 1103)"); + statement.execute("alter user thulab account unlock"); + statement.execute("alter user thulab set password 'newST@ongPassword1'"); + statement.execute("alter pipe a2b"); - statement.close(); - connection.close(); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(pressure) from root.vehicle.plane", + "count(root.vehicle.plane.pressure),", + Collections.singleton("3,")); + } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeReqAutoSliceIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeReqAutoSliceIT.java index 03d119de5878f..c65d03d5fb47d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeReqAutoSliceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeReqAutoSliceIT.java @@ -57,6 +57,7 @@ protected void setupConfig() { receiverEnv.getConfig().getCommonConfig().setPipeConnectorRequestSliceThresholdBytes(4); } + // A ten-row tablet exceeding the four-byte request threshold must arrive intact after reassembly. @Test public void insertTablet() { prepareReqAutoSliceTest( @@ -313,7 +314,7 @@ private void prepareReqAutoSliceTest( createPipe(senderSession, true); } else { createPipe(senderSession, false); - Thread.sleep(2000); + // Pipe creation has completed; verify() waits for delivery of the sliced requests. consumer.accept(senderSession, receiverSession, tablet); senderSession.executeNonQueryStatement("flush"); } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index 87571e1eb71e6..b04008e42cfa7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -20,21 +20,81 @@ package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import java.util.Collections; import java.util.HashSet; - -import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueryWithRetry; +import java.util.Set; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeTsFileDecompositionWithModsIT extends AbstractPipeDualTreeModelManualIT { +public class IoTDBPipeTsFileDecompositionWithModsIT + extends AbstractPipeTreeModelManualSharedClusterIT { + + // Larger SQL batches retain every timestamp and flush boundary while reducing write RPCs by 10x. + // Keep all 2000 individual DELETEs in each large case: they exercise many modification entries. + private static final int INSERT_BATCH_SIZE = 500; + + @Rule public final TestName testName = new TestName(); + + private ISession senderSession; + private ISession receiverSession; + private String database; + + @Before + public void openSessions() throws Exception { + database = "root.mods_" + Integer.toUnsignedString(testName.getMethodName().hashCode(), 16); + senderSession = senderEnv.getSessionConnection(); + receiverSession = receiverEnv.getSessionConnection(); + } + + @After + public void closeSessions() throws Exception { + // JUnit closes these before the inherited cluster reset, even if a test or session open fails. + try { + if (senderSession != null) { + senderSession.close(); + } + } finally { + if (receiverSession != null) { + receiverSession.close(); + } + } + } + + private void executeOnSender(final String sql) throws Exception { + senderSession.executeNonQueryStatement(inDatabase(sql)); + } + + private void executeOnReceiver(final String sql) throws Exception { + receiverSession.executeNonQueryStatement(inDatabase(sql)); + } + + private void assertDataEventually( + final BaseEnv env, + final String sql, + final String expectedHeader, + final Set expectedRows) { + TestUtils.assertDataEventuallyOnEnv( + env, inDatabase(sql), inDatabase(expectedHeader), expectedRows); + } + + private String inDatabase(final String sqlOrHeader) { + // Keep the original deletion scenarios readable, but qualify SQL, pipe filters and expected + // headers with this method's database so shared clusters cannot mix their schemas or data. + return sqlOrHeader.replace("root.sg1", database); + } @Override protected void setupConfig() { @@ -64,41 +124,31 @@ protected void setupConfig() { */ @Test public void testTsFileDecompositionWithMods() throws Exception { - TestUtils.executeNonQueryWithRetry(senderEnv, "CREATE DATABASE root.sg1"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "CREATE DATABASE root.sg1"); - - TestUtils.executeNonQueryWithRetry( - senderEnv, "CREATE ALIGNED TIMESERIES root.sg1.d1(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); - TestUtils.executeNonQueryWithRetry( - senderEnv, "CREATE TIMESERIES root.sg1.d2.s1 WITH DATATYPE=FLOAT"); - TestUtils.executeNonQueryWithRetry( - senderEnv, "CREATE TIMESERIES root.sg1.d2.s2 WITH DATATYPE=FLOAT"); - TestUtils.executeNonQueryWithRetry( - senderEnv, "CREATE ALIGNED TIMESERIES root.sg1.d3(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); - - TestUtils.executeNonQueryWithRetry( - senderEnv, + executeOnSender("CREATE DATABASE root.sg1"); + executeOnReceiver("CREATE DATABASE root.sg1"); + + executeOnSender("CREATE ALIGNED TIMESERIES root.sg1.d1(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + executeOnSender("CREATE TIMESERIES root.sg1.d2.s1 WITH DATATYPE=FLOAT"); + executeOnSender("CREATE TIMESERIES root.sg1.d2.s2 WITH DATATYPE=FLOAT"); + executeOnSender("CREATE ALIGNED TIMESERIES root.sg1.d3(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + + executeOnSender( "INSERT INTO root.sg1.d1(time, s1, s2, s3) ALIGNED VALUES (1, 1.0, 2.0, 3.0), (2, 1.1, 2.1, 3.1), (3, 1.2, 2.2, 3.2), (4, 1.3, 2.3, 3.3), (5, 1.4, 2.4, 3.4)"); - TestUtils.executeNonQueryWithRetry( - senderEnv, + executeOnSender( "INSERT INTO root.sg1.d2(time, s1, s2) VALUES (1, 10.0, 20.0), (2, 10.1, 20.1), (3, 10.2, 20.2), (4, 10.3, 20.3), (5, 10.4, 20.4)"); - TestUtils.executeNonQueryWithRetry( - senderEnv, + executeOnSender( "INSERT INTO root.sg1.d3(time, s1, s2, s3) ALIGNED VALUES (1, 100.0, 200.0, 300.0), (2, 100.1, 200.1, 300.1), (3, 100.2, 200.2, 300.2)"); - TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + executeOnSender("FLUSH"); - TestUtils.executeNonQueryWithRetry( - senderEnv, "DELETE FROM root.sg1.d1.s1 WHERE time >= 2 AND time <= 4"); + executeOnSender("DELETE FROM root.sg1.d1.s1 WHERE time >= 2 AND time <= 4"); - TestUtils.executeNonQueryWithRetry( - senderEnv, + executeOnSender( "INSERT INTO root.sg1.d3(time, s1, s2, s3) ALIGNED VALUES (1, 100.0, 200.0, 300.0), (2, 100.1, 200.1, 300.1), (3, 100.2, 200.2, 300.2)"); - TestUtils.executeNonQueryWithRetry( - senderEnv, "CREATE ALIGNED TIMESERIES root.sg1.d4(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + executeOnSender("CREATE ALIGNED TIMESERIES root.sg1.d4(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); String s = "INSERT INTO root.sg1.d4(time, s1, s2, s3) ALIGNED VALUES "; StringBuilder insertBuilder = new StringBuilder(s); for (int i = 1; i <= 11000; i++) { @@ -112,70 +162,69 @@ public void testTsFileDecompositionWithMods() throws Exception { .append(",") .append(3.0f) .append(")"); - if (i % 50 != 0) { + if (i % INSERT_BATCH_SIZE != 0) { insertBuilder.append(","); } else { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder.toString()); + executeOnSender(insertBuilder.toString()); insertBuilder = new StringBuilder(s); } } - TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + executeOnSender("FLUSH"); - TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d4.s1 WHERE time <= 10000"); - TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d4.s2 WHERE time > 1000"); - TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d4.s3 WHERE time <= 8000"); + executeOnSender("DELETE FROM root.sg1.d4.s1 WHERE time <= 10000"); + executeOnSender("DELETE FROM root.sg1.d4.s2 WHERE time > 1000"); + executeOnSender("DELETE FROM root.sg1.d4.s3 WHERE time <= 8000"); - TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d2.*"); + executeOnSender("DELETE FROM root.sg1.d2.*"); - TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d3.*"); + executeOnSender("DELETE FROM root.sg1.d3.*"); - TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + executeOnSender("FLUSH"); // Verify sender data integrity before creating pipe to avoid leader election issues // This ensures all data is properly persisted and consistent on sender side // before starting the pipe synchronization process - HashSet results = new HashSet<>(); + Set results = new HashSet<>(); results.add("1,3.0,1.0,2.0,"); results.add("2,3.1,null,2.1,"); results.add("3,3.2,null,2.2,"); results.add("4,3.3,null,2.3,"); results.add("5,3.4,1.4,2.4,"); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( senderEnv, - "SELECT * FROM root.sg1.d1 ORDER BY time", + "SELECT s3, s1, s2 FROM root.sg1.d1 ORDER BY time", "Time,root.sg1.d1.s3,root.sg1.d1.s1,root.sg1.d1.s2,", results); - executeNonQueryWithRetry( - senderEnv, + executeOnSender( String.format( - "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", + "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true','path'='root.sg1.**') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", receiverEnv.getDataNodeWrapperList().get(0).getIp(), receiverEnv.getDataNodeWrapperList().get(0).getPort())); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, - "SELECT * FROM root.sg1.d1 ORDER BY time", + "SELECT s3, s1, s2 FROM root.sg1.d1 ORDER BY time", "Time,root.sg1.d1.s3,root.sg1.d1.s1,root.sg1.d1.s2,", results); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT * FROM root.sg1.d2 ORDER BY time", "Time,", Collections.emptySet()); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT * FROM root.sg1.d3 ORDER BY time", "Time,", Collections.emptySet()); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT s1 FROM root.sg1.d1 WHERE time >= 2 AND time <= 4", "Time,root.sg1.d1.s1,", Collections.emptySet()); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, - "SELECT COUNT(**) FROM root.sg1.d4", + "SELECT COUNT(s3), COUNT(s1), COUNT(s2) FROM root.sg1.d4", "COUNT(root.sg1.d4.s3),COUNT(root.sg1.d4.s1),COUNT(root.sg1.d4.s2),", Collections.singleton("3000,1000,1000,")); } @@ -206,41 +255,31 @@ public void testTsFileDecompositionWithMods() throws Exception { */ @Test public void testTsFileDecompositionWithMods2() throws Exception { - TestUtils.executeNonQueryWithRetry(senderEnv, "CREATE DATABASE root.sg1"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "CREATE DATABASE root.sg1"); - - TestUtils.executeNonQueryWithRetry( - senderEnv, "CREATE ALIGNED TIMESERIES root.sg1.d1(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); - TestUtils.executeNonQueryWithRetry( - senderEnv, "CREATE TIMESERIES root.sg1.d2.s1 WITH DATATYPE=FLOAT"); - TestUtils.executeNonQueryWithRetry( - senderEnv, "CREATE TIMESERIES root.sg1.d2.s2 WITH DATATYPE=FLOAT"); - TestUtils.executeNonQueryWithRetry( - senderEnv, "CREATE ALIGNED TIMESERIES root.sg1.d3(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); - - TestUtils.executeNonQueryWithRetry( - senderEnv, + executeOnSender("CREATE DATABASE root.sg1"); + executeOnReceiver("CREATE DATABASE root.sg1"); + + executeOnSender("CREATE ALIGNED TIMESERIES root.sg1.d1(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + executeOnSender("CREATE TIMESERIES root.sg1.d2.s1 WITH DATATYPE=FLOAT"); + executeOnSender("CREATE TIMESERIES root.sg1.d2.s2 WITH DATATYPE=FLOAT"); + executeOnSender("CREATE ALIGNED TIMESERIES root.sg1.d3(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + + executeOnSender( "INSERT INTO root.sg1.d1(time, s1, s2, s3) ALIGNED VALUES (1, 1.0, 2.0, 3.0), (2, 1.1, 2.1, 3.1), (3, 1.2, 2.2, 3.2), (4, 1.3, 2.3, 3.3), (5, 1.4, 2.4, 3.4)"); - TestUtils.executeNonQueryWithRetry( - senderEnv, + executeOnSender( "INSERT INTO root.sg1.d2(time, s1, s2) VALUES (1, 10.0, 20.0), (2, 10.1, 20.1), (3, 10.2, 20.2), (4, 10.3, 20.3), (5, 10.4, 20.4)"); - TestUtils.executeNonQueryWithRetry( - senderEnv, + executeOnSender( "INSERT INTO root.sg1.d3(time, s1, s2, s3) ALIGNED VALUES (1, 100.0, 200.0, 300.0), (2, 100.1, 200.1, 300.1), (3, 100.2, 200.2, 300.2)"); - TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + executeOnSender("FLUSH"); - TestUtils.executeNonQueryWithRetry( - senderEnv, "DELETE FROM root.sg1.d1.s1 WHERE time >= 2 AND time <= 4"); + executeOnSender("DELETE FROM root.sg1.d1.s1 WHERE time >= 2 AND time <= 4"); - TestUtils.executeNonQueryWithRetry( - senderEnv, + executeOnSender( "INSERT INTO root.sg1.d3(time, s1, s2, s3) ALIGNED VALUES (1, 100.0, 200.0, 300.0), (2, 100.1, 200.1, 300.1), (3, 100.2, 200.2, 300.2)"); - TestUtils.executeNonQueryWithRetry( - senderEnv, "CREATE ALIGNED TIMESERIES root.sg1.d4(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + executeOnSender("CREATE ALIGNED TIMESERIES root.sg1.d4(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); String s = "INSERT INTO root.sg1.d4(time, s1, s2, s3) ALIGNED VALUES "; StringBuilder insertBuilder = new StringBuilder(s); for (int i = 1; i <= 11000; i++) { @@ -254,91 +293,87 @@ public void testTsFileDecompositionWithMods2() throws Exception { .append(",") .append(3.0f) .append(")"); - if (i % 50 != 0) { + if (i % INSERT_BATCH_SIZE != 0) { insertBuilder.append(","); } else { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder.toString()); + executeOnSender(insertBuilder.toString()); insertBuilder = new StringBuilder(s); } } - TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + executeOnSender("FLUSH"); - TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d4.s1 WHERE time <= 10000"); - TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d4.s2 WHERE time > 1000"); - TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d4.s3 WHERE time <= 8000"); + executeOnSender("DELETE FROM root.sg1.d4.s1 WHERE time <= 10000"); + executeOnSender("DELETE FROM root.sg1.d4.s2 WHERE time > 1000"); + executeOnSender("DELETE FROM root.sg1.d4.s3 WHERE time <= 8000"); - TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d2.*"); + executeOnSender("DELETE FROM root.sg1.d2.*"); - TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d3.*"); + executeOnSender("DELETE FROM root.sg1.d3.*"); - TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + executeOnSender("FLUSH"); // Verify sender data integrity before creating pipes to avoid leader election issues // This ensures all data is properly persisted and consistent on sender side // before starting the pipe synchronization process - HashSet results = new HashSet<>(); + Set results = new HashSet<>(); results.add("1,3.0,1.0,2.0,"); results.add("2,3.1,null,2.1,"); results.add("3,3.2,null,2.2,"); results.add("4,3.3,null,2.3,"); results.add("5,3.4,1.4,2.4,"); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( senderEnv, - "SELECT * FROM root.sg1.d1 ORDER BY time", + "SELECT s3, s1, s2 FROM root.sg1.d1 ORDER BY time", "Time,root.sg1.d1.s3,root.sg1.d1.s1,root.sg1.d1.s2,", results); - executeNonQueryWithRetry( - senderEnv, + executeOnSender( String.format( "CREATE PIPE test_pipe1 WITH SOURCE ('mods.enable'='true','path'='root.sg1.d1.**') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", receiverEnv.getDataNodeWrapperList().get(0).getIp(), receiverEnv.getDataNodeWrapperList().get(0).getPort())); - executeNonQueryWithRetry( - senderEnv, + executeOnSender( String.format( "CREATE PIPE test_pipe2 WITH SOURCE ('mods.enable'='true','path'='root.sg1.d2.**') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", receiverEnv.getDataNodeWrapperList().get(0).getIp(), receiverEnv.getDataNodeWrapperList().get(0).getPort())); - executeNonQueryWithRetry( - senderEnv, + executeOnSender( String.format( "CREATE PIPE test_pipe3 WITH SOURCE ('mods.enable'='true','path'='root.sg1.d3.**') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", receiverEnv.getDataNodeWrapperList().get(0).getIp(), receiverEnv.getDataNodeWrapperList().get(0).getPort())); - executeNonQueryWithRetry( - senderEnv, + executeOnSender( String.format( "CREATE PIPE test_pipe4 WITH SOURCE ('mods.enable'='true','path'='root.sg1.d4.**') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", receiverEnv.getDataNodeWrapperList().get(0).getIp(), receiverEnv.getDataNodeWrapperList().get(0).getPort())); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, - "SELECT * FROM root.sg1.d1 ORDER BY time", + "SELECT s3, s1, s2 FROM root.sg1.d1 ORDER BY time", "Time,root.sg1.d1.s3,root.sg1.d1.s1,root.sg1.d1.s2,", results); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT * FROM root.sg1.d2 ORDER BY time", "Time,", Collections.emptySet()); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT * FROM root.sg1.d3 ORDER BY time", "Time,", Collections.emptySet()); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT s1 FROM root.sg1.d1 WHERE time >= 2 AND time <= 4", "Time,root.sg1.d1.s1,", Collections.emptySet()); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, - "SELECT COUNT(**) FROM root.sg1.d4", + "SELECT COUNT(s3), COUNT(s1), COUNT(s2) FROM root.sg1.d4", "COUNT(root.sg1.d4.s3),COUNT(root.sg1.d4.s1),COUNT(root.sg1.d4.s2),", Collections.singleton("3000,1000,1000,")); } @@ -353,7 +388,7 @@ public void testTsFileDecompositionWithMods2() throws Exception { * 40001-60000 3. Execute FLUSH operation to persist data to TsFile 4. Execute 2000 single point * DELETE operations, each deleting one time point from different sensors 5. Execute FLUSH * operation again 6. Create pipe with mods enabled 7. Verify correctness of receiver data: - Each - * sensor should have 19800 remaining data points - Deleted points should not appear in receiver + * sensor should have 19600 remaining data points - Deleted points should not appear in receiver * *

Test purpose: Verify that IoTDB pipe can correctly handle large scale single point deletion * operations in TsFile under tree model, ensuring the binary search optimization in @@ -361,24 +396,24 @@ public void testTsFileDecompositionWithMods2() throws Exception { */ @Test public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletion() throws Exception { - TestUtils.executeNonQueryWithRetry(senderEnv, "CREATE DATABASE root.sg1"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "CREATE DATABASE root.sg1"); + executeOnSender("CREATE DATABASE root.sg1"); + executeOnReceiver("CREATE DATABASE root.sg1"); // Insert 20000 data points for s1 (time 1-20000) String s1 = "INSERT INTO root.sg1.d1(time, s1) ALIGNED VALUES "; StringBuilder insertBuilder1 = new StringBuilder(s1); for (int i = 1; i <= 20000; i++) { insertBuilder1.append("(").append(i).append(",").append(1.0f).append(")"); - if (i % 50 != 0) { + if (i % INSERT_BATCH_SIZE != 0) { insertBuilder1.append(","); } else { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder1.toString()); + executeOnSender(insertBuilder1.toString()); insertBuilder1 = new StringBuilder(s1); } } // Execute remaining data if any if (insertBuilder1.length() > s1.length()) { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder1.toString()); + executeOnSender(insertBuilder1.toString()); } // Insert 20000 data points for s2 (time 10001-30000) @@ -386,16 +421,16 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletion() throw StringBuilder insertBuilder2 = new StringBuilder(s2); for (int i = 10001; i <= 30000; i++) { insertBuilder2.append("(").append(i).append(",").append(2.0f).append(")"); - if (i % 50 != 0) { + if (i % INSERT_BATCH_SIZE != 0) { insertBuilder2.append(","); } else { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder2.toString()); + executeOnSender(insertBuilder2.toString()); insertBuilder2 = new StringBuilder(s2); } } // Execute remaining data if any if (insertBuilder2.length() > s2.length()) { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder2.toString()); + executeOnSender(insertBuilder2.toString()); } // Insert 20000 data points for s3 (time 20001-40000) @@ -403,16 +438,16 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletion() throw StringBuilder insertBuilder3 = new StringBuilder(s3); for (int i = 20001; i <= 40000; i++) { insertBuilder3.append("(").append(i).append(",").append(3.0f).append(")"); - if (i % 50 != 0) { + if (i % INSERT_BATCH_SIZE != 0) { insertBuilder3.append(","); } else { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder3.toString()); + executeOnSender(insertBuilder3.toString()); insertBuilder3 = new StringBuilder(s3); } } // Execute remaining data if any if (insertBuilder3.length() > s3.length()) { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder3.toString()); + executeOnSender(insertBuilder3.toString()); } // Insert 20000 data points for s4 (time 30001-50000) @@ -420,16 +455,16 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletion() throw StringBuilder insertBuilder4 = new StringBuilder(s4); for (int i = 30001; i <= 50000; i++) { insertBuilder4.append("(").append(i).append(",").append(4.0f).append(")"); - if (i % 50 != 0) { + if (i % INSERT_BATCH_SIZE != 0) { insertBuilder4.append(","); } else { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder4.toString()); + executeOnSender(insertBuilder4.toString()); insertBuilder4 = new StringBuilder(s4); } } // Execute remaining data if any if (insertBuilder4.length() > s4.length()) { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder4.toString()); + executeOnSender(insertBuilder4.toString()); } // Insert 20000 data points for s5 (time 40001-60000) @@ -437,95 +472,89 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletion() throw StringBuilder insertBuilder5 = new StringBuilder(s5); for (int i = 40001; i <= 60000; i++) { insertBuilder5.append("(").append(i).append(",").append(5.0f).append(")"); - if (i % 50 != 0) { + if (i % INSERT_BATCH_SIZE != 0) { insertBuilder5.append(","); } else { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder5.toString()); + executeOnSender(insertBuilder5.toString()); insertBuilder5 = new StringBuilder(s5); } } // Execute remaining data if any if (insertBuilder5.length() > s5.length()) { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder5.toString()); + executeOnSender(insertBuilder5.toString()); } - TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + executeOnSender("FLUSH"); // Execute 2000 single point DELETE operations // Delete 400 points from each sensor (distributed across different time ranges) for (int i = 0; i < 400; i++) { // Delete from s1: every 10th point starting from 10 - TestUtils.executeNonQueryWithRetry( - senderEnv, "DELETE FROM root.sg1.d1.s1 WHERE time = " + (10 + i * 10)); + executeOnSender("DELETE FROM root.sg1.d1.s1 WHERE time = " + (10 + i * 10)); // Delete from s2: every 10th point starting from 10010 - TestUtils.executeNonQueryWithRetry( - senderEnv, "DELETE FROM root.sg1.d1.s2 WHERE time = " + (10010 + i * 10)); + executeOnSender("DELETE FROM root.sg1.d1.s2 WHERE time = " + (10010 + i * 10)); // Delete from s3: every 10th point starting from 20010 - TestUtils.executeNonQueryWithRetry( - senderEnv, "DELETE FROM root.sg1.d1.s3 WHERE time = " + (20010 + i * 10)); + executeOnSender("DELETE FROM root.sg1.d1.s3 WHERE time = " + (20010 + i * 10)); // Delete from s4: every 10th point starting from 30010 - TestUtils.executeNonQueryWithRetry( - senderEnv, "DELETE FROM root.sg1.d1.s4 WHERE time = " + (30010 + i * 10)); + executeOnSender("DELETE FROM root.sg1.d1.s4 WHERE time = " + (30010 + i * 10)); // Delete from s5: every 10th point starting from 40010 - TestUtils.executeNonQueryWithRetry( - senderEnv, "DELETE FROM root.sg1.d1.s5 WHERE time = " + (40010 + i * 10)); + executeOnSender("DELETE FROM root.sg1.d1.s5 WHERE time = " + (40010 + i * 10)); } - TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + executeOnSender("FLUSH"); // Verify sender data integrity before creating pipe to avoid leader election issues // This ensures all data is properly persisted and consistent on sender side // before starting the pipe synchronization process - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( senderEnv, - "SELECT COUNT(**) FROM root.sg1.d1", + "SELECT COUNT(s3), COUNT(s4), COUNT(s5), COUNT(s1), COUNT(s2) FROM root.sg1.d1", "COUNT(root.sg1.d1.s3),COUNT(root.sg1.d1.s4),COUNT(root.sg1.d1.s5),COUNT(root.sg1.d1.s1),COUNT(root.sg1.d1.s2),", Collections.singleton("19600,19600,19600,19600,19600,")); - executeNonQueryWithRetry( - senderEnv, + executeOnSender( String.format( - "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", + "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true','path'='root.sg1.**') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", receiverEnv.getDataNodeWrapperList().get(0).getIp(), receiverEnv.getDataNodeWrapperList().get(0).getPort())); // Verify total count of all sensors using COUNT(*) - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, - "SELECT COUNT(**) FROM root.sg1.d1", + "SELECT COUNT(s3), COUNT(s4), COUNT(s5), COUNT(s1), COUNT(s2) FROM root.sg1.d1", "COUNT(root.sg1.d1.s3),COUNT(root.sg1.d1.s4),COUNT(root.sg1.d1.s5),COUNT(root.sg1.d1.s1),COUNT(root.sg1.d1.s2),", Collections.singleton("19600,19600,19600,19600,19600,")); // Verify individual sensor counts - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s1) FROM root.sg1.d1", "COUNT(root.sg1.d1.s1),", Collections.singleton("19600,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s2) FROM root.sg1.d1", "COUNT(root.sg1.d1.s2),", Collections.singleton("19600,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s3) FROM root.sg1.d1", "COUNT(root.sg1.d1.s3),", Collections.singleton("19600,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s4) FROM root.sg1.d1", "COUNT(root.sg1.d1.s4),", Collections.singleton("19600,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s5) FROM root.sg1.d1", "COUNT(root.sg1.d1.s5),", @@ -533,31 +562,31 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletion() throw // Verify count of deleted time ranges using COUNT with WHERE clause // These should return 0 since all points in these ranges were deleted - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s1) FROM root.sg1.d1 WHERE time >= 10 AND time <= 4000 AND time % 10 = 0", "COUNT(root.sg1.d1.s1),", Collections.singleton("0,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s2) FROM root.sg1.d1 WHERE time >= 10010 AND time <= 14000 AND time % 10 = 0", "COUNT(root.sg1.d1.s2),", Collections.singleton("0,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s3) FROM root.sg1.d1 WHERE time >= 20010 AND time <= 24000 AND time % 10 = 0", "COUNT(root.sg1.d1.s3),", Collections.singleton("0,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s4) FROM root.sg1.d1 WHERE time >= 30010 AND time <= 34000 AND time % 10 = 0", "COUNT(root.sg1.d1.s4),", Collections.singleton("0,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s5) FROM root.sg1.d1 WHERE time >= 40010 AND time <= 44000 AND time % 10 = 0", "COUNT(root.sg1.d1.s5),", @@ -565,93 +594,93 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletion() throw // Verify count of non-deleted time ranges using multiple range queries // Check ranges before deletion area - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s1) FROM root.sg1.d1 WHERE time >= 1 AND time < 10", "COUNT(root.sg1.d1.s1),", Collections.singleton("9,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s2) FROM root.sg1.d1 WHERE time >= 10001 AND time < 10010", "COUNT(root.sg1.d1.s2),", Collections.singleton("9,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s3) FROM root.sg1.d1 WHERE time >= 20001 AND time < 20010", "COUNT(root.sg1.d1.s3),", Collections.singleton("9,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s4) FROM root.sg1.d1 WHERE time >= 30001 AND time < 30010", "COUNT(root.sg1.d1.s4),", Collections.singleton("9,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s5) FROM root.sg1.d1 WHERE time >= 40001 AND time < 40010", "COUNT(root.sg1.d1.s5),", Collections.singleton("9,")); // Check ranges after deletion area - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s1) FROM root.sg1.d1 WHERE time > 4000 AND time <= 20000", "COUNT(root.sg1.d1.s1),", Collections.singleton("16000,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s2) FROM root.sg1.d1 WHERE time > 14000 AND time <= 30000", "COUNT(root.sg1.d1.s2),", Collections.singleton("16000,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s3) FROM root.sg1.d1 WHERE time > 24000 AND time <= 40000", "COUNT(root.sg1.d1.s3),", Collections.singleton("16000,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s4) FROM root.sg1.d1 WHERE time > 34000 AND time <= 50000", "COUNT(root.sg1.d1.s4),", Collections.singleton("16000,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s5) FROM root.sg1.d1 WHERE time > 44000 AND time <= 60000", "COUNT(root.sg1.d1.s5),", Collections.singleton("16000,")); // Check non-deleted points within deletion range (every 10th point except the ones we deleted) - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s1) FROM root.sg1.d1 WHERE time >= 10 AND time <= 4000 AND time % 10 != 0", "COUNT(root.sg1.d1.s1),", Collections.singleton("3591,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s2) FROM root.sg1.d1 WHERE time >= 10010 AND time <= 14000 AND time % 10 != 0", "COUNT(root.sg1.d1.s2),", Collections.singleton("3591,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s3) FROM root.sg1.d1 WHERE time >= 20010 AND time <= 24000 AND time % 10 != 0", "COUNT(root.sg1.d1.s3),", Collections.singleton("3591,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s4) FROM root.sg1.d1 WHERE time >= 30010 AND time <= 34000 AND time % 10 != 0", "COUNT(root.sg1.d1.s4),", Collections.singleton("3591,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s5) FROM root.sg1.d1 WHERE time >= 40010 AND time <= 44000 AND time % 10 != 0", "COUNT(root.sg1.d1.s5),", @@ -668,7 +697,7 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletion() throw * 40001-60000 3. Execute FLUSH operation to persist data to TsFile 4. Execute 2000 single point * DELETE operations, each deleting one time point from different sensors 5. Execute FLUSH * operation again 6. Create pipe with mods enabled 7. Verify correctness of receiver data: - Each - * sensor should have 19800 remaining data points - Deleted points should not appear in receiver + * sensor should have 19600 remaining data points - Deleted points should not appear in receiver * *

Test purpose: Verify that IoTDB pipe can correctly handle large scale single point deletion * operations in TsFile under tree model with non-aligned timeseries, ensuring the binary search @@ -677,24 +706,24 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletion() throw @Test public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletionNonAligned() throws Exception { - TestUtils.executeNonQueryWithRetry(senderEnv, "CREATE DATABASE root.sg1"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "CREATE DATABASE root.sg1"); + executeOnSender("CREATE DATABASE root.sg1"); + executeOnReceiver("CREATE DATABASE root.sg1"); // Insert 20000 data points for s1 (time 1-20000) String s1 = "INSERT INTO root.sg1.d1(time, s1) VALUES "; StringBuilder insertBuilder1 = new StringBuilder(s1); for (int i = 1; i <= 20000; i++) { insertBuilder1.append("(").append(i).append(",").append(1.0f).append(")"); - if (i % 50 != 0) { + if (i % INSERT_BATCH_SIZE != 0) { insertBuilder1.append(","); } else { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder1.toString()); + executeOnSender(insertBuilder1.toString()); insertBuilder1 = new StringBuilder(s1); } } // Execute remaining data if any if (insertBuilder1.length() > s1.length()) { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder1.toString()); + executeOnSender(insertBuilder1.toString()); } // Insert 20000 data points for s2 (time 10001-30000) @@ -702,16 +731,16 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletionNonAlign StringBuilder insertBuilder2 = new StringBuilder(s2); for (int i = 10001; i <= 30000; i++) { insertBuilder2.append("(").append(i).append(",").append(2.0f).append(")"); - if (i % 50 != 0) { + if (i % INSERT_BATCH_SIZE != 0) { insertBuilder2.append(","); } else { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder2.toString()); + executeOnSender(insertBuilder2.toString()); insertBuilder2 = new StringBuilder(s2); } } // Execute remaining data if any if (insertBuilder2.length() > s2.length()) { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder2.toString()); + executeOnSender(insertBuilder2.toString()); } // Insert 20000 data points for s3 (time 20001-40000) @@ -719,16 +748,16 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletionNonAlign StringBuilder insertBuilder3 = new StringBuilder(s3); for (int i = 20001; i <= 40000; i++) { insertBuilder3.append("(").append(i).append(",").append(3.0f).append(")"); - if (i % 50 != 0) { + if (i % INSERT_BATCH_SIZE != 0) { insertBuilder3.append(","); } else { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder3.toString()); + executeOnSender(insertBuilder3.toString()); insertBuilder3 = new StringBuilder(s3); } } // Execute remaining data if any if (insertBuilder3.length() > s3.length()) { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder3.toString()); + executeOnSender(insertBuilder3.toString()); } // Insert 20000 data points for s4 (time 30001-50000) @@ -736,16 +765,16 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletionNonAlign StringBuilder insertBuilder4 = new StringBuilder(s4); for (int i = 30001; i <= 50000; i++) { insertBuilder4.append("(").append(i).append(",").append(4.0f).append(")"); - if (i % 50 != 0) { + if (i % INSERT_BATCH_SIZE != 0) { insertBuilder4.append(","); } else { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder4.toString()); + executeOnSender(insertBuilder4.toString()); insertBuilder4 = new StringBuilder(s4); } } // Execute remaining data if any if (insertBuilder4.length() > s4.length()) { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder4.toString()); + executeOnSender(insertBuilder4.toString()); } // Insert 20000 data points for s5 (time 40001-60000) @@ -753,95 +782,89 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletionNonAlign StringBuilder insertBuilder5 = new StringBuilder(s5); for (int i = 40001; i <= 60000; i++) { insertBuilder5.append("(").append(i).append(",").append(5.0f).append(")"); - if (i % 50 != 0) { + if (i % INSERT_BATCH_SIZE != 0) { insertBuilder5.append(","); } else { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder5.toString()); + executeOnSender(insertBuilder5.toString()); insertBuilder5 = new StringBuilder(s5); } } // Execute remaining data if any if (insertBuilder5.length() > s5.length()) { - TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder5.toString()); + executeOnSender(insertBuilder5.toString()); } - TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + executeOnSender("FLUSH"); // Execute 2000 single point DELETE operations // Delete 400 points from each sensor (distributed across different time ranges) for (int i = 0; i < 400; i++) { // Delete from s1: every 10th point starting from 10 - TestUtils.executeNonQueryWithRetry( - senderEnv, "DELETE FROM root.sg1.d1.s1 WHERE time = " + (10 + i * 10)); + executeOnSender("DELETE FROM root.sg1.d1.s1 WHERE time = " + (10 + i * 10)); // Delete from s2: every 10th point starting from 10010 - TestUtils.executeNonQueryWithRetry( - senderEnv, "DELETE FROM root.sg1.d1.s2 WHERE time = " + (10010 + i * 10)); + executeOnSender("DELETE FROM root.sg1.d1.s2 WHERE time = " + (10010 + i * 10)); // Delete from s3: every 10th point starting from 20010 - TestUtils.executeNonQueryWithRetry( - senderEnv, "DELETE FROM root.sg1.d1.s3 WHERE time = " + (20010 + i * 10)); + executeOnSender("DELETE FROM root.sg1.d1.s3 WHERE time = " + (20010 + i * 10)); // Delete from s4: every 10th point starting from 30010 - TestUtils.executeNonQueryWithRetry( - senderEnv, "DELETE FROM root.sg1.d1.s4 WHERE time = " + (30010 + i * 10)); + executeOnSender("DELETE FROM root.sg1.d1.s4 WHERE time = " + (30010 + i * 10)); // Delete from s5: every 10th point starting from 40010 - TestUtils.executeNonQueryWithRetry( - senderEnv, "DELETE FROM root.sg1.d1.s5 WHERE time = " + (40010 + i * 10)); + executeOnSender("DELETE FROM root.sg1.d1.s5 WHERE time = " + (40010 + i * 10)); } - TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + executeOnSender("FLUSH"); // Verify sender data integrity before creating pipe to avoid leader election issues // This ensures all data is properly persisted and consistent on sender side // before starting the pipe synchronization process - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( senderEnv, - "SELECT COUNT(**) FROM root.sg1.d1", + "SELECT COUNT(s3), COUNT(s4), COUNT(s5), COUNT(s1), COUNT(s2) FROM root.sg1.d1", "COUNT(root.sg1.d1.s3),COUNT(root.sg1.d1.s4),COUNT(root.sg1.d1.s5),COUNT(root.sg1.d1.s1),COUNT(root.sg1.d1.s2),", Collections.singleton("19600,19600,19600,19600,19600,")); - executeNonQueryWithRetry( - senderEnv, + executeOnSender( String.format( - "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", + "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true','path'='root.sg1.**') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", receiverEnv.getDataNodeWrapperList().get(0).getIp(), receiverEnv.getDataNodeWrapperList().get(0).getPort())); // Verify total count of all sensors using COUNT(*) - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, - "SELECT COUNT(**) FROM root.sg1.d1", + "SELECT COUNT(s3), COUNT(s4), COUNT(s5), COUNT(s1), COUNT(s2) FROM root.sg1.d1", "COUNT(root.sg1.d1.s3),COUNT(root.sg1.d1.s4),COUNT(root.sg1.d1.s5),COUNT(root.sg1.d1.s1),COUNT(root.sg1.d1.s2),", Collections.singleton("19600,19600,19600,19600,19600,")); // Verify individual sensor counts - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s1) FROM root.sg1.d1", "COUNT(root.sg1.d1.s1),", Collections.singleton("19600,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s2) FROM root.sg1.d1", "COUNT(root.sg1.d1.s2),", Collections.singleton("19600,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s3) FROM root.sg1.d1", "COUNT(root.sg1.d1.s3),", Collections.singleton("19600,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s4) FROM root.sg1.d1", "COUNT(root.sg1.d1.s4),", Collections.singleton("19600,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s5) FROM root.sg1.d1", "COUNT(root.sg1.d1.s5),", @@ -849,31 +872,31 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletionNonAlign // Verify count of deleted time ranges using COUNT with WHERE clause // These should return 0 since all points in these ranges were deleted - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s1) FROM root.sg1.d1 WHERE time >= 10 AND time <= 4000 AND time % 10 = 0", "COUNT(root.sg1.d1.s1),", Collections.singleton("0,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s2) FROM root.sg1.d1 WHERE time >= 10010 AND time <= 14000 AND time % 10 = 0", "COUNT(root.sg1.d1.s2),", Collections.singleton("0,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s3) FROM root.sg1.d1 WHERE time >= 20010 AND time <= 24000 AND time % 10 = 0", "COUNT(root.sg1.d1.s3),", Collections.singleton("0,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s4) FROM root.sg1.d1 WHERE time >= 30010 AND time <= 34000 AND time % 10 = 0", "COUNT(root.sg1.d1.s4),", Collections.singleton("0,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s5) FROM root.sg1.d1 WHERE time >= 40010 AND time <= 44000 AND time % 10 = 0", "COUNT(root.sg1.d1.s5),", @@ -881,93 +904,93 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletionNonAlign // Verify count of non-deleted time ranges using multiple range queries // Check ranges before deletion area - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s1) FROM root.sg1.d1 WHERE time >= 1 AND time < 10", "COUNT(root.sg1.d1.s1),", Collections.singleton("9,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s2) FROM root.sg1.d1 WHERE time >= 10001 AND time < 10010", "COUNT(root.sg1.d1.s2),", Collections.singleton("9,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s3) FROM root.sg1.d1 WHERE time >= 20001 AND time < 20010", "COUNT(root.sg1.d1.s3),", Collections.singleton("9,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s4) FROM root.sg1.d1 WHERE time >= 30001 AND time < 30010", "COUNT(root.sg1.d1.s4),", Collections.singleton("9,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s5) FROM root.sg1.d1 WHERE time >= 40001 AND time < 40010", "COUNT(root.sg1.d1.s5),", Collections.singleton("9,")); // Check ranges after deletion area - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s1) FROM root.sg1.d1 WHERE time > 4000 AND time <= 20000", "COUNT(root.sg1.d1.s1),", Collections.singleton("16000,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s2) FROM root.sg1.d1 WHERE time > 14000 AND time <= 30000", "COUNT(root.sg1.d1.s2),", Collections.singleton("16000,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s3) FROM root.sg1.d1 WHERE time > 24000 AND time <= 40000", "COUNT(root.sg1.d1.s3),", Collections.singleton("16000,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s4) FROM root.sg1.d1 WHERE time > 34000 AND time <= 50000", "COUNT(root.sg1.d1.s4),", Collections.singleton("16000,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s5) FROM root.sg1.d1 WHERE time > 44000 AND time <= 60000", "COUNT(root.sg1.d1.s5),", Collections.singleton("16000,")); // Check non-deleted points within deletion range (every 10th point except the ones we deleted) - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s1) FROM root.sg1.d1 WHERE time >= 10 AND time <= 4000 AND time % 10 != 0", "COUNT(root.sg1.d1.s1),", Collections.singleton("3591,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s2) FROM root.sg1.d1 WHERE time >= 10010 AND time <= 14000 AND time % 10 != 0", "COUNT(root.sg1.d1.s2),", Collections.singleton("3591,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s3) FROM root.sg1.d1 WHERE time >= 20010 AND time <= 24000 AND time % 10 != 0", "COUNT(root.sg1.d1.s3),", Collections.singleton("3591,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s4) FROM root.sg1.d1 WHERE time >= 30010 AND time <= 34000 AND time % 10 != 0", "COUNT(root.sg1.d1.s4),", Collections.singleton("3591,")); - TestUtils.assertDataEventuallyOnEnv( + assertDataEventually( receiverEnv, "SELECT COUNT(s5) FROM root.sg1.d1 WHERE time >= 40010 AND time <= 44000 AND time % 10 != 0", "COUNT(root.sg1.d1.s5),", diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java index efb2eeee14a4d..d91c571e70976 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java @@ -20,13 +20,11 @@ package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.commons.utils.function.CheckedTriConsumer; -import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.db.pipe.receiver.transform.converter.ValueConverter; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; -import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.StatementExecutionException; @@ -52,7 +50,6 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.util.ArrayList; -import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Random; @@ -63,11 +60,18 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; +/** + * Exercises every ISession write form against all source/target type pairs. Each method uses its + * own database so this class can reuse one fixed-topology sender/receiver pair without allowing + * schemas or data regions from another method to affect the conversion being checked. + */ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualTreeModelManualIT { +public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeTreeModelManualSharedClusterIT { private static final int generateDataSize = 100; + // Insert 100 rows with insertTablet across all 100 type pairs; realtime delivery must match the + // receiver's precreated target types and converted values in this method's database. @Test public void insertTablet() { prepareTypeConversionTest( @@ -77,6 +81,8 @@ public void insertTablet() { false); } + // Insert 100 rows with insertTablet across all type pairs; TsFile delivery is expected to convert + // them to the receiver's target types once that currently disabled behavior is supported. @Test @Ignore("The receiver conversion is currently banned, will ignore conflict") public void insertTabletReceiveByTsFile() { @@ -87,6 +93,8 @@ public void insertTabletReceiveByTsFile() { true); } + // Insert 100 aligned rows with insertAlignedTablet across all type pairs; realtime delivery must + // match the receiver's target types and converted values in an isolated database. @Test public void insertAlignedTablet() { prepareTypeConversionTest( @@ -96,6 +104,8 @@ public void insertAlignedTablet() { false); } + // Insert 100 aligned rows with insertAlignedTablet across all type pairs; TsFile conversion must + // match the target schemas when this currently unsupported conversion is enabled. @Test @Ignore("The receiver conversion is currently banned, will ignore conflict") public void insertAlignedTabletReceiveByTsFile() { @@ -106,6 +116,8 @@ public void insertAlignedTabletReceiveByTsFile() { true); } + // Insert 100 rows in one insertRecords call across all type pairs; TsFile conversion must match + // the target schemas when this currently unsupported conversion is enabled. @Test @Ignore("The receiver conversion is currently banned, will ignore conflict") public void insertRecordsReceiveByTsFile() { @@ -121,6 +133,8 @@ public void insertRecordsReceiveByTsFile() { true); } + // Insert 100 individual rows with insertRecord across all type pairs; realtime delivery must + // preserve timestamps and convert every value to the receiver's target type. @Test public void insertRecord() { prepareTypeConversionTest( @@ -141,6 +155,8 @@ public void insertRecord() { false); } + // Insert 100 individual rows with insertRecord across all type pairs; TsFile conversion must + // match the target schemas when this currently unsupported conversion is enabled. @Test @Ignore("The receiver conversion is currently banned, will ignore conflict") public void insertRecordReceiveByTsFile() { @@ -162,6 +178,8 @@ public void insertRecordReceiveByTsFile() { true); } + // Insert 100 individual aligned rows with insertAlignedRecord across all type pairs; realtime + // delivery must preserve timestamps and convert every value to the receiver's target type. @Test public void insertAlignedRecord() { prepareTypeConversionTest( @@ -182,6 +200,8 @@ public void insertAlignedRecord() { false); } + // Insert 100 individual aligned rows with insertAlignedRecord across all type pairs; TsFile + // conversion must match the target schemas once this unsupported conversion is enabled. @Test @Ignore("The receiver conversion is currently banned, will ignore conflict") public void insertAlignedRecordReceiveByTsFile() { @@ -203,6 +223,8 @@ public void insertAlignedRecordReceiveByTsFile() { true); } + // Insert 100 rows in one insertRecords call across all type pairs; realtime delivery must match + // the receiver's target types and converted values in this method's database. @Test public void insertRecords() { prepareTypeConversionTest( @@ -217,6 +239,8 @@ public void insertRecords() { false); } + // Insert 100 aligned rows in one insertAlignedRecords call across all type pairs; realtime + // delivery must match the receiver's target types and converted values in an isolated database. @Test public void insertAlignedRecords() { prepareTypeConversionTest( @@ -231,6 +255,8 @@ public void insertAlignedRecords() { false); } + // Insert 100 aligned rows with insertAlignedRecords across all type pairs; TsFile conversion must + // match the target schemas when this currently unsupported conversion is enabled. @Test @Ignore("The receiver conversion is currently banned, will ignore conflict") public void insertAlignedRecordsReceiveByTsFile() { @@ -246,6 +272,8 @@ public void insertAlignedRecordsReceiveByTsFile() { true); } + // Insert 100 string-encoded rows with insertStringRecordsOfOneDevice across all type pairs; + // realtime delivery must parse the inputs and match the converted target values and timestamps. @Test public void insertStringRecordsOfOneDevice() { prepareTypeConversionTest( @@ -260,6 +288,8 @@ public void insertStringRecordsOfOneDevice() { false); } + // Insert 100 string-encoded rows with insertStringRecordsOfOneDevice across all type pairs; + // TsFile conversion must match target values once this unsupported conversion is enabled. @Test @Ignore("The receiver conversion is currently banned, will ignore conflict") public void insertStringRecordsOfOneDeviceReceiveByTsFile() { @@ -275,6 +305,8 @@ public void insertStringRecordsOfOneDeviceReceiveByTsFile() { true); } + // Insert 100 aligned string-encoded rows with insertAlignedStringRecordsOfOneDevice across all + // type pairs; realtime delivery must match the converted target values and timestamps. @Test public void insertAlignedStringRecordsOfOneDevice() { prepareTypeConversionTest( @@ -289,6 +321,8 @@ public void insertAlignedStringRecordsOfOneDevice() { false); } + // Insert 100 aligned string-encoded rows with insertAlignedStringRecordsOfOneDevice across all + // type pairs; TsFile conversion must match target values once this unsupported path is enabled. @Test @Ignore("The receiver conversion is currently banned, will ignore conflict") public void insertAlignedStringRecordsOfOneDeviceReceiveByTsFile() { @@ -323,30 +357,38 @@ private void prepareTypeConversionTest( List> measurementSchemas = generateMeasurementSchemas(); - // Generate createTimeSeries in sender and receiver - String uuid = "bcdedit" + Long.toHexString(System.nanoTime()); - for (Pair pair : measurementSchemas) { - createTimeSeries(uuid, pair.left.getMeasurementName(), pair.left.getType().name(), senderEnv); - createTimeSeries( - uuid, pair.right.getMeasurementName(), pair.right.getType().name(), receiverEnv); - } - try (ISession senderSession = senderEnv.getSessionConnection(); ISession receiverSession = receiverEnv.getSessionConnection()) { - Tablet tablet = generateTabletAndMeasurementSchema(measurementSchemas, "root.test." + uuid); + String database = "root.tc" + Long.toHexString(System.nanoTime()); + String deviceId = database + ".device"; + senderSession.executeNonQueryStatement("create database " + database); + receiverSession.executeNonQueryStatement("create database " + database); + // Reuse these sessions for all 200 schemas, writes, pipe creation, flush and receiver + // queries. + for (Pair pair : measurementSchemas) { + createTimeSeries( + senderSession, database, pair.left.getMeasurementName(), pair.left.getType().name()); + createTimeSeries( + receiverSession, + database, + pair.right.getMeasurementName(), + pair.right.getType().name()); + } + + Tablet tablet = generateTabletAndMeasurementSchema(measurementSchemas, deviceId); if (isTsFile) { // Send TsFile data to receiver consumer.accept(senderSession, receiverSession, tablet); Thread.sleep(2000); - createDataPipe(uuid, true); + createDataPipe(senderSession, database, true); senderSession.executeNonQueryStatement("flush"); } else { - // Send Tablet data to receiver - // Write once to create data regions, guarantee that no any tsFiles will be sent + // Create data regions before creating the pipe. History is disabled and realtime mode is + // forced-log, so only the second write exercises receiver-side tablet conversion. consumer.accept(senderSession, receiverSession, tablet); - createDataPipe(uuid, false); - Thread.sleep(2000); - // The actual implementation logic of inserting data + createDataPipe(senderSession, database, false); + // CREATE PIPE waits for its procedure to push the running pipe to DataNodes. Write + // immediately and let the bounded receiver assertion wait for delivery. consumer.accept(senderSession, receiverSession, tablet); senderSession.executeNonQueryStatement("flush"); } @@ -357,23 +399,19 @@ private void prepareTypeConversionTest( generateTabletResultSetForTable(tablet, measurementSchemas); await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(timeoutSeconds, TimeUnit.SECONDS) .untilAsserted( () -> { - try { - validateResultSet( - query(receiverSession, tablet.getSchemas(), tablet.getDeviceId()), - expectedValues, - tablet.getTimestamps()); + try (SessionDataSet dataSet = + query(receiverSession, tablet.getSchemas(), tablet.getDeviceId())) { + validateResultSet(dataSet, expectedValues, tablet.getTimestamps()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }); - senderSession.close(); - receiverSession.close(); tablet.reset(); } catch (Exception e) { e.printStackTrace(); @@ -381,29 +419,33 @@ private void prepareTypeConversionTest( } } - private void createTimeSeries(String diff, String measurementID, String dataType, BaseEnv env) { + private void createTimeSeries( + ISession session, String database, String measurementID, String dataType) + throws IoTDBConnectionException, StatementExecutionException { String timeSeriesCreation = String.format( - "create timeseries root.test.%s.%s with datatype=%s,encoding=PLAIN", - diff, measurementID, dataType); - TestUtils.executeNonQueries(env, Collections.singletonList(timeSeriesCreation), null); + "create timeseries %s.%s with datatype=%s,encoding=PLAIN", + database + ".device", measurementID, dataType); + session.executeNonQueryStatement(timeSeriesCreation); } - private void createDataPipe(String diff, boolean isTSFile) { + private void createDataPipe(ISession senderSession, String database, boolean isTSFile) + throws IoTDBConnectionException, StatementExecutionException { String sql = String.format( "create pipe test%s" - + " with source ('source'='iotdb-source','source.path'='root.test.**','realtime.mode'='%s','realtime.enable'='%s','history.enable'='%s')" + + " with source ('source'='iotdb-source','source.path'='%s.**','realtime.mode'='%s','realtime.enable'='%s','history.enable'='%s')" + " with processor ('processor'='do-nothing-processor')" + " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='%s')", - diff, + database.substring(database.lastIndexOf('.') + 1), + database, isTSFile ? "file" : "forced-log", !isTSFile, isTSFile, receiverEnv.getIP(), receiverEnv.getPort(), isTSFile ? "tsfile" : "tablet"); - TestUtils.executeNonQueries(senderEnv, Collections.singletonList(sql), null); + senderSession.executeNonQueryStatement(sql); } private void validateResultSet( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeWriteBackSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeWriteBackSinkIT.java index 9dbcd3a52972c..60aeea7a22210 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeWriteBackSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeWriteBackSinkIT.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; @@ -41,26 +42,28 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeWriteBackSinkIT extends AbstractPipeDualTreeModelManualIT { +public class IoTDBPipeWriteBackSinkIT extends AbstractPipeTreeModelManualSharedClusterIT { + // Verify write-back routes the source rows to a qualified target database. @Test public void testWriteBackSinkWithTargetDatabaseForTreeModel() throws Exception { - testWriteBackSinkWithTargetDatabaseForTreeModel("root.target.db"); + testWriteBackSinkWithTargetDatabaseForTreeModel("root.source_target", "root.target.db"); } + // Verify write-back preserves the mixed case of the configured target database name. @Test public void testWriteBackSinkPreservesTreeModelTargetDatabaseCase() throws Exception { - testWriteBackSinkWithTargetDatabaseForTreeModel("TargetDB"); + testWriteBackSinkWithTargetDatabaseForTreeModel("root.source_case", "TargetDB"); } - private void testWriteBackSinkWithTargetDatabaseForTreeModel(final String targetDatabase) - throws Exception { + private void testWriteBackSinkWithTargetDatabaseForTreeModel( + final String sourceDatabase, final String targetDatabase) throws Exception { final String qualifiedTargetDatabase = PathUtils.qualifyDatabaseName(targetDatabase); TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "create database root.source", - "create timeseries root.source.d1.s1 with datatype=INT32,encoding=PLAIN", + "create database " + sourceDatabase, + "create timeseries " + sourceDatabase + ".d1.s1 with datatype=INT32,encoding=PLAIN", "create database " + qualifiedTargetDatabase, "create timeseries " + qualifiedTargetDatabase @@ -75,12 +78,12 @@ private void testWriteBackSinkWithTargetDatabaseForTreeModel(final String target sourceAttributes.put("source.inclusion", "data.insert"); sourceAttributes.put("source.forwarding-pipe-requests", "false"); - sourceAttributes.put("source.path", "root.source.**"); - sourceAttributes.put("user", "root"); + sourceAttributes.put("source.path", sourceDatabase + ".**"); + sourceAttributes.put("user", SessionConfig.DEFAULT_USER); sinkAttributes.put("sink", "write-back-sink"); sinkAttributes.put("sink.database", targetDatabase); - sinkAttributes.put("user", "root"); + sinkAttributes.put("user", SessionConfig.DEFAULT_USER); final TSStatus status = client.createPipe( @@ -96,8 +99,8 @@ private void testWriteBackSinkWithTargetDatabaseForTreeModel(final String target TestUtils.executeNonQueries( senderEnv, Arrays.asList( - "insert into root.source.d1(time, s1) values (1, 1)", - "insert into root.source.d1(time, s1) values (2, 2)", + "insert into " + sourceDatabase + ".d1(time, s1) values (1, 1)", + "insert into " + sourceDatabase + ".d1(time, s1) values (2, 2)", "flush"), null); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java index dd36ff6064b14..5ee69858bd3f1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java @@ -45,6 +45,11 @@ public void setUp() throws Exception { @After public final void tearDown() { + cleanEnvironment(); + } + + // A shared-cluster subclass can reset its scenario without stopping the class-owned nodes. + protected void cleanEnvironment() { env.cleanClusterEnvironment(); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOperationDiagnosticsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOperationDiagnosticsIT.java index e83fddbfb0184..050f9bc80cb4e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOperationDiagnosticsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOperationDiagnosticsIT.java @@ -38,11 +38,11 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.awaitility.Awaitility; -import org.junit.After; import org.junit.Assert; -import org.junit.Before; +import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.ExternalResource; import org.junit.runner.RunWith; import java.io.IOException; @@ -64,24 +64,28 @@ public class IoTDBPipeOperationDiagnosticsIT { private static final String PIPE_NAME = "pipe_operation_diagnostics"; private static final String RECOVER_PIPE_NAME = "pipe_drop_recover"; - @Before - public void setUp() { - EnvFactory.getEnv() - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setDnConnectionTimeoutMs(2000) - .setPipeMetaSyncerSyncIntervalMinutes(1) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false) - .setPipeAutoSplitFullEnabled(false); - EnvFactory.getEnv().initClusterEnvironment(1, 2); - } + @ClassRule + public static final ExternalResource CLUSTER = + new ExternalResource() { + @Override + protected void before() { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setDnConnectionTimeoutMs(2000) + .setPipeMetaSyncerSyncIntervalMinutes(1) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false) + .setPipeAutoSplitFullEnabled(false); + EnvFactory.getEnv().initClusterEnvironment(1, 2); + } - @After - public void tearDown() { - EnvFactory.getEnv().cleanClusterEnvironment(); - } + @Override + protected void after() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + }; @Test public void testDropPipeExposesPreDeleteAndRejectsConcurrentOperations() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java index 2ab00c52f9fa5..d0d86396bd730 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java @@ -19,25 +19,37 @@ package org.apache.iotdb.pipe.it.single; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT1; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; import org.junit.Before; +import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.ExternalResource; import org.junit.runner.RunWith; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -48,21 +60,132 @@ @Category({MultiClusterIT1.class}) public class IoTDBPipePermissionIT extends AbstractPipeSingleIT { + private static BaseEnv sharedEnv; + + @ClassRule + public static final ExternalResource CLUSTER = + new ExternalResource() { + @Override + protected void after() { + try { + if (sharedEnv != null) { + sharedEnv.cleanClusterEnvironment(); + } + } finally { + sharedEnv = null; + } + } + }; + @Before - public void setUp() { - MultiEnvFactory.createEnv(1); - env = MultiEnvFactory.getEnv(0); - env.getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setDatanodeMemoryProportion("3:3:1:1:1:0") - .setPipeMemoryManagementEnabled(false) - .setDataReplicationFactor(1) - .setSchemaReplicationFactor(1) - .setIsPipeEnableMemoryCheck(false) - .setPipeAutoSplitFullEnabled(false); - // 1C1D to directly show the remaining count - env.initClusterEnvironment(1, 1); + public void setUp() throws Exception { + if (sharedEnv == null) { + try { + MultiEnvFactory.createEnv(1); + env = MultiEnvFactory.getEnv(0); + env.getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setDatanodeMemoryProportion("3:3:1:1:1:0") + .setPipeMemoryManagementEnabled(false) + .setDataReplicationFactor(1) + .setSchemaReplicationFactor(1) + .setIsPipeEnableMemoryCheck(false) + .setPipeAutoSplitFullEnabled(false); + // 1C1D to directly show the remaining count + env.initClusterEnvironment(1, 1); + sharedEnv = env; + } catch (Exception | Error e) { + if (env != null) { + env.cleanClusterEnvironment(); + } + throw e; + } + } + env = sharedEnv; + } + + @Override + protected void cleanEnvironment() { + if (sharedEnv == null) { + return; + } + try { + dropPipes(sharedEnv); + resetCatalog(sharedEnv); + } catch (Exception e) { + throw new AssertionError("Failed to reset shared permission test cluster", e); + } + } + + private static void dropPipes(final BaseEnv env) throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { + for (final boolean tableModel : new boolean[] {false, true}) { + final TShowPipeResp response = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(tableModel) + .setUserName(SessionConfig.DEFAULT_USER)); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + for (final TShowPipeInfo pipe : response.getPipeInfoList()) { + if (!pipe.getId().startsWith(PipeStaticMeta.SYSTEM_PIPE_PREFIX)) { + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(pipe.getId()).setIsTableModel(tableModel)) + .getCode()); + } + } + } + } + } + + private static void resetCatalog(final BaseEnv env) throws SQLException { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final List databases = new ArrayList<>(); + try (final ResultSet result = statement.executeQuery("show databases")) { + while (result.next()) { + final String database = result.getString(1); + if (!"information_schema".equals(database)) { + databases.add(database); + } + } + } + for (final String database : databases) { + TestUtils.dropDatabaseWithRetry(statement, "\"" + database.replace("\"", "\"\"") + "\""); + } + } + + try (final Connection connection = env.getConnection(); + final Statement statement = connection.createStatement()) { + final List databases = new ArrayList<>(); + try (final ResultSet result = statement.executeQuery("show databases")) { + while (result.next()) { + databases.add(result.getString(1)); + } + } + for (final String database : databases) { + statement.execute("delete database " + database); + } + + for (final String type : new String[] {"user", "role"}) { + final List names = new ArrayList<>(); + try (final ResultSet result = statement.executeQuery("list " + type)) { + while (result.next()) { + final String name = result.getString("user".equals(type) ? "User" : "Role"); + if (!"user".equals(type) || !SessionConfig.DEFAULT_USER.equals(name)) { + names.add(name); + } + } + } + for (final String name : names) { + statement.execute("drop " + type + " `" + name.replace("`", "``") + "`"); + } + } + } } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java index 34dd7263d4ced..598b0a7a3c3e6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java @@ -23,10 +23,14 @@ import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient; import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp; -import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV1Req; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.cluster.EnvUtils; @@ -39,8 +43,11 @@ import org.awaitility.Awaitility; import org.junit.Assert; +import org.junit.Before; +import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.ExternalResource; import org.junit.runner.RunWith; import java.sql.Connection; @@ -49,12 +56,117 @@ import java.sql.Statement; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT1.class}) public class IoTDBShowReceiversIT extends AbstractPipeSingleIT { + private static final int SETUP_DATA_NODE_INDEX = 0; + private static BaseEnv sharedEnv; + + @ClassRule + public static final ExternalResource CLUSTER = + new ExternalResource() { + @Override + protected void after() { + try { + if (sharedEnv != null) { + sharedEnv.cleanClusterEnvironment(); + } + } finally { + sharedEnv = null; + } + } + }; + + @Override + @Before + public void setUp() throws Exception { + if (sharedEnv == null) { + try { + super.setUp(); + sharedEnv = env; + } catch (Exception | Error e) { + // The class rule owns successful startup; release a partially started cluster here. + if (env != null) { + env.cleanClusterEnvironment(); + } + throw e; + } + } + env = sharedEnv; + } + + @Override + protected void cleanEnvironment() { + if (sharedEnv == null) { + return; + } + try { + // A stop/restart assertion can fail with a node still down. Restore it before querying or + // dropping pipes so the following method always sees the same healthy cluster. + for (int i = 0; i < env.getDataNodeWrapperList().size(); i++) { + if (!env.getDataNodeWrapper(i).isAlive()) { + env.startDataNode(i); + env.ensureNodeStatus( + Collections.singletonList(env.getDataNodeWrapper(i)), + Collections.singletonList(NodeStatus.Running)); + } + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { + for (final boolean tableModel : new boolean[] {false, true}) { + final TShowPipeResp response = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(tableModel) + .setUserName(SessionConfig.DEFAULT_USER)); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + for (final TShowPipeInfo pipe : response.getPipeInfoList()) { + if (!pipe.getId().startsWith(PipeStaticMeta.SYSTEM_PIPE_PREFIX)) { + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(pipe.getId()).setIsTableModel(tableModel)) + .getCode()); + } + } + } + } + + // Each method owns distinct database, pipe and user names. Keep its catalog until class + // teardown, but await asynchronous receiver removal so permission/aggregation assertions + // cannot observe a previous method's writeback or thrift sessions. + Awaitility.await() + .pollInSameThread() + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) + .atMost(60L, TimeUnit.SECONDS) + .untilAsserted( + () -> { + Assert.assertFalse( + hasAnyReceiver( + "show receivers", + BaseEnv.TREE_SQL_DIALECT, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD)); + Assert.assertFalse( + hasAnyReceiver( + "select * from information_schema.receivers", + BaseEnv.TABLE_SQL_DIALECT, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD)); + }); + } catch (Exception e) { + throw new AssertionError("Failed to reset shared receiver test cluster", e); + } + } + + // One writeback pipe must expose its runtime in both SHOW RECEIVERS and information_schema. @Test public void testShowReceiversInTreeAndTableModel() { createWriteBackPipe("root.show_receivers", "show_receivers_pipe"); @@ -63,6 +175,7 @@ public void testShowReceiversInTreeAndTableModel() { assertShowReceivers("select * from information_schema.receivers", BaseEnv.TABLE_SQL_DIALECT); } + // Filtering receiver metadata by protocol must retain this method's writeback pipe. @Test public void testInformationSchemaReceiversWithProtocolFilter() { createWriteBackPipe("root.show_receivers_filter", "show_receivers_filter_pipe"); @@ -76,6 +189,7 @@ public void testInformationSchemaReceiversWithProtocolFilter() { "show_receivers_filter_pipe"); } + // Projecting selected information_schema columns must retain the expected pipe and sender. @Test public void testInformationSchemaReceiversProjectedColumns() { final String pipeName = "show_receivers_project_pipe"; @@ -84,6 +198,7 @@ public void testInformationSchemaReceiversProjectedColumns() { assertProjectedShowReceivers(pipeName); } + // Two pipes from one sender must appear together with at least two pipes in the aggregated row. @Test public void testShowReceiversAggregatesMultiplePipesFromSameSender() { final String pipeName1 = "show_receivers_aggregate_pipe_1"; @@ -98,6 +213,7 @@ public void testShowReceiversAggregatesMultiplePipesFromSameSender() { pipeName2); } + // After stopping one of three DataNodes, both SQL dialects must omit that node's runtime. @Test public void testShowReceiversWithStoppedDataNode() throws Exception { ensureDataNodeCount(3); @@ -109,7 +225,7 @@ public void testShowReceiversWithStoppedDataNode() throws Exception { BaseEnv.TABLE_SQL_DIALECT, "show_receivers_ha_pipe"); - final int stoppedDataNodeIndex = 0; + final int stoppedDataNodeIndex = SETUP_DATA_NODE_INDEX; final DataNodeWrapper stoppedDataNode = env.getDataNodeWrapper(stoppedDataNodeIndex); final int stoppedDataNodeId = getDataNodeId(stoppedDataNode); final DataNodeWrapper queryDataNode = env.getDataNodeWrapper(1); @@ -128,6 +244,7 @@ public void testShowReceiversWithStoppedDataNode() throws Exception { stoppedDataNodeId); } + // A normal user must see no receivers, while a SYSTEM user can see the administrator's pipe. @Test public void testShowReceiversPermissionInTreeAndTableModel() { final String password = "Passwd123456@"; @@ -135,13 +252,11 @@ public void testShowReceiversPermissionInTreeAndTableModel() { final String systemUser = "show_receiver_system"; final String pipeName = "show_receivers_auth_pipe"; - TestUtils.executeNonQueries( - env, + executeSetupSql( Arrays.asList( "create user " + normalUser + " '" + password + "'", "create user " + systemUser + " '" + password + "'", - "grant system on root.** to user " + systemUser), - null); + "grant system on root.** to user " + systemUser)); createWriteBackPipe("root.show_receivers_auth", pipeName); assertShowReceivers("show receivers", BaseEnv.TREE_SQL_DIALECT, pipeName); @@ -165,6 +280,7 @@ public void testShowReceiversPermissionInTreeAndTableModel() { password); } + // A sink user must see its own writeback receiver in both dialects; an unrelated user must not. @Test public void testNormalUserCanSeeOwnReceiverInTreeAndTableModel() { final String password = "Passwd123456@"; @@ -172,13 +288,11 @@ public void testNormalUserCanSeeOwnReceiverInTreeAndTableModel() { final String otherUser = "show_receiver_other_user"; final String pipeName = "show_receivers_own_user_pipe"; - TestUtils.executeNonQueries( - env, + executeSetupSql( Arrays.asList( "create user " + sinkUser + " '" + password + "'", "create user " + otherUser + " '" + password + "'", - "grant write on root.** to user " + sinkUser), - null); + "grant write on root.** to user " + sinkUser)); createWriteBackPipeWithSinkUser("root.show_receivers_own_user", pipeName, sinkUser, password); assertShowReceiversAsUser( @@ -199,6 +313,7 @@ public void testNormalUserCanSeeOwnReceiverInTreeAndTableModel() { password); } + // A legacy handshake exposes Unknown pipe IDs, and closing its client removes that runtime. @Test public void testOldSenderRuntimeShowsUnknownPipeIdsAndDisappearsAfterClientClose() throws Exception { @@ -227,10 +342,11 @@ public void testOldSenderRuntimeShowsUnknownPipeIdsAndDisappearsAfterClientClose receiverDataNodeId); } + // Restarting a DataNode must remove its legacy runtime and permit a fresh handshake afterward. @Test public void testReceiverRuntimeClearedAfterDataNodeRestartAndCanReconnect() throws Exception { ensureDataNodeCount(2); - final int restartedDataNodeIndex = 0; + final int restartedDataNodeIndex = SETUP_DATA_NODE_INDEX; final DataNodeWrapper restartedDataNode = env.getDataNodeWrapper(restartedDataNodeIndex); final int restartedDataNodeId = getDataNodeId(restartedDataNode); @@ -269,8 +385,9 @@ public void testReceiverRuntimeClearedAfterDataNodeRestartAndCanReconnect() thro } private void createWriteBackPipe(final String database, final String pipeName) { - TestUtils.executeNonQueries( - env, + // The default pattern matches string prefixes. The dot excludes sibling method databases + // such as root.show_receivers_filter while preserving every series inside this database. + executeSetupSql( Arrays.asList( "create database " + database, "create timeseries " + database + ".d1.s1 with datatype=INT32, encoding=PLAIN", @@ -278,10 +395,9 @@ private void createWriteBackPipe(final String database, final String pipeName) { + pipeName + " with source ('pattern'='" + database - + "') with sink ('sink'='write-back-sink')", + + ".') with sink ('sink'='write-back-sink')", "insert into " + database + ".d1(time, s1) values (1, 1)", - "flush"), - null); + "flush")); } private void createWriteBackPipeWithSinkUser( @@ -289,8 +405,7 @@ private void createWriteBackPipeWithSinkUser( final String pipeName, final String sinkUser, final String sinkPassword) { - TestUtils.executeNonQueries( - env, + executeSetupSql( Arrays.asList( "create database " + database, "create timeseries " + database + ".d1.s1 with datatype=INT32, encoding=PLAIN", @@ -298,20 +413,18 @@ private void createWriteBackPipeWithSinkUser( + pipeName + " with source ('pattern'='" + database - + "') with sink ('sink'='write-back-sink', 'user'='" + + ".') with sink ('sink'='write-back-sink', 'user'='" + sinkUser + "', 'password'='" + sinkPassword + "')", "insert into " + database + ".d1(time, s1) values (1, 1)", - "flush"), - null); + "flush")); } private void createTwoWriteBackPipes( final String database, final String firstPipeName, final String secondPipeName) { - TestUtils.executeNonQueries( - env, + executeSetupSql( Arrays.asList( "create database " + database, "create timeseries " + database + ".d1.s1 with datatype=INT32, encoding=PLAIN", @@ -319,15 +432,31 @@ private void createTwoWriteBackPipes( + firstPipeName + " with source ('pattern'='" + database - + "') with sink ('sink'='write-back-sink')", + + ".') with sink ('sink'='write-back-sink')", "create pipe " + secondPipeName + " with source ('pattern'='" + database - + "') with sink ('sink'='write-back-sink')", + + ".') with sink ('sink'='write-back-sink')", "insert into " + database + ".d1(time, s1) values (1, 1)", - "flush"), - null); + "flush")); + } + + private void executeSetupSql(final List sqlList) { + // The failure scenarios restart this same target DataNode. Its scheduler clients are recreated + // with the process, so later setup cannot borrow another node's stale connection to it. + // Receiver queries still use all nodes; only fixture writes are pinned and share a connection. + String currentSql = null; + try (final Connection connection = + env.getConnection(env.getDataNodeWrapper(SETUP_DATA_NODE_INDEX)); + final Statement statement = connection.createStatement()) { + for (final String sql : sqlList) { + currentSql = sql; + statement.execute(sql); + } + } catch (SQLException e) { + throw new AssertionError("Failed to prepare receiver test with SQL: " + currentSql, e); + } } private void assertShowReceivers(final String sql, final String sqlDialect) { @@ -338,8 +467,8 @@ private void assertShowReceivers( final String sql, final String sqlDialect, final String pipeName) { Awaitility.await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(60L, TimeUnit.SECONDS) .untilAsserted(() -> Assert.assertTrue(hasExpectedReceiver(sql, sqlDialect, pipeName))); } @@ -362,8 +491,8 @@ private void assertShowReceiversAsUser( final String expectedReceiverUserName) { Awaitility.await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(60L, TimeUnit.SECONDS) .untilAsserted( () -> @@ -379,8 +508,8 @@ private void assertAggregatedShowReceivers( final String secondPipeName) { Awaitility.await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(60L, TimeUnit.SECONDS) .untilAsserted( () -> @@ -391,8 +520,8 @@ private void assertAggregatedShowReceivers( private void assertProjectedShowReceivers(final String pipeName) { Awaitility.await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(60L, TimeUnit.SECONDS) .untilAsserted(() -> Assert.assertTrue(hasProjectedReceiver(pipeName))); } @@ -401,8 +530,8 @@ private void assertUserCannotSeeReceivers( final String sql, final String sqlDialect, final String userName, final String password) { Awaitility.await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(60L, TimeUnit.SECONDS) .untilAsserted( () -> Assert.assertFalse(hasAnyReceiver(sql, sqlDialect, userName, password))); @@ -431,8 +560,8 @@ private void assertOldSenderReceiverRuntime( final String sql, final String sqlDialect, final int receiverDataNodeId) { Awaitility.await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(60L, TimeUnit.SECONDS) .untilAsserted( () -> @@ -473,8 +602,8 @@ private void assertNoThriftReceiverRuntimeOnDataNode( final String sql, final String sqlDialect, final int receiverDataNodeId) { Awaitility.await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(60L, TimeUnit.SECONDS) .untilAsserted( () -> @@ -687,8 +816,8 @@ private void assertShowReceiversWithoutDataNode( final int excludedDataNodeId) { Awaitility.await() .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) .atMost(60L, TimeUnit.SECONDS) .untilAsserted( () -> diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/AbstractPipeTripleManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/AbstractPipeTripleManualIT.java index 0640f3fb645ec..6e644f7153d1d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/AbstractPipeTripleManualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/AbstractPipeTripleManualIT.java @@ -26,6 +26,8 @@ import org.junit.After; import org.junit.Before; +import java.util.concurrent.CompletableFuture; + abstract class AbstractPipeTripleManualIT { protected BaseEnv env1; @@ -39,9 +41,14 @@ public void setUp() { env2 = MultiEnvFactory.getEnv(1); env3 = MultiEnvFactory.getEnv(2); setupConfig(); - env1.initClusterEnvironment(1, getEnv1DataNodeCount()); - env2.initClusterEnvironment(1, 1); - env3.initClusterEnvironment(1, 1); + // The environments have independent ports/directories. Await every start, including failures, + // so teardown cannot race with a cluster that is still being initialized. + CompletableFuture.allOf( + CompletableFuture.runAsync( + () -> env1.initClusterEnvironment(1, getEnv1DataNodeCount())), + CompletableFuture.runAsync(() -> env2.initClusterEnvironment(1, 1)), + CompletableFuture.runAsync(() -> env3.initClusterEnvironment(1, 1))) + .join(); } protected int getEnv1DataNodeCount() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBPipeForwardIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBPipeForwardIT.java index 6beec673feb92..477995ed4aa91 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBPipeForwardIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBPipeForwardIT.java @@ -125,6 +125,8 @@ public void testLocalWriteIsForwardedToDownstreamWithDefaultSourceParameters() t } } + // Mark the first hop as a local request so the middle cluster forwards schema and one row + // to the third cluster even with forwarding-pipe-requests disabled on the second hop. @Test public void testForwardingPipeRequests() throws Exception { final DataNodeWrapper env2DataNode = env2.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java index 7fec946aac751..464f75af7937b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java @@ -53,6 +53,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; @@ -76,9 +77,13 @@ public void setUp() { setupConfig(senderEnv2); setupConfig(receiverEnv); - senderEnv1.initClusterEnvironment(1, 1); - senderEnv2.initClusterEnvironment(1, 1); - receiverEnv.initClusterEnvironment(1, 3); + // Independent clusters can start together. Await all three starts even if one fails so no + // startup task can outlive setup and race with teardown. + CompletableFuture.allOf( + CompletableFuture.runAsync(() -> senderEnv1.initClusterEnvironment(1, 1)), + CompletableFuture.runAsync(() -> senderEnv2.initClusterEnvironment(1, 1)), + CompletableFuture.runAsync(() -> receiverEnv.initClusterEnvironment(1, 3))) + .join(); } private static void setupConfig(final BaseEnv env) { @@ -101,6 +106,8 @@ public void tearDown() { receiverEnv.cleanClusterEnvironment(); } + // Route two senders to different DataNodes in one 1C3D receiver cluster; both SHOW RECEIVERS + // and the table information schema must aggregate the two active pipes and their node IDs. @Test public void testShowReceiversAggregatesMultipleSendersOnMultipleDataNodes() throws Exception { Assert.assertTrue(receiverEnv.getDataNodeWrapperList().size() >= 2); @@ -191,37 +198,40 @@ private void assertShowReceiversContainPipesOnDifferentDataNodes( final String firstPipeName, final int firstReceiverDataNodeId, final String secondPipeName, - final int secondReceiverDataNodeId) { - Awaitility.await() - .pollInSameThread() - .pollDelay(1L, TimeUnit.SECONDS) - .pollInterval(1L, TimeUnit.SECONDS) - .atMost(60L, TimeUnit.SECONDS) - .untilAsserted( - () -> - Assert.assertTrue( - hasReceiversOnDifferentDataNodes( - sql, - sqlDialect, - firstPipeName, - firstReceiverDataNodeId, - secondPipeName, - secondReceiverDataNodeId))); + final int secondReceiverDataNodeId) + throws SQLException { + // Reuse the connection across polls; the running clusters do not change during this check. + try (final Connection connection = + receiverEnv.getConnection( + SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect); + final Statement statement = connection.createStatement()) { + Awaitility.await() + .pollInSameThread() + .pollDelay(0L, TimeUnit.MILLISECONDS) + .pollInterval(100L, TimeUnit.MILLISECONDS) + .atMost(60L, TimeUnit.SECONDS) + .untilAsserted( + () -> + Assert.assertTrue( + hasReceiversOnDifferentDataNodes( + statement, + sql, + firstPipeName, + firstReceiverDataNodeId, + secondPipeName, + secondReceiverDataNodeId))); + } } private boolean hasReceiversOnDifferentDataNodes( + final Statement statement, final String sql, - final String sqlDialect, final String firstPipeName, final int firstReceiverDataNodeId, final String secondPipeName, final int secondReceiverDataNodeId) throws SQLException { - try (final Connection connection = - receiverEnv.getConnection( - SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect); - final Statement statement = connection.createStatement(); - final ResultSet clusterResult = statement.executeQuery(sql)) { + try (final ResultSet clusterResult = statement.executeQuery(sql)) { for (ResultSet resultSet : getNodeResultSets(clusterResult)) { boolean hasFirstPipe = false; boolean hasSecondPipe = false; diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 2af7b3720749a..31fdba9a5f44b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -2826,8 +2826,11 @@ private Void randomDeviceDeletion( int finalI = i; Awaitility.await() .atMost(5, TimeUnit.MINUTES) - .pollDelay(2, TimeUnit.SECONDS) - .pollInterval(2, TimeUnit.SECONDS) + // Most counts already match after the synchronous delete. Avoid paying two seconds + // for every deleted range while retaining retries for asynchronous visibility. + .pollInSameThread() + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .until( () -> { ResultSet set = @@ -3023,6 +3026,8 @@ public void deleteTableOfTheSameNameTest() } } + // Concurrently write 100 files with 100 timestamps on each of four devices, randomly delete + // device-specific ranges, and check every range against its expected remaining point count. @Test public void testConcurrentFlushAndRandomDeviceDeletion() throws InterruptedException, ExecutionException, SQLException { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java index 9cb250c599ee1..f36117f70c8c8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java @@ -43,11 +43,15 @@ import org.apache.tsfile.write.schema.Schema; import org.apache.tsfile.write.writer.TsFileIOWriter; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,10 +65,15 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +/** + * Load table TsFiles into a shared cluster with separate databases and temporary files per method. + * Class cleanup removes the remaining databases and the user created by the permission scenario. + */ @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) public class IoTDBLoadTsFileIT { @@ -73,11 +82,20 @@ public class IoTDBLoadTsFileIT { private static final int connectionTimeoutInMS = (int) TimeUnit.SECONDS.toMillis(300); private static final long loadTsFileAnalyzeSchemaMemorySizeInBytes = 10 * 1024L; + @Rule public final TestName testName = new TestName(); + private File tmpDir; + private String database; @Before public void setUp() throws Exception { tmpDir = new File(Files.createTempDirectory("load").toUri()); + // Isolate table schemas and loaded rows without restarting the identically configured nodes. + database = "load_" + testName.getMethodName().toLowerCase(Locale.ROOT); + } + + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setTimePartitionInterval(PARTITION_INTERVAL); EnvFactory.getEnv().getConfig().getCommonConfig().setEnforceStrongPassword(false); EnvFactory.getEnv().getConfig().getCommonConfig().setDatanodeMemoryProportion("3:3:1:1:1:0"); @@ -91,13 +109,16 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); - if (!deleteDir()) { LOGGER.error("Can not delete tmp dir for loading tsfile."); } } + @AfterClass + public static void tearDownClass() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + private boolean deleteDir() { for (final File file : tmpDir.listFiles()) { if (!file.delete()) { @@ -165,6 +186,10 @@ private List> generateMeasurementSche return pairs; } + /** + * Load 10,000 rows into an empty database as a user with CREATE/INSERT privileges; expect + * automatic table creation and the complete input row count. + */ @Test public void testLoadWithEmptyDatabaseForTableModel() throws Exception { final int lineCount = 10000; @@ -188,26 +213,25 @@ public void testLoadWithEmptyDatabaseForTableModel() throws Exception { // Prepare normal user try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement adminStmt = adminCon.createStatement()) { - adminStmt.execute("create user test 'password123456'"); + adminStmt.execute("create user test 'IoTDB!Test2024'"); adminStmt.execute( String.format( - "grant create, insert on %s.%s to user test", - SchemaConfig.DATABASE_0, SchemaConfig.TABLE_0)); + "grant create, insert on %s.%s to user test", database, SchemaConfig.TABLE_0)); // auto-create table - adminStmt.execute(String.format("create database if not exists %s", SchemaConfig.DATABASE_0)); + adminStmt.execute(String.format("create database if not exists %s", database)); } try (final Connection connection = - EnvFactory.getEnv().getConnection("test", "password123456", BaseEnv.TABLE_SQL_DIALECT); + EnvFactory.getEnv().getConnection("test", "IoTDB!Test2024", BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { - statement.execute(String.format("use %s", SchemaConfig.DATABASE_0)); + statement.execute(String.format("use %s", database)); statement.execute(String.format("load '%s'", file.getAbsolutePath())); } try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement adminStmt = adminCon.createStatement()) { - adminStmt.execute(String.format("use %s", SchemaConfig.DATABASE_0)); + adminStmt.execute(String.format("use %s", database)); try (final ResultSet resultSet = adminStmt.executeQuery(String.format("select count(*) from %s", SchemaConfig.TABLE_0))) { if (resultSet.next()) { @@ -219,6 +243,10 @@ public void testLoadWithEmptyDatabaseForTableModel() throws Exception { } } + /** + * Load 10,000 rows with source and target field types covering all type pairs; expect the + * complete row count after conversion. Keep this scenario ignored while conversion is banned. + */ @Test @Ignore("Load with conversion is currently banned") public void testLoadWithConvertOnTypeMismatchForTableModel() throws Exception { @@ -244,12 +272,11 @@ public void testLoadWithConvertOnTypeMismatchForTableModel() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { - statement.execute(String.format("create database if not exists %s", SchemaConfig.DATABASE_0)); - statement.execute(String.format("use %s", SchemaConfig.DATABASE_0)); + statement.execute(String.format("create database if not exists %s", database)); + statement.execute(String.format("use %s", database)); statement.execute(convert2TableSQL(SchemaConfig.TABLE_0, schemaList1, columnCategories)); statement.execute( - String.format( - "load '%s' with ('database'='%s')", file.getAbsolutePath(), SchemaConfig.DATABASE_0)); + String.format("load '%s' with ('database'='%s')", file.getAbsolutePath(), database)); try (final ResultSet resultSet = statement.executeQuery(String.format("select count(*) from %s", SchemaConfig.TABLE_0))) { if (resultSet.next()) { @@ -261,6 +288,10 @@ public void testLoadWithConvertOnTypeMismatchForTableModel() throws Exception { } } + /** + * Generate two tables with 10,000 rows each and delete all rows from the second through TsFile + * mods; after loading, expect 10,000 rows in the first table and only that table to be visible. + */ @Test public void testLoadWithTableMod() throws Exception { final int lineCount = 10000; @@ -286,11 +317,10 @@ public void testLoadWithTableMod() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { - statement.execute(String.format("create database if not exists %s", SchemaConfig.DATABASE_0)); - statement.execute(String.format("use %s", SchemaConfig.DATABASE_0)); + statement.execute(String.format("create database if not exists %s", database)); + statement.execute(String.format("use %s", database)); statement.execute( - String.format( - "load '%s' with ('database'='%s')", file.getAbsolutePath(), SchemaConfig.DATABASE_0)); + String.format("load '%s' with ('database'='%s')", file.getAbsolutePath(), database)); try (final ResultSet resultSet = statement.executeQuery(String.format("select count(*) from %s", SchemaConfig.TABLE_0))) { if (resultSet.next()) { @@ -307,6 +337,10 @@ public void testLoadWithTableMod() throws Exception { } } + /** + * Load 10,000 rows across 13 source/target combinations of time column positions, names, and + * absent target schemas; each load must retain all rows and preserve existing target columns. + */ @Test public void testLoadWithTimeColumn() throws Exception { final int lineCount = 10000; @@ -417,16 +451,15 @@ private void testWithTimeColumn( try (final Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { - statement.execute(String.format("create database if not exists %s", SchemaConfig.DATABASE_0)); - statement.execute(String.format("use %s", SchemaConfig.DATABASE_0)); + statement.execute(String.format("create database if not exists %s", database)); + statement.execute(String.format("use %s", database)); ResultSet resultSetOld = null; if (Objects.nonNull(schemaList1)) { statement.execute(convert2TableSQL(SchemaConfig.TABLE_0, schemaList1, columnCategories)); resultSetOld = statement.executeQuery("desc " + SchemaConfig.TABLE_0); } statement.execute( - String.format( - "load '%s' with ('database'='%s')", file.getAbsolutePath(), SchemaConfig.DATABASE_0)); + String.format("load '%s' with ('database'='%s')", file.getAbsolutePath(), database)); try (final ResultSet resultSet = statement.executeQuery(String.format("select count(*) from %s", SchemaConfig.TABLE_0))) { if (resultSet.next()) { @@ -455,7 +488,7 @@ private void testWithTimeColumn( } } - statement.execute(String.format("drop database %s", SchemaConfig.DATABASE_0)); + statement.execute(String.format("drop database %s", database)); } } @@ -499,6 +532,10 @@ private String convert2TableSQL( return tableCreation; } + /** + * Load 20 rows at timestamps 1,000 through 20,000 with both fields null; expect all timestamps to + * survive, zero non-null field counts, and wasNull to be true for every field value. + */ @Test public void testLoadWithAllFieldsNullRows() throws Exception { final List schemas = @@ -530,11 +567,10 @@ public void testLoadWithAllFieldsNullRows() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { - statement.execute(String.format("create database if not exists %s", SchemaConfig.DATABASE_0)); - statement.execute(String.format("use %s", SchemaConfig.DATABASE_0)); + statement.execute(String.format("create database if not exists %s", database)); + statement.execute(String.format("use %s", database)); statement.execute( - String.format( - "load '%s' with ('database'='%s')", file.getAbsolutePath(), SchemaConfig.DATABASE_0)); + String.format("load '%s' with ('database'='%s')", file.getAbsolutePath(), database)); try (final ResultSet resultSet = statement.executeQuery(String.format("select count(*) from %s", SchemaConfig.TABLE_0))) { @@ -599,11 +635,10 @@ public void testLoadSparseBatchedAlignedTsFile() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { - statement.execute(String.format("create database if not exists %s", SchemaConfig.DATABASE_0)); - statement.execute(String.format("use %s", SchemaConfig.DATABASE_0)); + statement.execute(String.format("create database if not exists %s", database)); + statement.execute(String.format("use %s", database)); statement.execute( - String.format( - "load '%s' with ('database'='%s')", file.getAbsolutePath(), SchemaConfig.DATABASE_0)); + String.format("load '%s' with ('database'='%s')", file.getAbsolutePath(), database)); try (final ResultSet resultSet = statement.executeQuery( @@ -683,7 +718,6 @@ private static boolean isNullField2(final long time) { } private static class SchemaConfig { - private static final String DATABASE_0 = "root"; private static final String TABLE_0 = "test"; private static final String TABLE_1 = "test1"; } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRegionReadyAfterRestartTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRegionReadyAfterRestartTableIT.java index 72bb5d472d641..0b6b2b44a4358 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRegionReadyAfterRestartTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRegionReadyAfterRestartTableIT.java @@ -47,6 +47,8 @@ public class IoTDBRegionReadyAfterRestartTableIT { @BeforeClass public static void setUpClass() { + // Keep the pre-restart row durable independently of Windows process-stop timing. + EnvFactory.getEnv().getConfig().getCommonConfig().setWalMode("SYNC"); EnvFactory.getEnv().initClusterEnvironment(1, 3); } @@ -55,6 +57,7 @@ public static void tearDownClass() { EnvFactory.getEnv().cleanClusterEnvironment(); } + // Persist one row, restart all nodes, then require Running Regions and both rows after a write. @Test public void testWriteAfterClusterRestart() throws SQLException { try (final Connection connection = EnvFactory.getEnv().getTableConnection(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java index d24c4737c25e5..5e8dce8b49e81 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java @@ -31,7 +31,8 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.junit.After; -import org.junit.Before; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -61,8 +62,9 @@ public class IoTDBRestartTableIT { private final Logger logger = LoggerFactory.getLogger(IoTDBRestartTableIT.class); - @Before - public void setUp() throws Exception { + // Reuse the fixed WAL/schema configuration while retaining every scenario's recovery restarts. + @BeforeClass + public static void setUp() throws Exception { EnvFactory.getEnv() .getConfig() .getCommonConfig() @@ -82,8 +84,8 @@ public void setUp() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); } - @After - public void tearDown() throws Exception { + @AfterClass + public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); EnvFactory.getEnv().getConfig().getCommonConfig().setWalMode("ASYNC"); EnvFactory.getEnv() @@ -92,6 +94,18 @@ public void tearDown() throws Exception { .setSchemaRegionConsensusProtocolClass("org.apache.iotdb.consensus.simple.SimpleConsensus"); } + @After + public void cleanDatabase() throws SQLException { + // Recreate each scenario's tables without retaining data or schema from the preceding test. + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("drop database if exists \"test\""); + } + } + + /** + * Write flushed and unflushed table rows across two restarts, then verify all three timestamps. + */ @Test public void testRestart() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -139,6 +153,10 @@ public void testRestart() throws SQLException { } } + /** + * Recover three points and verify the remaining values after two range deletions, with a flush + * between them. + */ @Test public void testRestartDelete() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -255,6 +273,7 @@ private static void assertRestartTableData(Statement statement, String... target } } + /** Write rows across two restarts and verify the time filter returns only the row at time 4. */ @Test public void testRestartQueryLargerThanEndTime() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -307,6 +326,9 @@ public void testRestartQueryLargerThanEndTime() throws SQLException { } } + /** + * Add another field at earlier timestamps after recovery and verify both values survive restart. + */ @Test public void testRestartEndTime() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -356,6 +378,10 @@ public void testRestartEndTime() throws SQLException { } } + /** + * Retain the disabled column-deletion scenario: recreate a series with a different type and + * verify recovery discards its incompatible WAL value while preserving the other series. + */ @Ignore // delete column @Test public void testRecoverWALMismatchDataType() throws Exception { @@ -389,6 +415,10 @@ public void testRecoverWALMismatchDataType() throws Exception { } } + /** + * Retain the disabled column-deletion scenario: delete one series before restart and verify that + * only the remaining series and its value recover. + */ @Ignore // delete column @Test public void testRecoverWALDeleteSchema() throws Exception { @@ -419,6 +449,7 @@ public void testRecoverWALDeleteSchema() throws Exception { } } + /** Write fields at different times, restart DataNodes, and verify both values before time 3. */ @Test public void testRecoverWALDeleteSchemaCheckResourceTime() throws Exception { IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); @@ -461,6 +492,9 @@ public void testRecoverWALDeleteSchemaCheckResourceTime() throws Exception { config.setUnSeqTsFileSize(unFsFileSize); } + /** + * Restart after writing an unflushed table row and verify its timestamp and both field values. + */ @Test public void testRecoverFromFlushMemTableError() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java index b0cccba6f6cb9..58b3f0774df18 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java @@ -67,8 +67,7 @@ public class IoTDBInsertQueryIT { "CREATE TABLE IF NOT EXISTS vehicle0(deviceId STRING TAG, manufacturer STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)", "CREATE TABLE IF NOT EXISTS tb_0(s0 INT32 FIELD, s1 TEXT FIELD, deviceId STRING TAG)", }; - private static final String insertTemplate = - "INSERT INTO vehicle0(time,deviceId,manufacturer,s0,s1,s2,s3,s4) VALUES(%d,'d%d','%s',%d,%d,%f,'%d',%b)"; + private static final String insertValuesTemplate = "(%d,'d%d','%s',%d,%d,%f,'%d',%b)"; private static final String insertIntoQuery = "INSERT INTO vehicle%d SELECT * FROM vehicle0"; @@ -124,6 +123,7 @@ protected static void prepareDatabase() { } } + /** Copy all prepared rows with SELECT, TABLE and descending source order, preserving values. */ @Test public void testSimple() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -167,6 +167,7 @@ public void testSimple() throws SQLException { } } + /** Insert a table's own rows at shifted timestamps and verify that its row count doubles. */ @Test public void testInsertIntoSourceTable() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -196,6 +197,7 @@ public void testInsertIntoSourceTable() throws SQLException { } } + /** Copy rows whose source schema declares the tag last, preserving column categories and data. */ @Test public void testTagCategory() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -217,6 +219,7 @@ public void testTagCategory() throws SQLException { } } + /** Copy rows using fully qualified source and target names without selecting a database. */ @Test public void testQualifiedName() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -241,6 +244,7 @@ public void testQualifiedName() throws SQLException { } } + /** Reject copies without time or field columns, and fill omitted tags and fields with null. */ @Test public void testPartialColumn() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -305,13 +309,14 @@ public void testPartialColumn() throws SQLException { } } + /** Require both source SELECT and target INSERT privileges before allowing a copy. */ @Test public void testPrivileges() throws SQLException { Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement adminStmt = adminCon.createStatement(); try { // create user - adminStmt.execute("CREATE USER tmpuser 'tmppw123456789'"); + adminStmt.execute("CREATE USER tmpuser 'IoTDB!Test2024'"); // create table vehicle1 adminStmt.execute("USE test"); adminStmt.execute(String.format(createTableTemplate, 1)); @@ -319,7 +324,7 @@ public void testPrivileges() throws SQLException { try (Connection connection = EnvFactory.getEnv() - .getConnection("tmpuser", "tmppw123456789", BaseEnv.TABLE_SQL_DIALECT); + .getConnection("tmpuser", "IoTDB!Test2024", BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("USE test"); // insert into vehicle1 select * from vehicle0 @@ -339,7 +344,7 @@ public void testPrivileges() throws SQLException { try (Connection connection = EnvFactory.getEnv() - .getConnection("tmpuser", "tmppw123456789", BaseEnv.TABLE_SQL_DIALECT); + .getConnection("tmpuser", "IoTDB!Test2024", BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("USE test"); // insert into vehicle1 select * from vehicle0 @@ -358,7 +363,7 @@ public void testPrivileges() throws SQLException { // adminStmt.execute("GRANT WRITE_DATA ON test.vehicle1 TO USER tmpuser"); try (Connection connection = EnvFactory.getEnv() - .getConnection("tmpuser", "tmppw123456789", BaseEnv.TABLE_SQL_DIALECT); + .getConnection("tmpuser", "IoTDB!Test2024", BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("USE test"); // insert into vehicle1 select * from vehicle0 @@ -371,6 +376,7 @@ public void testPrivileges() throws SQLException { } } + /** Insert 50 ms tumbling-window aggregates and compare their values with the source query. */ @Test public void testAggregate() throws SQLException { String querySql = @@ -400,6 +406,7 @@ public void testAggregate() throws SQLException { } } + /** Reject INSERT SELECT when its target table has not been created. */ @Test public void testTableNotExists() { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -415,6 +422,7 @@ public void testTableNotExists() { } } + /** Reject a copy when the target field type is incompatible with the source column type. */ @Test public void testColumnTypeMismatch1() throws SQLException { String errMsg = @@ -436,6 +444,7 @@ public void testColumnTypeMismatch1() throws SQLException { } } + /** Reject a copy when the target schema has fewer columns than the source query. */ @Test public void testColumnTypeMismatch2() throws SQLException { String errMsg = @@ -457,6 +466,9 @@ public void testColumnTypeMismatch2() throws SQLException { } } + /** + * Reject INSERT statements passed to EXPLAIN or EXPLAIN ANALYZE with the expected parse errors. + */ @Test public void testExplain() throws SQLException { Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -486,6 +498,7 @@ public void testExplain() throws SQLException { } } + /** Copy through the table Session non-query API and verify all 1200 rows and numeric values. */ @Test public void testNonQueryStatement() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -514,6 +527,7 @@ public void testNonQueryStatement() throws IoTDBConnectionException, StatementEx } } + /** Passing INSERT SELECT to the query API must fail with the expected missing-column metadata. */ @Test public void testQueryStatement() throws IoTDBConnectionException, StatementExecutionException { ITableSession session = EnvFactory.getEnv().getTableSessionConnection(); @@ -536,6 +550,7 @@ public void testQueryStatement() throws IoTDBConnectionException, StatementExecu } } + /** Copy a boolean field from a schema with its tag last into a schema with its tag first. */ @Test public void testDeviceTagLast() { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -612,50 +627,52 @@ protected static void prepareData() throws SQLException { statement.execute("use test"); // prepare seq file for (int d = 0; d < deviceNum; d++) { - for (int i = 201; i <= 300; i++) { - String sql = - String.format( - insertTemplate, i, d, manufacturers[i % 2], i, i, (double) i, i, i % 2 == 0); - statement.execute(sql); - statement.execute( - String.format("insert into tb_0 values(%d, %d, '%d', 'd%d')", i, i, i, d)); - } + insertRange(statement, d, 201, 300); } statement.execute("flush"); // prepare unseq File for (int d = 0; d < deviceNum; d++) { - for (int i = 1; i <= 100; i++) { - String sql = - String.format( - insertTemplate, i, d, manufacturers[i % 2], i, i, (double) i, i, i % 2 == 0); - statement.execute(sql); - statement.execute( - String.format("insert into tb_0 values(%d, %d, '%d', 'd%d')", i, i, i, d)); - } + insertRange(statement, d, 1, 100); } statement.execute("flush"); for (int d = 0; d < deviceNum; d++) { // prepare BufferWrite cache - for (int i = 301; i <= 400; i++) { - String sql = - String.format( - insertTemplate, i, d, manufacturers[i % 2], i, i, (double) i, i, i % 2 == 0); - statement.execute(sql); - statement.execute( - String.format("insert into tb_0 values(%d, %d, '%d', 'd%d')", i, i, i, d)); - } + insertRange(statement, d, 301, 400); // prepare Overflow cache - for (int i = 101; i <= 200; i++) { - String sql = - String.format( - insertTemplate, i, d, manufacturers[i % 2], i, i, (double) i, i, i % 2 == 0); - statement.execute(sql); - statement.execute( - String.format("insert into tb_0 values(%d, %d, '%d', 'd%d')", i, i, i, d)); - } + insertRange(statement, d, 101, 200); + } + } + } + + private static void insertRange(Statement statement, int deviceId, int startTime, int endTime) + throws SQLException { + // These writes only prepare the source tables. Batch each 100-row device/time range while + // preserving every value, each table's write order and the explicit flush boundaries above. + StringBuilder vehicleInsert = + new StringBuilder( + "INSERT INTO vehicle0(time,deviceId,manufacturer,s0,s1,s2,s3,s4) VALUES "); + StringBuilder tagLastInsert = new StringBuilder("INSERT INTO tb_0 VALUES "); + for (int time = startTime; time <= endTime; time++) { + if (time > startTime) { + vehicleInsert.append(','); + tagLastInsert.append(','); } + vehicleInsert.append( + String.format( + insertValuesTemplate, + time, + deviceId, + manufacturers[time % 2], + time, + time, + (double) time, + time, + time % 2 == 0)); + tagLastInsert.append(String.format("(%d,%d,'%d','d%d')", time, time, time, deviceId)); } + statement.execute(vehicleInsert.toString()); + statement.execute(tagLastInsert.toString()); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/mqtt/IoTDBMQTTServiceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/mqtt/IoTDBMQTTServiceIT.java index 031322cc5027a..fc3c074fbb1d0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/mqtt/IoTDBMQTTServiceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/mqtt/IoTDBMQTTServiceIT.java @@ -18,6 +18,7 @@ package org.apache.iotdb.relational.it.mqtt; import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; @@ -33,43 +34,58 @@ import org.fusesource.mqtt.client.MQTT; import org.fusesource.mqtt.client.QoS; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; -import java.io.IOException; import java.util.List; +import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) public class IoTDBMQTTServiceIT { private BlockingConnection connection; - private static final String IP = System.getProperty("RemoteIp", "127.0.0.1"); - private static final String USER = System.getProperty("RemoteUser", "root"); - private static final String PASSWORD = System.getProperty("RemotePassword", "root"); - private static final String DATABASE = "mqtttest"; + private static final String USER = System.getProperty("RemoteUser", SessionConfig.DEFAULT_USER); + private static final String PASSWORD = + System.getProperty("RemotePassword", SessionConfig.DEFAULT_PASSWORD); public static final String FORMATTER = "line"; - @Before - public void setUp() throws Exception { + @BeforeClass + public static void setUpClass() { + // Both cases use the same MQTT configuration; separate databases and client IDs isolate them. BaseEnv baseEnv = EnvFactory.getEnv(); baseEnv.getConfig().getDataNodeConfig().setEnableMQTTService(true); baseEnv.getConfig().getDataNodeConfig().setMqttPayloadFormatter(FORMATTER); + // Serialize CONNECT and PUBLISH callbacks so the session is registered before each + // payload's single publish is processed. + baseEnv.getConfig().getDataNodeConfig().setMqttHandlerPoolSize(1); baseEnv.initClusterEnvironment(); - DataNodeWrapper portConflictDataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); - int port = portConflictDataNodeWrapper.getMqttPort(); + } + + @AfterClass + public static void tearDownClass() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Before + public void setUp() throws Exception { + DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); MQTT mqtt = new MQTT(); - mqtt.setHost(IP, port); + mqtt.setHost( + System.getProperty("RemoteIp", dataNodeWrapper.getIp()), dataNodeWrapper.getMqttPort()); mqtt.setUserName(USER); mqtt.setPassword(PASSWORD); mqtt.setConnectAttemptsMax(3); mqtt.setReconnectDelay(10); - mqtt.setClientId("clientId01"); + // A delayed disconnect must not remove the next test's broker session. Keep IDs within the + // 23-character MQTT 3.1 limit. + mqtt.setClientId("client-" + UUID.randomUUID().toString().substring(0, 16)); connection = mqtt.blockingConnection(); connection.connect(); @@ -77,30 +93,27 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - try { - if (connection != null) { - connection.disconnect(); - } - } catch (IOException e) { - e.printStackTrace(); - fail(e.getMessage()); + if (connection != null) { + connection.disconnect(); } - EnvFactory.getEnv().cleanClusterEnvironment(); } + // A single line-format publish without attributes must create the two tags and typed fields, + // with the expected values visible at time 1 in this test's database. @Test public void testNoAttr() throws Exception { + final String database = "mqtttest_no_attr"; try (final ITableSession session = - EnvFactory.getEnv().getTableSessionConnectionWithDB(DATABASE)) { - session.executeNonQueryStatement("CREATE DATABASE " + DATABASE); + EnvFactory.getEnv().getTableSessionConnectionWithDB(database)) { + session.executeNonQueryStatement("CREATE DATABASE " + database); String payload1 = "test1,tag1=t1,tag2=t2 field1=1,field2=1f,field3=1i32 1"; + connection.publish(database + "/myTopic", payload1.getBytes(), QoS.AT_LEAST_ONCE, false); Awaitility.await() .atMost(3, TimeUnit.MINUTES) - .pollInterval(1, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .until( () -> { - connection.publish( - DATABASE + "/myTopic", payload1.getBytes(), QoS.AT_LEAST_ONCE, false); try (final SessionDataSet dataSet = session.executeQueryStatement( "select tag1,tag2,field1,field2,field3 from test1 where time = 1")) { @@ -125,19 +138,22 @@ public void testNoAttr() throws Exception { } } + // A single line-format publish with attributes must preserve both attribute strings, the two + // tags and all three typed field values at time 1 in a separate database. @Test public void testWithAttr() throws Exception { + final String database = "mqtttest_with_attr"; try (final ITableSession session = - EnvFactory.getEnv().getTableSessionConnectionWithDB(DATABASE)) { - session.executeNonQueryStatement("CREATE DATABASE " + DATABASE); + EnvFactory.getEnv().getTableSessionConnectionWithDB(database)) { + session.executeNonQueryStatement("CREATE DATABASE " + database); String payload1 = "test2,tag1=t1,tag2=t2 attr3=a3,attr4=a4 field1=1,field2=1f,field3=1i32 1"; + connection.publish(database + "/myTopic", payload1.getBytes(), QoS.AT_LEAST_ONCE, false); Awaitility.await() .atMost(3, TimeUnit.MINUTES) - .pollInterval(1, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) .until( () -> { - connection.publish( - DATABASE + "/myTopic", payload1.getBytes(), QoS.AT_LEAST_ONCE, false); try (final SessionDataSet dataSet = session.executeQueryStatement( "select tag1,tag2,attr3,attr4,field1,field2,field3 from test2 where time = 1")) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java index 47e7655ff4428..1e278f10858ad 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java @@ -28,8 +28,9 @@ import com.google.common.collect.ImmutableSet; import org.apache.tsfile.enums.TSDataType; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -51,6 +52,7 @@ import java.util.Set; import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.dropDatabaseWithRetry; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -62,13 +64,23 @@ public class IoTDBSimpleQueryTableIT { private static final String DATABASE_NAME = "test"; - @Before - public void setUp() throws Exception { + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); } @After - public void tearDown() throws Exception { + public void tearDown() throws SQLException { + // Keep the catalog empty for the database-listing test while reusing the cluster. + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + dropDatabaseWithRetry(statement, DATABASE_NAME); + dropDatabaseWithRetry(statement, "group_with_hyphen"); + } + } + + @AfterClass + public static void tearDownClass() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java index e8089db62f13b..81fea94fd88fd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java @@ -19,7 +19,6 @@ package org.apache.iotdb.relational.it.query.recent; -import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.isession.SessionDataSet; @@ -31,10 +30,8 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; -import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -94,6 +91,9 @@ public static void setUpClass() { .setPartitionInterval(1000) .setMemtableSizeThreshold(10000); EnvFactory.getEnv().initClusterEnvironment(); + // CTE queries only read the same three rows. The privilege test removes its own user and + // extra table, so prepare the shared input once instead of recreating it for every method. + prepareData(); } @AfterClass @@ -101,21 +101,6 @@ public static void tearDownClass() { EnvFactory.getEnv().cleanClusterEnvironment(); } - @Before - public void setUp() throws SQLException { - prepareData(); - } - - @After - public void tearDown() { - try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - Statement statement = connection.createStatement()) { - TestUtils.dropDatabaseWithRetry(statement, "testdb"); - } catch (Exception e) { - fail(e.getMessage()); - } - } - @Test public void testMultipleWith() { String mainQuery = @@ -458,12 +443,13 @@ public void testRecursive() { } } + // Every CTE materialization mode requires SELECT on referenced tables, including unused CTEs. @Test public void testPrivileges() throws SQLException { Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement adminStmt = adminCon.createStatement(); try { - adminStmt.execute("CREATE USER tmpuser 'tmppw123456789'"); + adminStmt.execute("CREATE USER tmpuser 'IoTDB!Test2024'"); adminStmt.execute("USE testdb"); adminStmt.execute( "CREATE TABLE IF NOT EXISTS testtb1(deviceid STRING TAG, voltage FLOAT FIELD)"); @@ -471,7 +457,7 @@ public void testPrivileges() throws SQLException { try (Connection connection = EnvFactory.getEnv() - .getConnection("tmpuser", "tmppw123456789", BaseEnv.TABLE_SQL_DIALECT); + .getConnection("tmpuser", "IoTDB!Test2024", BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("USE testdb"); for (String keyword : cteKeywords) { @@ -482,7 +468,7 @@ public void testPrivileges() throws SQLException { try (Connection connection = EnvFactory.getEnv() - .getConnection("tmpuser", "tmppw123456789", BaseEnv.TABLE_SQL_DIALECT); + .getConnection("tmpuser", "IoTDB!Test2024", BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("USE testdb"); for (String keyword : cteKeywords) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBDistinctTagIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBDistinctTagIT.java index af450f226f8ac..ce28ee5a368af 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBDistinctTagIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBDistinctTagIT.java @@ -34,7 +34,6 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; -import java.util.concurrent.TimeUnit; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.fail; @@ -75,6 +74,7 @@ public class IoTDBDistinctTagIT { "insert into t2(time, deviceId, attr1, s1) values(3000, 'd2', 'a2', 12)", "insert into t2(time, deviceId, attr1, s1) values(2000, 'd1', 'a1', 13)", "clear attribute cache", + // Cache clearing completes on all DataNodes before the statement returns. "clear all cache", }; @@ -90,6 +90,7 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + // DISTINCT returns each tag and latest attribute once for both flushed and in-memory tables. @Test public void testDistinct() { // distinct(deviceId) @@ -123,6 +124,7 @@ public void testDistinct() { DATABASE_NAME); } + // Time filters restrict distinct tags and attributes to the matching rows in both tables. @Test public void testDistinctWithTimeFilter() { // distinct(deviceId) ... where time > 3000; @@ -156,6 +158,7 @@ public void testDistinctWithTimeFilter() { DATABASE_NAME); } + // Pushed-down field filters preserve the expected distinct tags and attributes in both tables. @Test public void testDistinctWithPushDownFilter() { // distinct(deviceId) ... where s1 = 11; @@ -189,6 +192,7 @@ public void testDistinctWithPushDownFilter() { DATABASE_NAME); } + // Deleting all flushed or in-memory rows leaves no distinct tags or attributes to query. @Test public void testDistinctWithDelete() throws SQLException { String[] sqls = @@ -258,9 +262,6 @@ private static void prepareData() { for (String sql : SQLs) { statement.execute(sql); } - - // try to wait attribute cache cleared - TimeUnit.SECONDS.sleep(5); } catch (Exception e) { fail(e.getMessage()); } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBSimpleQueryTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBSimpleQueryTableViewIT.java index 91f4998ac537b..fbf773e3df9cd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBSimpleQueryTableViewIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/IoTDBSimpleQueryTableViewIT.java @@ -27,8 +27,9 @@ import org.apache.tsfile.enums.TSDataType; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -46,6 +47,7 @@ import java.util.List; import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; +import static org.apache.iotdb.db.it.utils.TestUtils.dropDatabaseWithRetry; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -57,13 +59,32 @@ public class IoTDBSimpleQueryTableViewIT { private static final String DATABASE_NAME = "test"; - @Before - public void setUp() throws Exception { + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); } @After - public void tearDown() throws Exception { + public void tearDown() throws SQLException { + // Remove each method's views before deleting their Tree-model source database. + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + dropDatabaseWithRetry(statement, DATABASE_NAME); + } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + boolean databaseExists; + try (ResultSet resultSet = statement.executeQuery("SHOW DATABASES root.sg1")) { + databaseExists = resultSet.next(); + } + if (databaseExists) { + statement.execute("DELETE DATABASE root.sg1"); + } + } + } + + @AfterClass + public static void tearDownClass() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java index cec9028edb298..0b256b4d71932 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.relational.it.query.view.recent; +import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; @@ -26,9 +27,10 @@ import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; import org.apache.tsfile.read.common.RowRecord; -import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -39,9 +41,6 @@ import java.util.Collection; import java.util.List; -import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; -import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; - @RunWith(Parameterized.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) public class IoTDBTableViewQueryIT { @@ -102,8 +101,6 @@ public class IoTDBTableViewQueryIT { }; protected static String[] createTableSqls = { - "CREATE DATABASE " + DATABASE_NAME, - "USE " + DATABASE_NAME, "CREATE VIEW view1 (battery TAG, voltage INT32 FIELD, current FLOAT FIELD) as root.db.battery.**", "CREATE VIEW view2 (battery TAG, voltage INT32 FIELD FROM voltage, current_rename FLOAT FIELD FROM current) as root.db.battery.**", "CREATE VIEW view3 (battery TAG, voltage INT32 FIELD FROM voltage, current_rename FLOAT FIELD FROM current) with (ttl=1) as root.db.battery.**", @@ -137,32 +134,59 @@ public static Collection data() { private final boolean aligned; private final boolean flush; + private final String databaseName; + private final String treeDatabaseName; public IoTDBTableViewQueryIT(boolean aligned, boolean flush) { this.aligned = aligned; this.flush = flush; + // Isolate both models so parameter cases can share the cluster without reusing fixture state. + databaseName = + DATABASE_NAME + (aligned ? "_aligned" : "_non_aligned") + (flush ? "_flushed" : "_memory"); + treeDatabaseName = "root." + databaseName; } - @Before - public void setUp() throws Exception { + @BeforeClass + public static void setUp() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); EnvFactory.getEnv().initClusterEnvironment(); - prepareData(aligned ? createTreeAlignedDataSqls : createTreeNonAlignedDataSqls); - if (flush) { - prepareData(new String[] {"flush"}); + } + + @Before + public void prepareData() throws Exception { + // Reuse the tree session for schema, writes and the optional flush of this case's source. + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE " + treeDatabaseName); + for (String sql : aligned ? createTreeAlignedDataSqls : createTreeNonAlignedDataSqls) { + session.executeNonQueryStatement(sql.replace("root.db.", treeDatabaseName + ".")); + } + if (flush) { + // A global flush would also flush another case's source intended to stay in memory. + session.executeNonQueryStatement("FLUSH " + treeDatabaseName); + } + } + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE " + databaseName); + session.executeNonQueryStatement("USE " + databaseName); + for (String sql : createTableSqls) { + session.executeNonQueryStatement(sql.replace("root.db.", treeDatabaseName + ".")); + } } - prepareTableData(createTableSqls); } - @After - public void tearDown() throws Exception { + @AfterClass + public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + // Compare tree-backed views with a physical table for aligned/non-aligned, flushed/memory + // fixtures; nulls, TTL, filters, fills, joins and aggregations must keep the same results. @Test public void test() throws Exception { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + // All comparisons use this case's database, so select it only once per session. + session.executeNonQueryStatement("USE " + databaseName); compareQueryResults(session, "select * from view1", "select * from table1", true); compareQueryResults( session, @@ -436,7 +460,6 @@ private static void compareQueryResults( ITableSession session, String sql1, String sql2, boolean sort) throws Exception { List records1 = new ArrayList<>(); List records2 = new ArrayList<>(); - session.executeNonQueryStatement("USE " + DATABASE_NAME); SessionDataSet sessionDataSet = session.executeQueryStatement(sql1); while (sessionDataSet.hasNext()) { RowRecord record = sessionDataSet.next(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryWithNotMatchedDataTypeIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryWithNotMatchedDataTypeIT.java index 068942ad25071..9dd00fd897556 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryWithNotMatchedDataTypeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryWithNotMatchedDataTypeIT.java @@ -27,9 +27,10 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; -import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -48,6 +49,8 @@ public class IoTDBTableViewQueryWithNotMatchedDataTypeIT { protected static final String DATABASE_NAME = "test"; private final boolean aligned; + private final String databaseName; + private final String treeDatabaseName; protected static final String[] createTreeAlignedDataSqls = new String[] { @@ -61,14 +64,18 @@ public class IoTDBTableViewQueryWithNotMatchedDataTypeIT { "INSERT INTO root.db.battery.b1(time, current) values (1, 1)", }; - protected static String[] createTableSqls = { - "create database " + DATABASE_NAME, - "use " + DATABASE_NAME, - "CREATE VIEW view1 (battery TAG, current BLOB FIELD) as root.db.battery.**", - }; + protected static String[] createTableSqls(String databaseName, String treeDatabaseName) { + return new String[] { + "create database " + databaseName, + "use " + databaseName, + "CREATE VIEW view1 (battery TAG, current BLOB FIELD) as " + treeDatabaseName + ".battery.**", + }; + } public IoTDBTableViewQueryWithNotMatchedDataTypeIT(boolean aligned) { this.aligned = aligned; + databaseName = DATABASE_NAME + (aligned ? "_aligned" : "_non_aligned"); + treeDatabaseName = "root." + databaseName; } @Parameterized.Parameters(name = "aligned={0}") @@ -76,24 +83,35 @@ public static Collection data() { return Arrays.asList(new Object[][] {{true}, {false}}); } - @Before - public void setUp() throws Exception { + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); EnvFactory.getEnv().initClusterEnvironment(); - prepareData(aligned ? createTreeAlignedDataSqls : createTreeNonAlignedDataSqls); - prepareTableData(createTableSqls); } - @After - public void tearDown() throws Exception { + @Before + public void prepareDataForCase() { + String[] dataSqls = aligned ? createTreeAlignedDataSqls : createTreeNonAlignedDataSqls; + String[] caseDataSqls = new String[dataSqls.length + 1]; + caseDataSqls[0] = "CREATE DATABASE " + treeDatabaseName; + for (int i = 0; i < dataSqls.length; i++) { + caseDataSqls[i + 1] = dataSqls[i].replace("root.db.", treeDatabaseName + "."); + } + prepareData(caseDataSqls); + prepareTableData(createTableSqls(databaseName, treeDatabaseName)); + } + + @AfterClass + public static void tearDownClass() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + // A BLOB view must exclude the FLOAT tree measurement for both aligned and non-aligned sources. @Test public void test() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { - session.executeNonQueryStatement("USE " + DATABASE_NAME); + session.executeNonQueryStatement("USE " + databaseName); SessionDataSet sessionDataSet = session.executeQueryStatement("select * from view1 where current is not null"); Assert.assertFalse(sessionDataSet.hasNext()); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java index cdb7254b8c7a5..6c6a49a47228f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java @@ -30,8 +30,9 @@ import org.apache.iotdb.jdbc.IoTDBSQLException; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -41,6 +42,7 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -59,8 +61,9 @@ @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) public class IoTDBDatabaseIT { - @Before - public void setUp() throws Exception { + // All scenarios use the same node configuration; reset their catalogs between methods. + @BeforeClass + public static void setUp() throws Exception { EnvFactory.getEnv() .getConfig() .getCommonConfig() @@ -78,11 +81,74 @@ public void setUp() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); } - @After - public void tearDown() throws Exception { + @AfterClass + public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + @After + public void resetCatalog() throws SQLException { + // These scenarios assert complete catalogs, requiring an empty namespace between methods. + try (final Connection connection = EnvFactory.getEnv().getTableConnection(); + final Statement statement = connection.createStatement(); + final Connection treeConnection = EnvFactory.getEnv().getConnection(); + final Statement treeStatement = treeConnection.createStatement()) { + // Drop pipes first so write-back sinks cannot recreate tables during database cleanup. + // Double-living creates separate tree/table pipes with the same name; drop both copies. + for (final Statement pipeStatement : new Statement[] {statement, treeStatement}) { + final List names = new ArrayList<>(); + try (final ResultSet resultSet = pipeStatement.executeQuery("SHOW PIPES")) { + while (resultSet.next()) { + names.add(resultSet.getString(1)); + } + } + for (final String name : names) { + pipeStatement.execute("DROP PIPE " + name); + } + } + + final List databases = new ArrayList<>(); + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) { + while (resultSet.next()) { + final String database = resultSet.getString(1); + if (!"information_schema".equals(database)) { + databases.add(database); + } + } + } + for (final String database : databases) { + // Database-name tests include backticks and numeric identifiers; always quote names. + statement.execute("DROP DATABASE \"" + database.replace("\"", "\"\"") + "\""); + } + + // Dropping the test user also removes grants before the next permission scenario reuses it. + boolean testUserExists = false; + try (final ResultSet resultSet = statement.executeQuery("LIST USER")) { + while (resultSet.next()) { + if ("test".equals(resultSet.getString("User"))) { + testUserExists = true; + break; + } + } + } + if (testUserExists) { + statement.execute("DROP USER test"); + } + // The mixed-model scenario also creates a tree database with the same unqualified name. + final boolean databaseExists; + try (final ResultSet resultSet = treeStatement.executeQuery("SHOW DATABASES root.test")) { + databaseExists = resultSet.next(); + } + if (databaseExists) { + treeStatement.execute("DELETE DATABASE root.test"); + } + } + } + + /** + * Create, alter, list, count and drop databases, checking default/custom properties, duplicate + * and missing names, invalid identifiers, and the 64-character name boundary. + */ @Test public void testManageDatabase() { try (final Connection connection = @@ -298,6 +364,10 @@ public void testManageDatabase() { } } + /** + * Create a database with explicit TTL and partition interval, then verify SHOW CREATE retains + * these values and includes the resolved region-group limits. + */ @Test public void testShowCreateDatabase() throws SQLException { try (final Connection connection = @@ -323,6 +393,10 @@ public void testShowCreateDatabase() throws SQLException { } } + /** + * Create databases with disabled and default last caches, alter/reset the property, and verify + * both metadata views agree while a non-boolean property is rejected. + */ @Test public void testNeedLastCacheDatabaseProperty() throws SQLException { try (final Connection connection = @@ -354,6 +428,10 @@ public void testNeedLastCacheDatabaseProperty() throws SQLException { } } + /** + * Create a pipe using the do-nothing sink and verify SHOW CREATE returns the pipe name and the + * exact sink configuration needed to recreate it. + */ @Test public void testShowCreatePipe() throws SQLException { try (final Connection connection = @@ -369,6 +447,10 @@ public void testShowCreatePipe() throws SQLException { } } + /** + * Request CREATE statements for information_schema and __audit, verifying both system databases + * reject SHOW CREATE with the expected diagnostic rather than exposing a creation statement. + */ @Test public void testShowCreateInformationSchemaDatabase() throws SQLException { try (final Connection connection = @@ -413,6 +495,10 @@ private static void assertShowCreateSystemDatabaseFails( } } + /** + * Reject illegal database-name characters, then create a backtick-containing name and verify + * quoted table creation, insertion, queries, device metadata, and attribute updates succeed. + */ @Test public void testDatabaseWithSpecificCharacters() throws SQLException { try (final Connection connection = @@ -478,17 +564,21 @@ public void testDatabaseWithSpecificCharacters() throws SQLException { } } + /** + * Check read-only system metadata and ordinary-user visibility, then verify administrator + * metadata for tables, views, pipes, functions, configuration, and the running cluster. + */ @Test public void testInformationSchema() throws SQLException { // Use a normal user to test visibility try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement adminStmt = adminCon.createStatement()) { - adminStmt.execute("create user test 'password123456'"); + adminStmt.execute("create user test 'IoTDB!Test2024'"); adminStmt.execute("create pipe test ('sink'='do-nothing-sink')"); } try (final Connection connection = - EnvFactory.getEnv().getConnection("test", "password123456", BaseEnv.TABLE_SQL_DIALECT); + EnvFactory.getEnv().getConnection("test", "IoTDB!Test2024", BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { // Test unsupported write plans final Set writeSQLs = @@ -1011,6 +1101,10 @@ public void testInformationSchema() throws SQLException { } } + /** + * Create tree and table databases named test, exercise independent data and cache properties, + * then verify dropping the table database preserves the tree database and model-specific counts. + */ @Test public void testMixedDatabase() throws SQLException { try (final Connection connection = @@ -1089,6 +1183,10 @@ public void testMixedDatabase() throws SQLException { } } + /** + * Create two table databases and verify both COUNT DATABASES and information_schema include + * exactly these databases plus the built-in information_schema database in their totals. + */ @Test public void testCountDatabases() throws SQLException { try (final Connection connection = @@ -1106,6 +1204,10 @@ public void testCountDatabases() throws SQLException { } } + /** + * Under the default AUTO region policy, reject explicit maximum data-region counts in both CREATE + * and ALTER DATABASE while accepting database creation without that property. + */ @Test public void testMaxRegionGroupNumRejectedInAutoPolicy() throws SQLException { try (final Connection connection = @@ -1134,18 +1236,22 @@ public void testMaxRegionGroupNumRejectedInAutoPolicy() throws SQLException { } } + /** + * Start a new user without database access, grant SELECT, then grant DROP and verify metadata + * visibility and rejected ALTER/DROP operations follow the permissions at each step. + */ @Test public void testDBAuth() throws SQLException { try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement adminStmt = adminCon.createStatement()) { - adminStmt.execute("create user test 'password123456'"); + adminStmt.execute("create user test 'IoTDB!Test2024'"); adminStmt.execute("create database db"); adminStmt.execute( "create pipe a2b with source('double-living'='true') with sink ('sink'='write-back-sink')"); } try (final Connection userCon = - EnvFactory.getEnv().getConnection("test", "password123456", BaseEnv.TABLE_SQL_DIALECT); + EnvFactory.getEnv().getConnection("test", "IoTDB!Test2024", BaseEnv.TABLE_SQL_DIALECT); final Statement userStmt = userCon.createStatement()) { TestUtils.assertResultSetEqual( userStmt.executeQuery("count databases"), "count,", Collections.singleton("1,")); @@ -1179,7 +1285,7 @@ public void testDBAuth() throws SQLException { } try (final Connection userCon = - EnvFactory.getEnv().getConnection("test", "password123456", BaseEnv.TABLE_SQL_DIALECT); + EnvFactory.getEnv().getConnection("test", "IoTDB!Test2024", BaseEnv.TABLE_SQL_DIALECT); final Statement userStmt = userCon.createStatement()) { TestUtils.assertResultSetEqual( userStmt.executeQuery("count databases"), "count,", Collections.singleton("2,")); @@ -1215,7 +1321,7 @@ public void testDBAuth() throws SQLException { } try (final Connection userCon = - EnvFactory.getEnv().getConnection("test", "password123456", BaseEnv.TABLE_SQL_DIALECT); + EnvFactory.getEnv().getConnection("test", "IoTDB!Test2024", BaseEnv.TABLE_SQL_DIALECT); final Statement userStmt = userCon.createStatement()) { userStmt.execute("drop database db"); } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseMaxRegionGroupNumIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseMaxRegionGroupNumIT.java index cdca1fd1b7665..4520d14f6c843 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseMaxRegionGroupNumIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseMaxRegionGroupNumIT.java @@ -29,8 +29,9 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -55,8 +56,8 @@ public class IoTDBDatabaseMaxRegionGroupNumIT { private static final String TABLE_NAME = "table1"; private static final BKDRHashExecutor PARTITION_EXECUTOR = new BKDRHashExecutor(SERIES_SLOT_NUM); - @Before - public void setUp() throws Exception { + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv() .getConfig() .getCommonConfig() @@ -71,7 +72,29 @@ public void setUp() throws Exception { } @After - public void tearDown() throws Exception { + public void tearDown() throws SQLException { + // Keep each method's region-quota assertions independent while reusing the cluster. + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + for (final String database : + new String[] { + "test_create", + "test_create_schema", + "test_create_data", + "test_alter", + "test_deprecated", + "test_decrease_data", + "test_decrease_schema", + "test_partition" + }) { + TestUtils.dropDatabaseWithRetry(statement, database); + } + } + } + + @AfterClass + public static void tearDownClass() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java index 73e9881dfcaa2..0fe69987b9e1b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java @@ -68,6 +68,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -1468,50 +1469,41 @@ public void insertWrongFieldNumTest() throws IoTDBConnectionException { } private void testOneCastWithTablet( - TSDataType from, TSDataType to, int testNum, boolean partialInsert) + ITableSession session, + TSDataType from, + TSDataType to, + String tableName, + boolean partialInsert) throws IoTDBConnectionException, StatementExecutionException { - try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { - session.executeNonQueryStatement("USE \"db1\""); - // create a column with type of "to" - session.executeNonQueryStatement( - "CREATE TABLE table" - + testNum - + " (tag1 string tag," - + "m1 " - + to.toString() - + " field)"); - if (partialInsert) { - session.executeNonQueryStatement("SET CONFIGURATION enable_partial_insert='true'"); - } else { - session.executeNonQueryStatement("SET CONFIGURATION enable_partial_insert='false'"); - } - - // insert a tablet with type of "from" - List schemaList = new ArrayList<>(); - schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); - schemaList.add(new MeasurementSchema("m1", from)); - final List columnTypes = - Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD); - Tablet tablet = - new Tablet( - "table" + testNum, - IMeasurementSchema.getMeasurementNameList(schemaList), - IMeasurementSchema.getDataTypeList(schemaList), - columnTypes, - 15); - - tablet.addTimestamp(0, 0); - tablet.addTimestamp(1, 1); - tablet.addValue(0, 0, "d1"); - tablet.addValue(1, 0, "d1"); - // the field in the first row is null - tablet.addValue("m1", 1, genValue(from, 1)); - if (to.isCompatible(from)) { - // can cast, insert and check the result - session.insert(tablet); - // time, tag1, m1 - SessionDataSet dataSet = - session.executeQueryStatement("select * from table" + testNum + " order by time"); + // create a column with type of "to" + session.executeNonQueryStatement( + "CREATE TABLE " + tableName + " (tag1 string tag," + "m1 " + to.toString() + " field)"); + // insert a tablet with type of "from" + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", from)); + final List columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD); + Tablet tablet = + new Tablet( + tableName, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + tablet.addTimestamp(0, 0); + tablet.addTimestamp(1, 1); + tablet.addValue(0, 0, "d1"); + tablet.addValue(1, 0, "d1"); + // the field in the first row is null + tablet.addValue("m1", 1, genValue(from, 1)); + if (to.isCompatible(from)) { + // can cast, insert and check the result + session.insert(tablet); + // time, tag1, m1 + try (SessionDataSet dataSet = + session.executeQueryStatement("select * from " + tableName + " order by time")) { RowRecord rec = dataSet.next(); assertEquals(0, rec.getFields().get(0).getLongV()); assertEquals("d1", rec.getFields().get(1).toString()); @@ -1521,62 +1513,46 @@ private void testOneCastWithTablet( assertEquals("d1", rec.getFields().get(1).toString()); switch (to) { - case BLOB: - assertEquals(genValue(to, 1), rec.getFields().get(2).getBinaryV()); - break; - case DATE: - assertEquals(genValue(to, 1), rec.getFields().get(2).getDateV()); - break; - case FLOAT: - assertEquals(genValue(to, 1), rec.getFields().get(2).getFloatV()); - break; - case DOUBLE: - assertEquals(genValue(to, 1), rec.getFields().get(2).getDoubleV()); - break; - case STRING: - case TEXT: - if (from == TSDataType.DATE) { - assertEquals( - new Binary(genValue(from, 1).toString(), StandardCharsets.UTF_8), - rec.getFields().get(2).getBinaryV()); - } else { + case BLOB -> assertEquals(genValue(to, 1), rec.getFields().get(2).getBinaryV()); + case DATE -> assertEquals(genValue(to, 1), rec.getFields().get(2).getDateV()); + case FLOAT -> assertEquals(genValue(to, 1), rec.getFields().get(2).getFloatV()); + case DOUBLE -> assertEquals(genValue(to, 1), rec.getFields().get(2).getDoubleV()); + case STRING, TEXT -> assertEquals( new Binary(genValue(from, 1).toString(), StandardCharsets.UTF_8), rec.getFields().get(2).getBinaryV()); - } - break; - default: - assertEquals(String.valueOf(genValue(from, 1)), rec.getFields().get(2).toString()); + case BOOLEAN, INT32, INT64, TIMESTAMP -> + assertEquals(String.valueOf(genValue(from, 1)), rec.getFields().get(2).toString()); + case VECTOR, UNKNOWN, OBJECT -> + throw new IllegalArgumentException("Unsupported data type: " + to); } assertFalse(dataSet.hasNext()); + } + } else { + if (partialInsert) { + // cannot cast, but partial insert + try { + session.insert(tablet); + fail("Exception expected: from=" + from + ", to=" + to); + } catch (StatementExecutionException e) { + assertEquals( + "507: No column other than Time present, please check the request", e.getMessage()); + } + // time, tag1, m1 + try (SessionDataSet dataSet = + session.executeQueryStatement("select * from " + tableName + " order by time")) { + assertFalse(dataSet.hasNext()); + } } else { - if (partialInsert) { - // cannot cast, but partial insert - try { - session.insert(tablet); - fail("Exception expected: from=" + from + ", to=" + to); - } catch (StatementExecutionException e) { - assertEquals( - "507: No column other than Time present, please check the request", e.getMessage()); - } - // time, tag1, m1 - try (SessionDataSet dataSet = - session.executeQueryStatement("select * from table" + testNum + " order by time")) { - assertFalse(dataSet.hasNext()); - } - } else { - // cannot cast, expect an exception - try { - session.insert(tablet); - fail("Exception expected"); - } catch (StatementExecutionException e) { - assertEquals( - "614: Incompatible data type of column m1: " + from + "/" + to, e.getMessage()); - } + // cannot cast, expect an exception + try { + session.insert(tablet); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals( + "614: Incompatible data type of column m1: " + from + "/" + to, e.getMessage()); } } - - session.executeNonQueryStatement("DROP TABLE table" + testNum); } } @@ -1697,25 +1673,40 @@ public static Object genValue(TSDataType dataType, int i) { } } - @Test - public void insertRelationalTabletWithAutoCastTest() + private void testAllCastsWithTablet(int testNum) throws IoTDBConnectionException, StatementExecutionException { - int testNum = 14; Set dataTypes = TSDataTypeTestUtils.getSupportedTypes(); - try { - for (TSDataType from : dataTypes) { - for (TSDataType to : dataTypes) { - System.out.println("from: " + from + ", to: " + to); - testOneCastWithTablet(from, to, testNum, false); - System.out.println("partial insert"); - testOneCastWithTablet(from, to, testNum, true); + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE db1"); + // Keep the complete matrix, but change cluster-wide configuration only once per mode. + for (boolean partialInsert : new boolean[] {false, true}) { + session.executeNonQueryStatement( + "SET CONFIGURATION enable_partial_insert='" + partialInsert + "'"); + for (TSDataType from : dataTypes) { + for (TSDataType to : dataTypes) { + // Avoid waiting for asynchronous deletion before reusing a table name. + String tableName = + ("table" + testNum + "_" + partialInsert + "_" + from + "_" + to) + .toLowerCase(Locale.ROOT); + testOneCastWithTablet(session, from, to, tableName, partialInsert); + } } } } finally { + // Configuration is global to this shared cluster, including when an assertion fails. try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + // Tree SQL requires quoted configuration keys and values, unlike Table SQL above. session.executeNonQueryStatement("SET CONFIGURATION \"enable_partial_insert\"=\"true\""); } } + // Per-method teardown removes the databases and every matrix table in one operation. + } + + // Check every supported type pair, null rows and both partial-insert modes through tablets. + @Test + public void insertRelationalTabletWithAutoCastTest() + throws IoTDBConnectionException, StatementExecutionException { + testAllCastsWithTablet(14); } @Test @@ -1791,24 +1782,11 @@ public void dropTableOfTheSameNameTest() } } + // Preserve this regression matrix of tablet casts, null rows and both partial-insert modes. @Test public void insertRelationalRowWithAutoCastTest() throws IoTDBConnectionException, StatementExecutionException { - int testNum = 17; - Set dataTypes = TSDataTypeTestUtils.getSupportedTypes(); - - for (TSDataType from : dataTypes) { - for (TSDataType to : dataTypes) { - System.out.println("from: " + from + ", to: " + to); - testOneCastWithTablet(from, to, testNum, false); - System.out.println("partial insert"); - testOneCastWithTablet(from, to, testNum, true); - } - } - - try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - session.executeNonQueryStatement("SET CONFIGURATION \"enable_partial_insert\"=\"true\""); - } + testAllCastsWithTablet(17); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java index b3014ff56ee62..d6e0af9467be9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.relational.it.session; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; @@ -30,12 +31,18 @@ import org.apache.tsfile.read.common.RowRecord; import org.junit.After; -import org.junit.Before; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showTablesColumnHeaders; +import static org.apache.iotdb.itbase.env.BaseEnv.TABLE_SQL_DIALECT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -43,13 +50,23 @@ @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) public class IoTDBTableModelSessionIT { - @Before - public void setUp() throws Exception { + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); } @After - public void tearDown() throws Exception { + public void tearDown() throws SQLException { + // Each new session must still open before its configured database is created. + try (final Connection connection = EnvFactory.getEnv().getConnection(TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + TestUtils.dropDatabaseWithRetry(statement, "test1"); + TestUtils.dropDatabaseWithRetry(statement, "test2"); + } + } + + @AfterClass + public static void tearDownClass() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBConnectionsIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBConnectionsIT.java index 90795d969ffcd..430bec98bd45c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBConnectionsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBConnectionsIT.java @@ -67,7 +67,7 @@ public class IoTDBConnectionsIT { public static void setUp() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig(); EnvFactory.getEnv().initClusterEnvironment(1, 2); - createUser("test", "test123123456"); + createUser("test", "IoTDB!Test2024"); try (Connection connection = EnvFactory.getEnv().getTableConnection(); Statement statement = connection.createStatement()) { // Get all data nodes @@ -239,7 +239,8 @@ public void testSameDataNodeGetConnections() { } } - // Validate normal test case when close one datanode. + // Stopping one of two DataNodes removes its connection rows, and restarting it restores its + // Running status and RPC connectivity. @Test public void testClosedDataNodeGetConnections() throws Exception { if (allDataNodeId.size() <= 1) { @@ -277,25 +278,9 @@ public void testClosedDataNodeGetConnections() throws Exception { try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - // Wait for shutdown check - boolean isShutdownDetected = false; - for (int retry = 0; retry < 60; retry++) { - TShowDataNodesResp showDataNodesResp = client.showDataNodes(); - for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) { - // A gracefully stopped DataNode is reported as Stopped by its shutdown hook; if - // the report fails it becomes Unknown instead - if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus()) - || NodeStatus.Stopped.getStatus().equals(dataNodeInfo.getStatus())) { - isShutdownDetected = true; - break; - } - } - if (isShutdownDetected) { - break; - } - TimeUnit.SECONDS.sleep(1); - } - Assert.assertTrue(isShutdownDetected); + // A graceful shutdown reports Stopped; if its shutdown report fails, ConfigNode reports + // Unknown. + awaitDataNodeStatus(client, closedDataNodeId, NodeStatus.Stopped, NodeStatus.Unknown); } int activeDataNodeId = (int) allDataNodeId.toArray()[1]; @@ -327,26 +312,7 @@ public void testClosedDataNodeGetConnections() throws Exception { closedDataNode.start(); try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - // Wait for restart check - boolean isRestartDetected = false; - for (int retry = 0; retry < 60; retry++) { - TShowDataNodesResp showDataNodesResp = client.showDataNodes(); - boolean containDown = false; - for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) { - // The restarted DataNode keeps Stopped until its first heartbeat revives it - if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus()) - || NodeStatus.Stopped.getStatus().equals(dataNodeInfo.getStatus())) { - containDown = true; - break; - } - } - if (!containDown) { - isRestartDetected = true; - break; - } - TimeUnit.SECONDS.sleep(1); - } - Assert.assertTrue(isRestartDetected); + awaitDataNodeStatus(client, closedDataNodeId, NodeStatus.Running); } // The ConfigNode may report the restarted DataNode as Running before its client RPC service is @@ -370,10 +336,34 @@ public void testClosedDataNodeGetConnections() throws Exception { }); } + // Frequent bounded polling avoids a full one-second delay after quick heartbeat transitions. + private static void awaitDataNodeStatus( + SyncConfigNodeIServiceClient client, int dataNodeId, NodeStatus... expectedStatuses) { + Awaitility.await() + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .atMost(60, TimeUnit.SECONDS) + .until( + () -> { + TShowDataNodesResp showDataNodesResp = client.showDataNodes(); + for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) { + if (dataNodeInfo.getDataNodeId() == dataNodeId) { + for (NodeStatus expectedStatus : expectedStatuses) { + if (expectedStatus.getStatus().equals(dataNodeInfo.getStatus())) { + return true; + } + } + } + } + return false; + }); + } + + // A valid ordinary user without SYSTEM privilege must be denied the connections catalog. @Test public void testNoAuthUserGetConnections() { try (Connection connection = - EnvFactory.getEnv().getConnection("test", "test123123456", TABLE_SQL_DIALECT); + EnvFactory.getEnv().getConnection("test", "IoTDB!Test2024", TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("USE information_schema"); ResultSet resultSet = statement.executeQuery("SELECT * FROM connections"); diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionAlignedInsertIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionAlignedInsertIT.java index 2340cf3f326b2..86330cf82396b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionAlignedInsertIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionAlignedInsertIT.java @@ -32,14 +32,16 @@ import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; -import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -48,42 +50,57 @@ @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBSessionAlignedInsertIT { - private static final String ROOT_SG1_D1_VECTOR1 = "root.sg_1.d1.vector"; - private static final String ROOT_SG1_D1 = "root.sg_1.d1"; - private static final String ROOT_SG1_D2 = "root.sg_1.d2"; private static final double DELTA_DOUBLE = 1e-7d; - @Before - public void setUp() throws Exception { + private static final AtomicInteger NEXT_DATABASE_ID = new AtomicInteger(); + + private String database; + private String alignedDevice; + private String device1; + private String device2; + + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setMaxDegreeOfIndexNode(3); EnvFactory.getEnv().initClusterEnvironment(); } - @After - public void tearDown() throws Exception { + @Before + public void setUp() { + // Keep each method's schema and data independent while sharing the same cluster. + database = "root.aligned_insert_" + NEXT_DATABASE_ID.incrementAndGet(); + alignedDevice = database + ".d1.vector"; + device1 = database + ".d1"; + device2 = database + ".d2"; + } + + @AfterClass + public static void tearDownClass() { + // All method-specific databases are bounded by this class and removed with the cluster. EnvFactory.getEnv().cleanClusterEnvironment(); } + // Insert and flush 100 aligned tablet rows, then verify both typed columns and timestamps. @Test public void insertAlignedTabletTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { insertTabletWithAlignedTimeseriesMethod(session); session.executeNonQueryStatement("flush"); - SessionDataSet dataSet = session.executeQueryStatement("select * from root.sg_1.d1.vector"); - assertEquals(dataSet.getColumnNames().size(), 3); - assertEquals(dataSet.getColumnNames().get(0), "Time"); - assertEquals(dataSet.getColumnNames().get(1), ROOT_SG1_D1_VECTOR1 + ".s1"); - assertEquals(dataSet.getColumnNames().get(2), ROOT_SG1_D1_VECTOR1 + ".s2"); - long time = 0; - while (dataSet.hasNext()) { - RowRecord rowRecord = dataSet.next(); - assertEquals(time * 10 + 1, rowRecord.getFields().get(0).getLongV()); - assertEquals(time * 10 + 2, rowRecord.getFields().get(1).getIntV()); - time += 1; + try (SessionDataSet dataSet = + session.executeQueryStatement("select * from " + database + ".d1.vector")) { + assertEquals(dataSet.getColumnNames().size(), 3); + assertEquals(dataSet.getColumnNames().get(0), "Time"); + assertEquals(dataSet.getColumnNames().get(1), alignedDevice + ".s1"); + assertEquals(dataSet.getColumnNames().get(2), alignedDevice + ".s2"); + long time = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + assertEquals(time * 10 + 1, rowRecord.getFields().get(0).getLongV()); + assertEquals(time * 10 + 2, rowRecord.getFields().get(1).getIntV()); + time += 1; + } + assertEquals(100, time); } - assertEquals(100, time); - - dataSet.closeOperationHandle(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -113,174 +130,181 @@ private void insertAlignedRecord(ISession session, String deviceId) } } + // Insert 100 typed aligned records and verify s2 after flush. @Test public void insertAlignedRecordTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - insertAlignedRecord(session, ROOT_SG1_D1_VECTOR1); + insertAlignedRecord(session, alignedDevice); session.executeNonQueryStatement("flush"); - SessionDataSet dataSet = session.executeQueryStatement("select s2 from root.sg_1.d1.vector"); - assertEquals(2, dataSet.getColumnNames().size()); - assertEquals("Time", dataSet.getColumnNames().get(0)); - assertEquals(dataSet.getColumnNames().get(1), ROOT_SG1_D1_VECTOR1 + ".s2"); - long time = 0; - while (dataSet.hasNext()) { - RowRecord rowRecord = dataSet.next(); - assertEquals(time * 10 + 3, rowRecord.getFields().get(0).getLongV()); - time += 1; + try (SessionDataSet dataSet = + session.executeQueryStatement("select s2 from " + database + ".d1.vector")) { + assertEquals(2, dataSet.getColumnNames().size()); + assertEquals("Time", dataSet.getColumnNames().get(0)); + assertEquals(dataSet.getColumnNames().get(1), alignedDevice + ".s2"); + long time = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + assertEquals(time * 10 + 3, rowRecord.getFields().get(0).getLongV()); + time += 1; + } + assertEquals(100, time); } - assertEquals(100, time); - - dataSet.closeOperationHandle(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Insert 100 aligned string records and verify their numeric s2 values after flush. @Test public void insertAlignedStringRecordTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - insertAlignedStringRecord(session, ROOT_SG1_D1_VECTOR1); + insertAlignedStringRecord(session, alignedDevice); session.executeNonQueryStatement("flush"); - SessionDataSet dataSet = session.executeQueryStatement("select s2 from root.sg_1.d1.vector"); - assertEquals(2, dataSet.getColumnNames().size()); - assertEquals("Time", dataSet.getColumnNames().get(0)); - assertEquals(ROOT_SG1_D1_VECTOR1 + ".s2", dataSet.getColumnNames().get(1)); - long time = 0; - while (dataSet.hasNext()) { - RowRecord rowRecord = dataSet.next(); - assertEquals(time * 10 + 3, rowRecord.getFields().get(0).getDoubleV(), DELTA_DOUBLE); - time += 1; + try (SessionDataSet dataSet = + session.executeQueryStatement("select s2 from " + database + ".d1.vector")) { + assertEquals(2, dataSet.getColumnNames().size()); + assertEquals("Time", dataSet.getColumnNames().get(0)); + assertEquals(alignedDevice + ".s2", dataSet.getColumnNames().get(1)); + long time = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + assertEquals(time * 10 + 3, rowRecord.getFields().get(0).getDoubleV(), DELTA_DOUBLE); + time += 1; + } + assertEquals(100, time); } - assertEquals(100, time); - - dataSet.closeOperationHandle(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Batch aligned string records and verify all 100 converted s2 values after flush. @Test public void insertAlignedStringRecordsTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - insertAlignedStringRecords(session, ROOT_SG1_D1_VECTOR1); + insertAlignedStringRecords(session, alignedDevice); session.executeNonQueryStatement("flush"); - SessionDataSet dataSet = session.executeQueryStatement("select s2 from root.sg_1.d1.vector"); - assertEquals(2, dataSet.getColumnNames().size()); - assertEquals("Time", dataSet.getColumnNames().get(0)); - assertEquals(dataSet.getColumnNames().get(1), ROOT_SG1_D1_VECTOR1 + ".s2"); - long time = 0; - while (dataSet.hasNext()) { - RowRecord rowRecord = dataSet.next(); - assertEquals(time * 10 + 3, rowRecord.getFields().get(0).getDoubleV(), DELTA_DOUBLE); - time += 1; + try (SessionDataSet dataSet = + session.executeQueryStatement("select s2 from " + database + ".d1.vector")) { + assertEquals(2, dataSet.getColumnNames().size()); + assertEquals("Time", dataSet.getColumnNames().get(0)); + assertEquals(dataSet.getColumnNames().get(1), alignedDevice + ".s2"); + long time = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + assertEquals(time * 10 + 3, rowRecord.getFields().get(0).getDoubleV(), DELTA_DOUBLE); + time += 1; + } + assertEquals(100, time); } - assertEquals(100, time); - - dataSet.closeOperationHandle(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Batch typed aligned records and verify all 100 s2 values after flush. @Test public void insertAlignedRecordsTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - insertAlignedRecords(session, ROOT_SG1_D1_VECTOR1); + insertAlignedRecords(session, alignedDevice); session.executeNonQueryStatement("flush"); - SessionDataSet dataSet = session.executeQueryStatement("select s2 from root.sg_1.d1.vector"); - assertEquals(2, dataSet.getColumnNames().size()); - assertEquals("Time", dataSet.getColumnNames().get(0)); - assertEquals(dataSet.getColumnNames().get(1), ROOT_SG1_D1_VECTOR1 + ".s2"); - long time = 0; - while (dataSet.hasNext()) { - RowRecord rowRecord = dataSet.next(); - assertEquals(time * 10 + 3, rowRecord.getFields().get(0).getLongV()); - time += 1; + try (SessionDataSet dataSet = + session.executeQueryStatement("select s2 from " + database + ".d1.vector")) { + assertEquals(2, dataSet.getColumnNames().size()); + assertEquals("Time", dataSet.getColumnNames().get(0)); + assertEquals(dataSet.getColumnNames().get(1), alignedDevice + ".s2"); + long time = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + assertEquals(time * 10 + 3, rowRecord.getFields().get(0).getLongV()); + time += 1; + } + assertEquals(100, time); } - assertEquals(100, time); - - dataSet.closeOperationHandle(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Batch aligned records for one device and verify all 100 s2 values after flush. @Test public void insertAlignedRecordsOfOneDeviceTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - insertAlignedRecordsOfOneDevice(session, ROOT_SG1_D1_VECTOR1); + insertAlignedRecordsOfOneDevice(session, alignedDevice); session.executeNonQueryStatement("flush"); - SessionDataSet dataSet = session.executeQueryStatement("select s2 from root.sg_1.d1.vector"); - assertEquals(2, dataSet.getColumnNames().size()); - assertEquals("Time", dataSet.getColumnNames().get(0)); - assertEquals(dataSet.getColumnNames().get(1), ROOT_SG1_D1_VECTOR1 + ".s2"); - long time = 0; - while (dataSet.hasNext()) { - RowRecord rowRecord = dataSet.next(); - assertEquals(time * 10 + 3, rowRecord.getFields().get(0).getLongV()); - time += 1; + try (SessionDataSet dataSet = + session.executeQueryStatement("select s2 from " + database + ".d1.vector")) { + assertEquals(2, dataSet.getColumnNames().size()); + assertEquals("Time", dataSet.getColumnNames().get(0)); + assertEquals(dataSet.getColumnNames().get(1), alignedDevice + ".s2"); + long time = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + assertEquals(time * 10 + 3, rowRecord.getFields().get(0).getLongV()); + time += 1; + } + assertEquals(100, time); } - assertEquals(100, time); - - dataSet.closeOperationHandle(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Mix aligned and non-aligned devices, then verify the aligned s2 projection after flush. @Test public void nonAlignedSingleSelectTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - insertNonAlignedRecord(session, ROOT_SG1_D1); + insertNonAlignedRecord(session, device1); insertTabletWithAlignedTimeseriesMethod(session); - insertNonAlignedRecord(session, ROOT_SG1_D2); + insertNonAlignedRecord(session, device2); session.executeNonQueryStatement("flush"); - SessionDataSet dataSet = session.executeQueryStatement("select s2 from root.sg_1.d1.vector"); - assertEquals(2, dataSet.getColumnNames().size()); - assertEquals("Time", dataSet.getColumnNames().get(0)); - assertEquals(dataSet.getColumnNames().get(1), ROOT_SG1_D1_VECTOR1 + ".s2"); - long time = 0; - while (dataSet.hasNext()) { - RowRecord rowRecord = dataSet.next(); - assertEquals(time * 10 + 2, rowRecord.getFields().get(0).getIntV()); - time += 1; + try (SessionDataSet dataSet = + session.executeQueryStatement("select s2 from " + database + ".d1.vector")) { + assertEquals(2, dataSet.getColumnNames().size()); + assertEquals("Time", dataSet.getColumnNames().get(0)); + assertEquals(dataSet.getColumnNames().get(1), alignedDevice + ".s2"); + long time = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + assertEquals(time * 10 + 2, rowRecord.getFields().get(0).getIntV()); + time += 1; + } + assertEquals(100, time); } - assertEquals(100, time); - - dataSet.closeOperationHandle(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Mix aligned and non-aligned devices, then verify both aligned vector columns after flush. @Test public void nonAlignedVectorSelectTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - insertNonAlignedRecord(session, ROOT_SG1_D1); + insertNonAlignedRecord(session, device1); insertTabletWithAlignedTimeseriesMethod(session); - insertNonAlignedRecord(session, ROOT_SG1_D2); + insertNonAlignedRecord(session, device2); session.executeNonQueryStatement("flush"); - SessionDataSet dataSet = session.executeQueryStatement("select * from root.sg_1.d1.vector"); - assertEquals(3, dataSet.getColumnNames().size()); - assertEquals("Time", dataSet.getColumnNames().get(0)); - assertEquals(ROOT_SG1_D1_VECTOR1 + ".s1", dataSet.getColumnNames().get(1)); - assertEquals(ROOT_SG1_D1_VECTOR1 + ".s2", dataSet.getColumnNames().get(2)); - long time = 0; - while (dataSet.hasNext()) { - RowRecord rowRecord = dataSet.next(); - assertEquals(time * 10 + 1, rowRecord.getFields().get(0).getLongV()); - assertEquals(time * 10 + 2, rowRecord.getFields().get(1).getIntV()); - time += 1; + try (SessionDataSet dataSet = + session.executeQueryStatement("select * from " + database + ".d1.vector")) { + assertEquals(3, dataSet.getColumnNames().size()); + assertEquals("Time", dataSet.getColumnNames().get(0)); + assertEquals(alignedDevice + ".s1", dataSet.getColumnNames().get(1)); + assertEquals(alignedDevice + ".s2", dataSet.getColumnNames().get(2)); + long time = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + assertEquals(time * 10 + 1, rowRecord.getFields().get(0).getLongV()); + assertEquals(time * 10 + 2, rowRecord.getFields().get(1).getIntV()); + time += 1; + } + assertEquals(100, time); } - assertEquals(100, time); - - dataSet.closeOperationHandle(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -296,7 +320,7 @@ private void insertTabletWithAlignedTimeseriesMethod(ISession session) schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT32)); - Tablet tablet = new Tablet(ROOT_SG1_D1_VECTOR1, schemaList); + Tablet tablet = new Tablet(alignedDevice, schemaList); long timestamp = 0; for (long row = 0; row < 100; row++) { diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionComplexIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionComplexIT.java index 3643851206a08..0f7f73f8b232f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionComplexIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionComplexIT.java @@ -38,8 +38,10 @@ import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -67,15 +69,38 @@ @RunWith(IoTDBTestRunner.class) public class IoTDBSessionComplexIT { - @Before - public void setUp() throws Exception { + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setEnforceStrongPassword(false); EnvFactory.getEnv().initClusterEnvironment(); - createUser("test", "test123123456"); + } + + @Before + public void setUp() throws Exception { + createUser("test", "IoTDB!Test2024"); } @After public void tearDown() throws Exception { + // Keep the shared cluster isolated by removing data and users created by this test method. + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + // Restore the dynamic setting changed by insertNotAutoCreateSchemaTest for the next method. + session.executeNonQueryStatement("SET CONFIGURATION 'enable_auto_create_schema'='true'"); + for (String database : Arrays.asList("root.sg1", "root.sg2", "root.sg3")) { + boolean exists; + try (SessionDataSet dataSet = session.executeQueryStatement("SHOW DATABASES " + database)) { + exists = dataSet.hasNext(); + } + if (exists) { + session.deleteDatabase(database); + } + } + } + executeNonQuery("DROP USER test"); + } + + @AfterClass + public static void tearDownClass() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } @@ -163,6 +188,7 @@ private void queryByDevice(ISession session, String deviceId) sessionDataSet.closeOperationHandle(); } + // Object inserts require all data/schema/system grants before the ordinary user can write. @Test @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertByObjectTest() { @@ -183,7 +209,7 @@ public void insertByObjectTest() { // auth test try (ISession authSession = - EnvFactory.getEnv().getSessionConnection("test", "test123123456")) { + EnvFactory.getEnv().getSessionConnection("test", "IoTDB!Test2024")) { grantUserSeriesPrivilege("test", PrivilegeType.WRITE_DATA, "root.sg1.d1.s1"); grantUserSeriesPrivilege("test", PrivilegeType.WRITE_DATA, "root.sg1.d1.s2"); try { @@ -599,11 +625,13 @@ public void insertNotAutoCreateSchemaTest() { } } + // Record and Tablet APIs enforce missing privileges, then accept writes after all grants. @Test @Category({LocalStandaloneIT.class, ClusterIT.class}) public void testAuth() { // auth test - try (ISession authSession = EnvFactory.getEnv().getSessionConnection("test", "test123123456")) { + try (ISession authSession = + EnvFactory.getEnv().getSessionConnection("test", "IoTDB!Test2024")) { grantUserSeriesPrivilege("test", PrivilegeType.WRITE_DATA, "root.sg1.d1.s1"); grantUserSeriesPrivilege("test", PrivilegeType.WRITE_DATA, "root.sg1.d1.s2"); grantUserSeriesPrivilege("test", PrivilegeType.WRITE_DATA, "root.sg1.d2.**"); diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java index 5fdf733578f75..dd5e9ddc58730 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java @@ -35,19 +35,24 @@ import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.MeasurementSchema; -import org.junit.After; +import org.awaitility.Awaitility; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -57,63 +62,81 @@ @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBSessionInsertNullIT { - private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSessionInsertNullIT.class); - private final int retry = 30; + private static final AtomicInteger NEXT_DATABASE_ID = new AtomicInteger(); - @Before - public void setUp() throws Exception { + private String database; + + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); } - @After - public void tearDown() throws Exception { + @Before + public void setUp() { + // Keep each method's schema and data independent while sharing the same cluster. + database = "root.insert_null_" + NEXT_DATABASE_ID.incrementAndGet(); + } + + @AfterClass + public static void tearDownClass() { + // All method-specific databases are bounded by this class and removed with the cluster. EnvFactory.getEnv().cleanClusterEnvironment(); } private void prepareData(ISession session) throws IoTDBConnectionException, StatementExecutionException { - session.setStorageGroup("root.sg1"); + session.setStorageGroup(database); session.createTimeseries( - "root.sg1.clsu.d1.s1", TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY); + database + ".clsu.d1.s1", TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY); session.createTimeseries( - "root.sg1.clsu.d1.s2", TSDataType.INT32, TSEncoding.PLAIN, CompressionType.SNAPPY); + database + ".clsu.d1.s2", TSDataType.INT32, TSEncoding.PLAIN, CompressionType.SNAPPY); session.createTimeseries( - "root.sg1.clsu.d1.s3", TSDataType.INT64, TSEncoding.PLAIN, CompressionType.SNAPPY); + database + ".clsu.d1.s3", TSDataType.INT64, TSEncoding.PLAIN, CompressionType.SNAPPY); session.createTimeseries( - "root.sg1.clsu.d1.s4", TSDataType.FLOAT, TSEncoding.PLAIN, CompressionType.SNAPPY); + database + ".clsu.d1.s4", TSDataType.FLOAT, TSEncoding.PLAIN, CompressionType.SNAPPY); session.createTimeseries( - "root.sg1.clsu.d1.s5", TSDataType.DOUBLE, TSEncoding.PLAIN, CompressionType.SNAPPY); + database + ".clsu.d1.s5", TSDataType.DOUBLE, TSEncoding.PLAIN, CompressionType.SNAPPY); session.createTimeseries( - "root.sg1.clsu.d1.s6", TSDataType.TEXT, TSEncoding.PLAIN, CompressionType.SNAPPY); + database + ".clsu.d1.s6", TSDataType.TEXT, TSEncoding.PLAIN, CompressionType.SNAPPY); session.createTimeseries( - "root.sg1.clsu.d2.s1", TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY); + database + ".clsu.d2.s1", TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY); } private long queryCountRecords(ISession session, String sql) throws StatementExecutionException, IoTDBConnectionException { - SessionDataSet dataSetWrapper = session.executeQueryStatement(sql, 60_000); - long count = 0; - while (dataSetWrapper.hasNext()) { - RowRecord record = dataSetWrapper.next(); - Field field = record.getFields().get(0); - switch (field.getDataType()) { - case INT32: - count = field.getIntV(); - break; - case INT64: - count = field.getLongV(); - break; + try (SessionDataSet dataSetWrapper = session.executeQueryStatement(sql, 60_000)) { + long count = 0; + while (dataSetWrapper.hasNext()) { + RowRecord record = dataSetWrapper.next(); + Field field = record.getFields().get(0); + switch (field.getDataType()) { + case INT32 -> count = field.getIntV(); + case INT64 -> count = field.getLongV(); + case BOOLEAN, + FLOAT, + DOUBLE, + TEXT, + VECTOR, + UNKNOWN, + TIMESTAMP, + DATE, + BLOB, + STRING, + OBJECT -> + throw new AssertionError("Unexpected count type: " + field.getDataType()); + } } + return count; } - return count; } + // Mix non-null and null typed/string records; only three non-null s1 values should remain. @Test public void insertRecordNullTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { prepareData(session); - String deviceId = "root.sg1.clsu.d1"; + String deviceId = database + ".clsu.d1"; session.insertRecord(deviceId, 100, Arrays.asList("s1"), Arrays.asList("true")); List t = new ArrayList<>(); t.add(null); @@ -144,12 +167,13 @@ public void insertRecordNullTest() { } } + // Mix aligned typed/string records with nulls; eventually count exactly three s1 values. @Test public void insertAlignedRecordNullTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { prepareData(session); - String deviceId = "root.sg1.clsu.aligned_d1"; + String deviceId = database + ".clsu.aligned_d1"; session.insertAlignedRecord(deviceId, 100, Arrays.asList("s1"), Arrays.asList("true")); List t = new ArrayList<>(); t.add(null); @@ -172,29 +196,31 @@ public void insertAlignedRecordNullTest() { Arrays.asList("s1", "s2"), Arrays.asList(TSDataType.BOOLEAN, TSDataType.INT32), Arrays.asList(null, null)); - for (int i = 0; i < retry; i++) { - try { - long nums = queryCountRecords(session, "select count(s1) from " + deviceId); - assertEquals(3, nums); - return; - } catch (Exception e) { - LOGGER.info("query records failed, retry: " + i); - Thread.sleep(1000); - } - } + Awaitility.await() + .pollInSameThread() + .pollDelay(Duration.ZERO) + .pollInterval(Duration.ofMillis(100)) + .atMost(Duration.ofSeconds(30)) + .ignoreExceptionsMatching(e -> e instanceof IoTDBConnectionException) + .untilAsserted( + () -> { + long nums = queryCountRecords(session, "select count(s1) from " + deviceId); + assertEquals(3, nums); + }); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Batch typed/string records with nulls and verify each device retains three values. @Test public void insertRecordsNullTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { prepareData(session); - String deviceId1 = "root.sg1.clsu.d2"; - String deviceId2 = "root.sg1.clsu.d3"; + String deviceId1 = database + ".clsu.d2"; + String deviceId2 = database + ".clsu.d3"; session.insertRecords( Arrays.asList(deviceId1, deviceId2), Arrays.asList(300L, 300L), @@ -222,31 +248,32 @@ public void insertRecordsNullTest() { Arrays.asList(TSDataType.BOOLEAN, TSDataType.INT32)), Arrays.asList(Arrays.asList(true, null), Arrays.asList(null, null))); - for (int i = 0; i < retry; i++) { - try { - long nums = queryCountRecords(session, "select count(s1) from " + deviceId1); - assertEquals(3, nums); - nums = queryCountRecords(session, "select count(s2) from " + deviceId2); - assertEquals(3, nums); - return; - } catch (Exception e) { - LOGGER.info("read records failed, retry: " + i); - Thread.sleep(1000); - } - } - + Awaitility.await() + .pollInSameThread() + .pollDelay(Duration.ZERO) + .pollInterval(Duration.ofMillis(100)) + .atMost(Duration.ofSeconds(30)) + .ignoreExceptionsMatching(e -> e instanceof IoTDBConnectionException) + .untilAsserted( + () -> { + long nums = queryCountRecords(session, "select count(s1) from " + deviceId1); + assertEquals(3, nums); + nums = queryCountRecords(session, "select count(s2) from " + deviceId2); + assertEquals(3, nums); + }); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Batch aligned records with nulls and verify the expected per-device column counts. @Test public void insertAlignedRecordsNullTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { prepareData(session); - String deviceId1 = "root.sg1.clsu.aligned_d2"; - String deviceId2 = "root.sg1.clsu.aligned_d3"; + String deviceId1 = database + ".clsu.aligned_d2"; + String deviceId2 = database + ".clsu.aligned_d3"; session.insertAlignedRecords( Arrays.asList(deviceId1, deviceId2), Arrays.asList(300L, 300L), @@ -284,11 +311,12 @@ public void insertAlignedRecordsNullTest() { } } + // Mix typed/string batches for one device; nulls must leave exactly six s1 values. @Test public void insertRecordsOfOneDeviceNullTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { prepareData(session); - String deviceId1 = "root.sg1.clsu.InsertRecordsOfOneDevice"; + String deviceId1 = database + ".clsu.InsertRecordsOfOneDevice"; session.insertRecordsOfOneDevice( deviceId1, Arrays.asList(300L, 301L), @@ -323,11 +351,12 @@ public void insertRecordsOfOneDeviceNullTest() { } } + // Mix aligned batches for one device; nulls must leave exactly six s1 values. @Test public void insertAlignedRecordsOfOneDeviceNullTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { prepareData(session); - String deviceId1 = "root.sg1.clsu.InsertAlignedRecordsOfOneDevice"; + String deviceId1 = database + ".clsu.InsertAlignedRecordsOfOneDevice"; session.insertAlignedRecordsOfOneDevice( deviceId1, Arrays.asList(300L, 301L), @@ -362,12 +391,13 @@ public void insertAlignedRecordsOfOneDeviceNullTest() { } } + // Insert an all-null tablet; s1 must stay empty before and after flush. @Test public void insertTabletNullTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { prepareData(session); - String deviceId = "root.sg1.clsu.d1"; + String deviceId = database + ".clsu.d1"; Tablet tablet = new Tablet( deviceId, @@ -396,12 +426,13 @@ public void insertTabletNullTest() { } } + // Flush an all-null aligned tablet; counts stay zero and no broken TsFile is produced. @Test public void insertAlignedTabletNullTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { prepareData(session); - String deviceId = "root.sg1.clsu.aligned_d1"; + String deviceId = database + ".clsu.aligned_d1"; Tablet tablet = new Tablet( deviceId, @@ -425,27 +456,11 @@ public void insertAlignedTabletNullTest() { nums = queryCountRecords(session, "select count(s1) from " + deviceId); assertEquals(0, nums); for (DataNodeWrapper dn : EnvFactory.getEnv().getDataNodeWrapperList()) { - File dir = - new File( - dn.getDataDir() - + File.separator - + "datanode" - + File.separator - + "data" - + File.separator - + "sequence" - + File.separator - + "root.sg1" - + File.separator - + "1" - + File.separator - + "0"); - if (dir.exists() && dir.isDirectory()) { - File[] files = dir.listFiles(); - if (files != null) { - for (File file : files) { - assertFalse(file.getName().endsWith("broken")); - } + Path databaseDir = Paths.get(dn.getDataDir(), "datanode", "data", "sequence", database); + if (Files.isDirectory(databaseDir)) { + // A shared cluster assigns different region IDs; inspect every region of this database. + try (Stream files = Files.walk(databaseDir)) { + assertFalse(files.anyMatch(file -> file.getFileName().toString().endsWith("broken"))); } } } @@ -455,10 +470,11 @@ public void insertAlignedTabletNullTest() { } } + // Reject aligned tablets containing a null measurement name for both value setters. @Test public void insertTabletNullMeasurementTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - String deviceId = "root.sg1.clsu.aligned_d1"; + String deviceId = database + ".clsu.aligned_d1"; Tablet tablet = new Tablet( deviceId, @@ -476,7 +492,7 @@ public void insertTabletNullMeasurementTest() { } try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - String deviceId = "root.sg1.clsu.aligned_d1"; + String deviceId = database + ".clsu.aligned_d1"; Tablet tablet = new Tablet( deviceId, @@ -494,7 +510,7 @@ public void insertTabletNullMeasurementTest() { } try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - String deviceId = "root.sg1.clsu.aligned_d1"; + String deviceId = database + ".clsu.aligned_d1"; Tablet tablet = new Tablet( deviceId, diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSyntaxConventionIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSyntaxConventionIT.java index ac65c6be784e6..e8bab19a2af2f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSyntaxConventionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSyntaxConventionIT.java @@ -36,8 +36,9 @@ import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; -import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -46,6 +47,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -56,25 +58,37 @@ @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBSessionSyntaxConventionIT { - @Before - public void setUp() throws Exception { + private static final AtomicInteger NEXT_DATABASE_ID = new AtomicInteger(); + + private String database; + + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); } - @After - public void tearDown() throws Exception { + @Before + public void setUp() { + // Keep each method's schema and data independent while sharing the same cluster. + database = "root.syntax_" + NEXT_DATABASE_ID.incrementAndGet(); + } + + @AfterClass + public static void tearDownClass() { + // All method-specific databases are bounded by this class and removed with the cluster. EnvFactory.getEnv().cleanClusterEnvironment(); } + // Reject mismatched backticks in timeseries names without creating any schema. @Test public void createTimeSeriesTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - String database = "root.sg"; + String database = this.database; session.setStorageGroup(database); try { session.createTimeseries( - "root.sg.d1.`a.s1", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); + database + ".d1.`a.s1", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); fail(); } catch (Exception e) { assertTrue(e.getMessage().contains("is not a legal path")); @@ -82,14 +96,16 @@ public void createTimeSeriesTest() { try { session.createTimeseries( - "root.sg.d1.a`.s1", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); + database + ".d1.a`.s1", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY); fail(); } catch (Exception e) { assertTrue(e.getMessage().contains("is not a legal path")); } - final SessionDataSet dataSet = session.executeQueryStatement("SHOW TIMESERIES root.sg.**"); - assertFalse(dataSet.hasNext()); + try (SessionDataSet dataSet = + session.executeQueryStatement("SHOW TIMESERIES " + database + ".**")) { + assertFalse(dataSet.hasNext()); + } session.deleteDatabase(database); } catch (Exception e) { @@ -98,10 +114,11 @@ public void createTimeSeriesTest() { } } + // Accept quoted special-character measurement names and verify their schema exists. @Test public void insertTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - String deviceId = "root.sg1.d1"; + String deviceId = database + ".d1"; List measurements = new ArrayList<>(); measurements.add("`\"a“(Φ)”b\"`"); @@ -121,25 +138,26 @@ public void insertTest() { session.insertRecord(deviceId, 1L, measurements, values); - assertTrue(session.checkTimeseriesExists("root.sg1.d1.`\"a“(Φ)”b\"`")); - assertTrue(session.checkTimeseriesExists("root.sg1.d1.`\"a>b\"`")); - assertTrue(session.checkTimeseriesExists("root.sg1.d1.```a.b```")); - assertTrue(session.checkTimeseriesExists("root.sg1.d1.`'a“(Φ)”b'`")); - assertTrue(session.checkTimeseriesExists("root.sg1.d1.`'a>b'`")); - assertTrue(session.checkTimeseriesExists("root.sg1.d1.`a“(Φ)”b`")); - assertTrue(session.checkTimeseriesExists("root.sg1.d1.`a>b`")); - assertTrue(session.checkTimeseriesExists("root.sg1.d1.`\\\"a`")); - assertTrue(session.checkTimeseriesExists("root.sg1.d1.aaa")); + assertTrue(session.checkTimeseriesExists(database + ".d1.`\"a“(Φ)”b\"`")); + assertTrue(session.checkTimeseriesExists(database + ".d1.`\"a>b\"`")); + assertTrue(session.checkTimeseriesExists(database + ".d1.```a.b```")); + assertTrue(session.checkTimeseriesExists(database + ".d1.`'a“(Φ)”b'`")); + assertTrue(session.checkTimeseriesExists(database + ".d1.`'a>b'`")); + assertTrue(session.checkTimeseriesExists(database + ".d1.`a“(Φ)”b`")); + assertTrue(session.checkTimeseriesExists(database + ".d1.`a>b`")); + assertTrue(session.checkTimeseriesExists(database + ".d1.`\\\"a`")); + assertTrue(session.checkTimeseriesExists(database + ".d1.aaa")); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Reject illegal measurement names in single-record insertion without creating schema. @Test public void inserRecordWithIllegalMeasurementTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - String deviceId = "root.sg1.d1"; + String deviceId = database + ".d1"; List measurements = new ArrayList<>(); List values = new ArrayList<>(); measurements.add("a.b"); @@ -210,18 +228,21 @@ public void inserRecordWithIllegalMeasurementTest() { e.printStackTrace(); } - SessionDataSet dataSet = session.executeQueryStatement("show timeseries root"); - assertFalse(dataSet.hasNext()); + try (SessionDataSet dataSet = + session.executeQueryStatement("show timeseries " + database + ".**")) { + assertFalse(dataSet.hasNext()); + } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Reject batched records with illegal measurement names without creating schema. @Test public void insertRecordsWithIllegalMeasurementTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - String deviceId = "root.sg1.d1"; + String deviceId = database + ".d1"; List measurements = new ArrayList<>(); measurements.add("a.b"); measurements.add("111"); @@ -246,17 +267,20 @@ public void insertRecordsWithIllegalMeasurementTest() { e.printStackTrace(); } - SessionDataSet dataSet = session.executeQueryStatement("show timeseries root"); - assertFalse(dataSet.hasNext()); + try (SessionDataSet dataSet = + session.executeQueryStatement("show timeseries " + database + ".**")) { + assertFalse(dataSet.hasNext()); + } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Reject tablet schemas with illegal measurement names without creating schema. @Test public void insertTabletWithIllegalMeasurementTest() { - String deviceId = "root.sg1.d1"; + String deviceId = database + ".d1"; List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("wrong`", TSDataType.INT64, TSEncoding.RLE)); schemaList.add(new MeasurementSchema("s2", TSDataType.DOUBLE, TSEncoding.RLE)); @@ -281,18 +305,21 @@ public void insertTabletWithIllegalMeasurementTest() { e.printStackTrace(); } - SessionDataSet dataSet = session.executeQueryStatement("show timeseries root"); - assertFalse(dataSet.hasNext()); + try (SessionDataSet dataSet = + session.executeQueryStatement("show timeseries " + database + ".**")) { + assertFalse(dataSet.hasNext()); + } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Reject illegal aligned measurement names and leave the database schema empty. @Test public void createAlignedTimeseriesWithIllegalMeasurementTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - String deviceId = "root.sg1.d1"; + String deviceId = database + ".d1"; List measurements = new ArrayList<>(); measurements.add("111"); measurements.add("a.b"); @@ -311,18 +338,21 @@ public void createAlignedTimeseriesWithIllegalMeasurementTest() { } catch (Exception e) { e.printStackTrace(); } - SessionDataSet dataSet = session.executeQueryStatement("show timeseries root"); - assertFalse(dataSet.hasNext()); + try (SessionDataSet dataSet = + session.executeQueryStatement("show timeseries " + database + ".**")) { + assertFalse(dataSet.hasNext()); + } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Reject an illegal aligned measurement alias without creating timeseries. @Test public void createAlignedTimeseriesWithIllegalMeasurementAliasTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - String deviceId = "root.sg1.d1"; + String deviceId = database + ".d1"; List measurements = new ArrayList<>(); measurements.add("s1"); measurements.add("s2"); @@ -351,20 +381,23 @@ public void createAlignedTimeseriesWithIllegalMeasurementAliasTest() { } catch (Exception e) { e.printStackTrace(); } - SessionDataSet dataSet = session.executeQueryStatement("show timeseries root"); - assertFalse(dataSet.hasNext()); + try (SessionDataSet dataSet = + session.executeQueryStatement("show timeseries " + database + ".**")) { + assertFalse(dataSet.hasNext()); + } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Reject an illegal alias in multi-timeseries creation without creating schema. @Test public void createMultiTimeseriesWithIllegalMeasurementAliasTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { List paths = new ArrayList<>(); - paths.add("root.sg1.d1.s1"); - paths.add("root.sg1.d1.s2"); + paths.add(database + ".d1.s1"); + paths.add(database + ".d1.s2"); List tsDataTypes = new ArrayList<>(); tsDataTypes.add(TSDataType.DOUBLE); tsDataTypes.add(TSDataType.DOUBLE); @@ -391,14 +424,17 @@ public void createMultiTimeseriesWithIllegalMeasurementAliasTest() { } catch (StatementExecutionException e) { e.printStackTrace(); } - SessionDataSet dataSet = session.executeQueryStatement("show timeseries root"); - assertFalse(dataSet.hasNext()); + try (SessionDataSet dataSet = + session.executeQueryStatement("show timeseries " + database + ".**")) { + assertFalse(dataSet.hasNext()); + } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Reject template measurements containing illegal names without creating timeseries. @Test public void createSchemaTemplateWithIllegalMeasurementTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { @@ -417,14 +453,17 @@ public void createSchemaTemplateWithIllegalMeasurementTest() { } catch (Exception e) { e.printStackTrace(); } - SessionDataSet dataSet = session.executeQueryStatement("show timeseries root"); - assertFalse(dataSet.hasNext()); + try (SessionDataSet dataSet = + session.executeQueryStatement("show timeseries " + database + ".**")) { + assertFalse(dataSet.hasNext()); + } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } } + // Reject a null template measurement name with the expected metadata error. @Test public void createSchemaTemplateWithIllegalMeasurementNameTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/SessionIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/SessionIT.java index 52638352ee37c..ba2cff7542f40 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/SessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/SessionIT.java @@ -40,8 +40,10 @@ import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -52,6 +54,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -60,20 +63,48 @@ @Category({LocalStandaloneIT.class, ClusterIT.class}) public class SessionIT { - @Before - public void setUp() throws Exception { + private static final AtomicInteger NEXT_DATABASE_ID = new AtomicInteger(); + + private String database; + + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().initClusterEnvironment(); } + @Before + public void setUp() { + database = "root.session_" + NEXT_DATABASE_ID.incrementAndGet(); + } + @After public void tearDown() throws Exception { + dropDatabaseIfExists(database); + } + + @AfterClass + public static void tearDownClass() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + private void dropDatabaseIfExists(String databasePath) throws Exception { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + boolean exists; + try (SessionDataSet dataSet = + session.executeQueryStatement("SHOW DATABASES " + databasePath)) { + exists = dataSet.hasNext(); + } + if (exists) { + session.deleteDatabase(databasePath); + } + } + } + + // A tablet with incompatible value types fails without returning partial rows. @Test public void testInsertByStrAndSelectFailedData() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - String deviceId = "root.sg1.d1"; + String deviceId = database + ".d1"; session.createTimeseries( deviceId + ".s1", TSDataType.INT64, TSEncoding.RLE, CompressionType.UNCOMPRESSED); @@ -90,7 +121,7 @@ public void testInsertByStrAndSelectFailedData() { schemaList.add(new MeasurementSchema("s3", TSDataType.TEXT, TSEncoding.PLAIN)); schemaList.add(new MeasurementSchema("s4", TSDataType.INT64, TSEncoding.PLAIN)); - Tablet tablet = new Tablet("root.sg1.d1", schemaList, 10); + Tablet tablet = new Tablet(database + ".d1", schemaList, 10); for (long time = 0; time < 10; time++) { int row = tablet.getRowSize(); @@ -109,7 +140,7 @@ public void testInsertByStrAndSelectFailedData() { } SessionDataSet dataSet = - session.executeQueryStatement("select s1, s2, s3, s4 from root.sg1.d1"); + session.executeQueryStatement("select s1, s2, s3, s4 from " + database + ".d1"); int i = 0; while (dataSet.hasNext()) { RowRecord record = dataSet.next(); @@ -126,13 +157,14 @@ public void testInsertByStrAndSelectFailedData() { } } + // DATE, TIMESTAMP, BLOB, and STRING values round-trip through typed record insertion. @Test public void testInsertRecord() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { List dataTypeList = Arrays.asList(TSDataType.DATE, TSDataType.TIMESTAMP, TSDataType.BLOB, TSDataType.STRING); List measurements = Arrays.asList("s1", "s2", "s3", "s4"); - String deviceId = "root.db.d1"; + String deviceId = database + ".d1"; for (int i = 0; i < dataTypeList.size(); i++) { String tsPath = deviceId + "." + measurements.get(i); if (!session.checkTimeseriesExists(tsPath)) { @@ -151,7 +183,8 @@ public void testInsertRecord() { values.add("" + time); session.insertRecord(deviceId, time, measurements, dataTypeList, values); } - try (SessionDataSet dataSet = session.executeQueryStatement("select * from root.db.d1")) { + try (SessionDataSet dataSet = + session.executeQueryStatement("select * from " + database + ".d1")) { HashSet columnNames = new HashSet<>(dataSet.getColumnNames()); assertEquals(5, columnNames.size()); for (int i = 0; i < 4; i++) { @@ -194,13 +227,14 @@ public void testInsertRecord() { } } + // Text-form records are parsed into DATE, TIMESTAMP, BLOB, and STRING values. @Test public void testInsertStrRecord() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { List dataTypeList = Arrays.asList(TSDataType.DATE, TSDataType.TIMESTAMP, TSDataType.BLOB, TSDataType.STRING); List measurements = Arrays.asList("s1", "s2", "s3", "s4"); - String deviceId = "root.db.d1"; + String deviceId = database + ".d1"; for (int i = 0; i < dataTypeList.size(); i++) { String tsPath = deviceId + "." + measurements.get(i); if (!session.checkTimeseriesExists(tsPath)) { @@ -219,7 +253,8 @@ public void testInsertStrRecord() { values.add("" + time); session.insertRecord(deviceId, time, measurements, values); } - try (SessionDataSet dataSet = session.executeQueryStatement("select * from root.db.d1")) { + try (SessionDataSet dataSet = + session.executeQueryStatement("select * from " + database + ".d1")) { HashSet columnNames = new HashSet<>(dataSet.getColumnNames()); assertEquals(5, columnNames.size()); for (int i = 0; i < 4; i++) { @@ -263,11 +298,12 @@ public void testInsertStrRecord() { } } + // Tablet insertion preserves typed values across both populated batches. @Test public void testInsertTablet() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { List schemaList = new ArrayList<>(); - String deviceId = "root.db.d1"; + String deviceId = database + ".d1"; schemaList.add(new MeasurementSchema("s1", TSDataType.DATE)); schemaList.add(new MeasurementSchema("s2", TSDataType.TIMESTAMP)); schemaList.add(new MeasurementSchema("s3", TSDataType.BLOB)); @@ -299,7 +335,8 @@ public void testInsertTablet() { } session.insertTablet(tablet); tablet.reset(); - try (SessionDataSet dataSet = session.executeQueryStatement("select * from root.db.d1")) { + try (SessionDataSet dataSet = + session.executeQueryStatement("select * from " + database + ".d1")) { HashSet columnNames = new HashSet<>(dataSet.getColumnNames()); assertEquals(5, columnNames.size()); for (int i = 0; i < 4; i++) { @@ -342,13 +379,14 @@ public void testInsertTablet() { } } + // Session operations before open and after close consistently report a closed session. @Test public void testSessionMisuse() throws StatementExecutionException, IoTDBConnectionException { final DataNodeWrapper dataNode = EnvFactory.getEnv().getDataNodeWrapperList().get(0); final Session session = new Builder().host(dataNode.getIp()).port(dataNode.getPort()).build(); // operate before open try { - session.executeNonQueryStatement("INSERT INTO root.db1.d1 (time, s1) VALUES (1,1)"); + session.executeNonQueryStatement("INSERT INTO " + database + ".d1 (time, s1) VALUES (1,1)"); } catch (IoTDBConnectionException e) { assertEquals("Session is not open, please invoke Session.open() first", e.getMessage()); } @@ -365,13 +403,13 @@ public void testSessionMisuse() throws StatementExecutionException, IoTDBConnect } try { - session.insertTablet(new Tablet("root.db1.d1", Collections.emptyList())); + session.insertTablet(new Tablet(database + ".d1", Collections.emptyList())); } catch (IoTDBConnectionException e) { assertEquals("Session is not open, please invoke Session.open() first", e.getMessage()); } try { - session.deleteDatabase("root.db"); + session.deleteDatabase(database); } catch (IoTDBConnectionException e) { assertEquals("Session is not open, please invoke Session.open() first", e.getMessage()); } @@ -388,7 +426,7 @@ public void testSessionMisuse() throws StatementExecutionException, IoTDBConnect session.close(); try { - session.executeNonQueryStatement("INSERT INTO root.db1.d1 (time, s1) VALUES (1,1)"); + session.executeNonQueryStatement("INSERT INTO " + database + ".d1 (time, s1) VALUES (1,1)"); } catch (IoTDBConnectionException e) { assertEquals("Session is not open, please invoke Session.open() first", e.getMessage()); } @@ -407,14 +445,14 @@ public void testSessionMisuse() throws StatementExecutionException, IoTDBConnect try { session.insertTablet( new Tablet( - "root.db1.d1", + database + ".d1", Collections.singletonList(new MeasurementSchema("s1", TSDataType.INT64)))); } catch (IoTDBConnectionException e) { assertEquals("Session is not open, please invoke Session.open() first", e.getMessage()); } try { - session.deleteDatabase("root.db"); + session.deleteDatabase(database); } catch (IoTDBConnectionException e) { assertEquals("Session is not open, please invoke Session.open() first", e.getMessage()); } diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/SessionSeedNodeFallbackIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/SessionSeedNodeFallbackIT.java index 8ab83b6233506..8c60b6f1f2ed1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/SessionSeedNodeFallbackIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/SessionSeedNodeFallbackIT.java @@ -37,8 +37,9 @@ import org.apache.iotdb.session.pool.SessionPool; import org.awaitility.Awaitility; -import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -64,10 +65,11 @@ @RunWith(IoTDBTestRunner.class) @Category(ClusterIT.class) public class SessionSeedNodeFallbackIT { + private static final int DATA_NODE_COUNT = 3; private static final int RECOVERY_CONNECTION_TIMEOUT_MS = 30_000; - @Before - public void setUp() { + @BeforeClass + public static void setUp() { EnvFactory.getEnv() .getConfig() .getCommonConfig() @@ -75,14 +77,36 @@ public void setUp() { .setDataReplicationFactor(3) .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) .setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); - EnvFactory.getEnv().initClusterEnvironment(1, 3); + EnvFactory.getEnv().initClusterEnvironment(1, DATA_NODE_COUNT); + } + + @Before + public void restoreDataNodes() { + // The omitted-seed case leaves a node stopped. Restore it without restarting healthy nodes, + // including when the previous case failed during its rolling restart. + List restartedNodes = new ArrayList<>(); + for (int index = 0; index < DATA_NODE_COUNT; index++) { + if (!EnvFactory.getEnv().getDataNodeWrapper(index).isAlive()) { + EnvFactory.getEnv().startDataNode(index); + restartedNodes.add(index); + } + } + for (int index = 0; index < DATA_NODE_COUNT; index++) { + awaitStatus(index, NodeStatus.Running); + } + // A Running heartbeat does not prove that a restarted node can serve the next case's clients. + for (int index : restartedNodes) { + awaitLogin(index); + } } - @After - public void tearDown() { + @AfterClass + public static void tearDown() { EnvFactory.getEnv().cleanClusterEnvironment(); } + // With three Ratis replicas, a Session and SessionPool must keep reading the sole stored row + // while each DataNode is stopped and restarted, and discover the current Running endpoints. @Test public void testQueriesDuringRollingRestart() throws Exception { SessionPool pool = @@ -100,7 +124,7 @@ public void testQueriesDuringRollingRestart() throws Exception { NodesSupplier poolSupplier = supplier(pool); Set runningNodes = new HashSet<>(Arrays.asList(endpoint(0), endpoint(1), endpoint(2))); - for (int index = 0; index < 3; index++) { + for (int index = 0; index < DATA_NODE_COUNT; index++) { EnvFactory.getEnv().shutdownDataNode(index); awaitStatus(index, EnvUtils.getNodeStatusAfterLocalStop()); runningNodes.remove(endpoint(index)); @@ -127,6 +151,8 @@ public void testQueriesDuringRollingRestart() throws Exception { } } + // Leave only node 1 in the clients' snapshots, then stop it after omitted seed node 0 recovers; + // business reconnection and background discovery must fall back to seed 0 and retain both seeds. @Test public void testBusinessAndBackgroundRecoveryThroughOmittedSeed() throws Exception { EnvFactory.getEnv().shutdownDataNode(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java index 0ce3be416d400..d40ff8efb9f69 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java @@ -19,7 +19,9 @@ package org.apache.iotdb.session.it.pool; import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.isession.pool.ISessionPool; import org.apache.iotdb.isession.pool.SessionDataSetWrapper; import org.apache.iotdb.it.env.EnvFactory; @@ -33,7 +35,9 @@ import org.apache.tsfile.enums.TSDataType; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -47,6 +51,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -60,19 +65,46 @@ public class SessionPoolIT { private static final Logger logger = LoggerFactory.getLogger(SessionPoolIT.class); private static final long DEFAULT_QUERY_TIMEOUT = -1; + private static final AtomicInteger NEXT_DATABASE_ID = new AtomicInteger(); - @Before - public void setUp() throws Exception { + private String database; + + @BeforeClass + public static void setUpClass() throws Exception { EnvFactory.getEnv().getConfig().getCommonConfig().setWalMode("SYNC"); // As this IT is only testing SessionPool itself, there's no need to launch a large cluster EnvFactory.getEnv().initClusterEnvironment(1, 1); } + @Before + public void setUp() { + database = "root.session_pool_" + NEXT_DATABASE_ID.incrementAndGet(); + } + @After public void tearDown() throws Exception { + dropDatabaseIfExists(database); + } + + @AfterClass + public static void tearDownClass() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + private void dropDatabaseIfExists(String databasePath) throws Exception { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + boolean exists; + try (SessionDataSet dataSet = + session.executeQueryStatement("SHOW DATABASES " + databasePath)) { + exists = dataSet.hasNext(); + } + if (exists) { + session.deleteDatabase(databasePath); + } + } + } + + // Concurrent inserts return all borrowed sessions to the pool after completing. @Test public void insert() { ISessionPool pool = EnvFactory.getEnv().getSessionPool(3); @@ -83,7 +115,7 @@ public void insert() { () -> { try { pool.insertRecord( - "root.sg1.d1", + database + ".d1", 1, Collections.singletonList("s" + no), Collections.singletonList(TSDataType.INT64), @@ -106,13 +138,14 @@ public void insert() { } } + // An invalid device path is rejected without leaving the pool unusable. @Test public void incorrectSQL() { ISessionPool pool = EnvFactory.getEnv().getSessionPool(3); assertEquals(0, pool.currentAvailableSize()); try { pool.insertRecord( - ".root.sg1.d1", + "." + database + ".d1", 1, Collections.singletonList("s"), Collections.singletonList(TSDataType.INT64), @@ -125,6 +158,7 @@ public void incorrectSQL() { } } + // Unclosed query results occupy the pool and prevent queued requests from completing. @Test public void incorrectExecuteQueryStatement() { ISessionPool pool = EnvFactory.getEnv().getSessionPool(3); @@ -137,7 +171,8 @@ public void incorrectExecuteQueryStatement() { () -> { try { SessionDataSetWrapper wrapper = - pool.executeQueryStatement("select * from root.sg1.d1 where time = " + no); + pool.executeQueryStatement( + "select * from " + database + ".d1 where time = " + no); // this is incorrect because wrapper is not closed. // so all other 7 queries will be blocked } catch (IoTDBConnectionException | StatementExecutionException e) { @@ -158,6 +193,7 @@ public void incorrectExecuteQueryStatement() { } } + // Concurrent point queries complete and release every result set to the pool. @Test public void executeQueryStatement() { ISessionPool pool = EnvFactory.getEnv().getSessionPool(3); @@ -165,6 +201,7 @@ public void executeQueryStatement() { pool.close(); } + // Point queries with an explicit timeout complete and release their sessions. @Test public void executeQueryStatementWithTimeout() { ISessionPool pool = EnvFactory.getEnv().getSessionPool(3); @@ -184,11 +221,12 @@ private void correctQuery(ISessionPool pool, long timeoutInMs) { SessionDataSetWrapper wrapper; if (timeoutInMs == DEFAULT_QUERY_TIMEOUT) { wrapper = - pool.executeQueryStatement("select * from root.sg1.d1 where time = " + no); + pool.executeQueryStatement( + "select * from " + database + ".d1 where time = " + no); } else { wrapper = pool.executeQueryStatement( - "select * from root.sg1.d1 where time = " + no, timeoutInMs); + "select * from " + database + ".d1 where time = " + no, timeoutInMs); } pool.closeResultSet(wrapper); } catch (Exception e) { @@ -208,13 +246,14 @@ private void correctQuery(ISessionPool pool, long timeoutInMs) { } } + // Concurrent raw-data queries return the timestamp selected by each worker. @Test public void executeRawDataQuery() { ISessionPool pool = EnvFactory.getEnv().getSessionPool(3); ExecutorService service = Executors.newFixedThreadPool(10); write10Data(pool, true); List pathList = new ArrayList<>(); - pathList.add("root.sg1.d1.s1"); + pathList.add(database + ".d1.s1"); for (int i = 0; i < 10; i++) { final int no = i; service.submit( @@ -243,13 +282,14 @@ public void executeRawDataQuery() { } } + // A query interrupted by a DataNode stop can be retried after the node recovers. @Test public void tryIfTheServerIsRestart() throws InterruptedException { ISessionPool pool = EnvFactory.getEnv().getSessionPool(3); SessionDataSetWrapper wrapper = null; BaseNodeWrapper node = EnvFactory.getEnv().getDataNodeWrapper(0); try { - wrapper = pool.executeQueryStatement("select * from root.sg1.d1 where time > 1"); + wrapper = pool.executeQueryStatement("select * from " + database + ".d1 where time > 1"); node.stop(); EnvFactory.getEnv() .ensureNodeStatus( @@ -320,6 +360,7 @@ public void tryIfTheServerIsRestart() throws InterruptedException { fail("should throw exception but not"); } + // A query whose result is already available releases its pool slot after consumption. @Test public void tryIfTheServerIsRestartButDataIsGotten() { ISessionPool pool = EnvFactory.getEnv().getSessionPool(3); @@ -327,7 +368,7 @@ public void tryIfTheServerIsRestartButDataIsGotten() { assertEquals(1, pool.currentAvailableSize()); SessionDataSetWrapper wrapper; try { - wrapper = pool.executeQueryStatement("select * from root.sg1.d1 where time > 1"); + wrapper = pool.executeQueryStatement("select * from " + database + ".d1 where time > 1"); // user does not know what happens. assertEquals(0, pool.currentAvailableSize()); assertEquals(1, pool.currentOccupiedSize()); @@ -345,6 +386,7 @@ public void tryIfTheServerIsRestartButDataIsGotten() { } } + // Writes fail while the DataNode is stopped and succeed after it restarts. @Test public void restart() throws InterruptedException { ISessionPool pool = EnvFactory.getEnv().getSessionPool(1); @@ -375,7 +417,7 @@ private void write10Data(ISessionPool pool, boolean failWhenThrowException) { for (int i = 0; i < 10; i++) { try { pool.insertRecord( - "root.sg1.d1", + database + ".d1", i, Collections.singletonList("s" + i), Collections.singletonList(TSDataType.INT64), @@ -389,13 +431,14 @@ private void write10Data(ISessionPool pool, boolean failWhenThrowException) { } } + // Closing a pool rejects later writes with the closed-pool error. @Test public void testClose() { ISessionPool pool = EnvFactory.getEnv().getSessionPool(3); pool.close(); try { pool.insertRecord( - "root.sg1.d1", + database + ".d1", 1, Collections.singletonList("s1"), Collections.singletonList(TSDataType.INT64), @@ -410,6 +453,7 @@ public void testClose() { // session. } + // Pool setters update their corresponding observable configuration values. @Test public void testSetters() { ISessionPool pool = EnvFactory.getEnv().getSessionPool(3);