diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/InternalTableSessionRedirectIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/InternalTableSessionRedirectIT.java index 9bd60069ccb61..a9b0b162d13cd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/InternalTableSessionRedirectIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/InternalTableSessionRedirectIT.java @@ -114,8 +114,11 @@ public void testTabletInsertRedirectsToInternalLeaderEndpoint() throws Exception session.insert(createTablet(2L)); final Session underlyingSession = InternalSessionITUtils.getUnderlyingSession(session); - final Map tableModelDeviceIdToEndpoint = + final Map> tableModelDeviceIdToEndpointByDatabase = InternalSessionITUtils.getSessionField(underlyingSession, "tableModelDeviceIdToEndpoint"); + assertEquals(1, tableModelDeviceIdToEndpointByDatabase.size()); + final Map tableModelDeviceIdToEndpoint = + tableModelDeviceIdToEndpointByDatabase.values().iterator().next(); assertEquals(1, tableModelDeviceIdToEndpoint.size()); assertTrue(leaderEndPoints.contains(tableModelDeviceIdToEndpoint.values().iterator().next())); } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java index 0fe69987b9e1b..fe18810776761 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java @@ -178,6 +178,69 @@ public static void main(String[] args) } } + @Test + public void testInsertTabletIntoSpecifiedDatabase() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement( + "CREATE TABLE db1.tablet_database (tag1 STRING TAG, m1 DOUBLE FIELD)"); + session.executeNonQueryStatement( + "CREATE TABLE db2.tablet_database (tag1 STRING TAG, m1 DOUBLE FIELD)"); + + List columnNames = Arrays.asList("tag1", "m1"); + List dataTypes = Arrays.asList(TSDataType.STRING, TSDataType.DOUBLE); + List columnCategories = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD); + Tablet tablet = new Tablet("tablet_database", columnNames, dataTypes, columnCategories, 2); + for (int row = 0; row < 2; row++) { + tablet.addTimestamp(row, row); + tablet.addValue("tag1", row, "tag:" + row); + tablet.addValue("m1", row, row * 1.0); + } + + // The distinct tag values exercise the grouped multi-device write path. + session.insert(tablet, "db1"); + session.executeNonQueryStatement("USE db1"); + session.insert(tablet, "db2"); + Tablet currentDatabaseTablet = + new Tablet("tablet_database", columnNames, dataTypes, columnCategories, 1); + currentDatabaseTablet.addTimestamp(0, 2); + currentDatabaseTablet.addValue("tag1", 0, "tag:2"); + currentDatabaseTablet.addValue("m1", 0, 2.0); + session.insert(currentDatabaseTablet, null); + + try (SessionDataSet dataSet = + session.executeQueryStatement("SELECT * FROM db1.tablet_database")) { + assertTrue(dataSet.hasNext()); + dataSet.next(); + assertTrue(dataSet.hasNext()); + dataSet.next(); + assertTrue(dataSet.hasNext()); + dataSet.next(); + assertFalse(dataSet.hasNext()); + } + try (SessionDataSet dataSet = + session.executeQueryStatement("SELECT * FROM db2.tablet_database")) { + assertTrue(dataSet.hasNext()); + dataSet.next(); + assertTrue(dataSet.hasNext()); + dataSet.next(); + assertFalse(dataSet.hasNext()); + } + try (SessionDataSet dataSet = session.executeQueryStatement("SHOW CURRENT_DATABASE")) { + assertTrue(dataSet.hasNext()); + assertEquals("db1", dataSet.next().getField(0).getStringValue()); + } + + for (String invalidDatabase : Arrays.asList("", " ", "a.b", "root.treedb", "ROOT.treedb")) { + StatementExecutionException exception = + assertThrows( + StatementExecutionException.class, () -> session.insert(tablet, invalidDatabase)); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), exception.getStatusCode()); + } + } + } + private static void insertRelationalTabletPerformanceTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/InternalSubscriptionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/InternalSubscriptionTableIT.java index 57468e4bb2567..260bbcac187c4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/InternalSubscriptionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/InternalSubscriptionTableIT.java @@ -163,10 +163,12 @@ public void testInternalPullConsumerPollsAcrossDataNodes() throws Exception { session.executeNonQueryStatement("flush"); final Session underlyingSession = InternalSessionITUtils.getUnderlyingSession(session); - final Map deviceToEndpoint = + final Map> deviceToEndpointByDatabase = InternalSessionITUtils.getSessionField( underlyingSession, "tableModelDeviceIdToEndpoint"); - leaders.addAll(deviceToEndpoint.values()); + deviceToEndpointByDatabase + .values() + .forEach(deviceToEndpoint -> leaders.addAll(deviceToEndpoint.values())); } Assert.assertTrue( diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java index 70d4266d6c889..299dff20f4542 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java @@ -70,7 +70,15 @@ void open( boolean enableRPCCompression, int connectionTimeoutInMs, Map deviceIdToEndpoint, - Map tabletModelDeviceIdToEndpoint, + Map tableModelDeviceIdToEndpoint, + INodeSupplier nodeSupplier) + throws IoTDBConnectionException; + + void openWithTableModelDatabaseCache( + boolean enableRPCCompression, + int connectionTimeoutInMs, + Map deviceIdToEndpoint, + Map> tableModelDeviceIdToEndpoint, INodeSupplier nodeSupplier) throws IoTDBConnectionException; diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java index 75a37f8384719..dfb7372f3ac34 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java @@ -46,6 +46,21 @@ public interface ITableSession extends AutoCloseable { */ void insert(Tablet tablet) throws StatementExecutionException, IoTDBConnectionException; + /** + * Inserts a {@link Tablet} into the specified database for this invocation only. + * + *

The database does not become the current database of this session and subsequent operations + * are unaffected. + * + * @param tablet the tablet containing time-series data to be inserted. + * @param database the target database for this insertion; if null, the current database of this + * session is used. + * @throws StatementExecutionException if an error occurs while executing the statement. + * @throws IoTDBConnectionException if there is an issue with the IoTDB connection. + */ + void insert(Tablet tablet, String database) + throws StatementExecutionException, IoTDBConnectionException; + /** * Executes a non-query SQL statement, such as a DDL or DML command. * diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionConfig.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionConfig.java index 7885d50cf5da1..cbd507e943d2f 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionConfig.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionConfig.java @@ -58,5 +58,7 @@ public class SessionConfig { public static final String DEFAULT_SSL_PROTOCOL = "TLS"; + public static final String SUPPORTS_DATABASE_SCOPED_TABLET = "supports_database_scoped_tablet"; + private SessionConfig() {} } diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/util/TableModelDatabaseUtils.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/util/TableModelDatabaseUtils.java new file mode 100644 index 0000000000000..dbe689e2d8c96 --- /dev/null +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/util/TableModelDatabaseUtils.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.isession.util; + +import java.util.regex.Pattern; + +/** Utilities for table-model database names. */ +public final class TableModelDatabaseUtils { + + private static final Pattern DATABASE_NAME_PATTERN = + Pattern.compile("([a-zA-Z0-9`_\\-\\u2E80-\\u9FFF]+)"); + + private TableModelDatabaseUtils() {} + + public static boolean isValidDatabaseName(String databaseName, int maxLength) { + return databaseName != null + && databaseName.length() <= maxLength + && databaseName.indexOf('.') < 0 + && DATABASE_NAME_PATTERN.matcher(databaseName).matches(); + } +} diff --git a/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java b/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java index 280a8414d6e8e..27245e48b1cc6 100644 --- a/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java +++ b/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java @@ -312,6 +312,11 @@ private SessionMessages() {} public static final String EXCEPTION_SESSION_CAN_NOT_CONNECT_TO_ARG_226FBA66 = "Session can not connect to {}"; + public static final String + EXCEPTION_SERVER_DOES_NOT_SUPPORT_DATABASE_SCOPED_TABLET_INSERTION_01CFFAF8 = + "The connected server does not support database-scoped Tablet insertion. Upgrade the" + + " server to use insert(Tablet, String database)."; + public static final String EXCEPTION_ALL_VALUES_ARE_NULL_AND_THIS_SUBMISSION_IS_IGNORED_DEVICEID_IS_ARG_TIME_IS_ARG_MEASUREMENTS_IS_ARG_5524221C = "All values are null and this submission is ignored,deviceId is [{}],time is" diff --git a/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java b/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java index cee0512a23a05..5b4c152f30b8d 100644 --- a/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java +++ b/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java @@ -309,6 +309,10 @@ private SessionMessages() {} public static final String EXCEPTION_SESSION_CAN_NOT_CONNECT_TO_ARG_226FBA66 = "Session 无法连接到 {}"; + public static final String + EXCEPTION_SERVER_DOES_NOT_SUPPORT_DATABASE_SCOPED_TABLET_INSERTION_01CFFAF8 = + "连接的服务器不支持按 database 写入 Tablet。请升级服务器后使用 insert(Tablet, String database)。"; + public static final String EXCEPTION_ALL_VALUES_ARE_NULL_AND_THIS_SUBMISSION_IS_IGNORED_DEVICEID_IS_ARG_TIME_IS_ARG_MEASUREMENTS_IS_ARG_5524221C = "所有值均为空,本次提交被忽略,deviceId 为 [{}],time 为 [{}],measurements 为 [{}]"; diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index d8bcce5e13def..e59a3d522a3ee 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -91,6 +91,7 @@ import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; @@ -173,12 +174,13 @@ public class Session implements ISession { private static final double CONVERT_THRESHOLD = 0.5; private static final double SAMPLE_PROPORTION = 0.05; private static final int MIN_RECORDS_SIZE = 40; + private static final String UNSPECIFIED_DATABASE_KEY = ""; @SuppressWarnings("squid:S3077") // Non-primitive fields should not be "volatile" protected volatile Map deviceIdToEndpoint; @SuppressWarnings("squid:S3077") // Non-primitive fields should not be "volatile" - protected volatile Map tableModelDeviceIdToEndpoint; + protected volatile Map> tableModelDeviceIdToEndpoint; @SuppressWarnings("squid:S3077") // Non-primitive fields should not be "volatile" protected volatile Map endPointToSessionConnection; @@ -666,6 +668,28 @@ public synchronized void open( Map tableModelDeviceIdToEndpoint, INodeSupplier nodesSupplier) throws IoTDBConnectionException { + Map> databaseToDeviceIdToEndpoint = new ConcurrentHashMap<>(); + if (tableModelDeviceIdToEndpoint != null) { + databaseToDeviceIdToEndpoint.put( + resolveTableModelRouteDatabase(null), tableModelDeviceIdToEndpoint); + } + openWithTableModelDatabaseCache( + enableThriftRpcCompaction, + connectionTimeoutInMs, + deviceIdToEndpoint, + databaseToDeviceIdToEndpoint, + nodesSupplier); + } + + /** Opens this Session with a database-aware table-model route cache for internal callers. */ + @Override + public synchronized void openWithTableModelDatabaseCache( + boolean enableThriftRpcCompaction, + int connectionTimeoutInMs, + Map deviceIdToEndpoint, + Map> tableModelDeviceIdToEndpoint, + INodeSupplier nodesSupplier) + throws IoTDBConnectionException { if (!isClosed) { return; } @@ -1420,12 +1444,17 @@ private SessionConnection getSessionConnection(String deviceId) throws IoTDBConn } } - private SessionConnection getSessionConnection(IDeviceID deviceId) + private Map getTableModelDatabaseRoutes(String routeDatabase) { + Map databaseToEndpoint = tableModelDeviceIdToEndpoint.get(routeDatabase); + return databaseToEndpoint == null ? new ConcurrentHashMap<>() : databaseToEndpoint; + } + + private SessionConnection getSessionConnection( + Map databaseToEndpoint, IDeviceID deviceId) throws IoTDBConnectionException { TEndPoint endPoint; if (enableRedirection - && tableModelDeviceIdToEndpoint != null - && (endPoint = tableModelDeviceIdToEndpoint.get(deviceId)) != null + && (endPoint = databaseToEndpoint.get(deviceId)) != null && endPointToSessionConnection.containsKey(endPoint)) { return endPointToSessionConnection.get(endPoint); } else { @@ -1465,12 +1494,13 @@ public void removeBrokenSessionConnection(SessionConnection sessionConnection) { } } if (tableModelDeviceIdToEndpoint != null && !tableModelDeviceIdToEndpoint.isEmpty()) { - for (Iterator> it = - tableModelDeviceIdToEndpoint.entrySet().iterator(); - it.hasNext(); ) { - Entry entry = it.next(); - if (entry.getValue().equals(endPoint)) { - it.remove(); + for (Map deviceIdToEndpoint : tableModelDeviceIdToEndpoint.values()) { + for (Iterator> it = deviceIdToEndpoint.entrySet().iterator(); + it.hasNext(); ) { + Entry entry = it.next(); + if (Objects.equals(entry.getValue(), endPoint)) { + it.remove(); + } } } } @@ -1503,15 +1533,21 @@ private void handleRedirection(String deviceId, TEndPoint endpoint) { } } - private void handleRedirection(IDeviceID deviceId, TEndPoint endpoint) { + private void handleRedirection( + String routeDatabase, + Map databaseToEndpoint, + IDeviceID deviceId, + TEndPoint endpoint) { if (enableRedirection) { // no need to redirection if (UrlUtils.isWildcardAddress(endpoint.ip)) { return; } - if (!tableModelDeviceIdToEndpoint.containsKey(deviceId) - || !tableModelDeviceIdToEndpoint.get(deviceId).equals(endpoint)) { - tableModelDeviceIdToEndpoint.put(deviceId, endpoint); + Map resolvedDatabaseToEndpoint = + tableModelDeviceIdToEndpoint.computeIfAbsent( + routeDatabase, ignored -> databaseToEndpoint); + if (!endpoint.equals(resolvedDatabaseToEndpoint.get(deviceId))) { + resolvedDatabaseToEndpoint.put(deviceId, endpoint); } SessionConnection connection = endPointToSessionConnection.computeIfAbsent( @@ -1524,11 +1560,22 @@ private void handleRedirection(IDeviceID deviceId, TEndPoint endpoint) { } }); if (connection == null) { - tableModelDeviceIdToEndpoint.remove(deviceId); + resolvedDatabaseToEndpoint.remove(deviceId, endpoint); } } } + private String resolveTableModelRouteDatabase(String database) { + String routeDatabase = database == null ? this.database : database; + return routeDatabase == null + ? UNSPECIFIED_DATABASE_KEY + : routeDatabase.toLowerCase(Locale.ENGLISH); + } + + private static String normalizeTableModelDatabaseName(String database) { + return database == null ? null : database.toLowerCase(Locale.ENGLISH); + } + private void handleQueryRedirection(TEndPoint endPoint) throws IoTDBConnectionException { if (enableQueryRedirection) { AtomicReference exceptionReference = new AtomicReference<>(); @@ -2907,11 +2954,25 @@ private void insertTabletInternal(Tablet tablet, TSInsertTabletReq request) */ public void insertRelationalTablet(Tablet tablet) throws IoTDBConnectionException, StatementExecutionException { + insertRelationalTablet(tablet, null); + } + + /** + * Inserts a relational {@link Tablet} into the specified database for this invocation only. + * + * @param tablet data batch + * @param database target database; if null, use the current database of this session + */ + public void insertRelationalTablet(Tablet tablet, String database) + throws IoTDBConnectionException, StatementExecutionException { + String normalizedDatabase = normalizeTableModelDatabaseName(database); if (tablet.getRowSize() == 0) { return; } + // Capture the effective database before grouping so a later USE cannot change cached routes. + String routeDatabase = resolveTableModelRouteDatabase(normalizedDatabase); if (enableRedirection) { - insertRelationalTabletWithLeaderCache(tablet); + insertRelationalTabletWithLeaderCache(tablet, normalizedDatabase, routeDatabase); } else { TSInsertTabletReq request = genTSInsertTabletReq(tablet, false, false); if (request == null) { @@ -2923,6 +2984,7 @@ public void insertRelationalTablet(Tablet tablet) return; } request.setWriteToTable(true); + setRelationalRequestDatabase(request, normalizedDatabase); try { getDefaultSessionConnection().insertTablet(request); } catch (RedirectException ignored) { @@ -2930,17 +2992,20 @@ public void insertRelationalTablet(Tablet tablet) } } - private void insertRelationalTabletWithLeaderCache(Tablet tablet) + private void insertRelationalTabletWithLeaderCache( + Tablet tablet, String database, String routeDatabase) throws IoTDBConnectionException, StatementExecutionException { Map relationalTabletGroup = new HashMap<>(); - if (tableModelDeviceIdToEndpoint.isEmpty()) { + Map databaseToEndpoint = getTableModelDatabaseRoutes(routeDatabase); + if (databaseToEndpoint.isEmpty()) { relationalTabletGroup.put(getDefaultSessionConnection(), tablet); } else if (SessionUtils.isTabletContainsSingleDevice(tablet)) { - relationalTabletGroup.put(getSessionConnection(tablet.getDeviceID(0)), tablet); + relationalTabletGroup.put( + getSessionConnection(databaseToEndpoint, tablet.getDeviceID(0)), tablet); } else { for (int i = 0; i < tablet.getRowSize(); i++) { IDeviceID iDeviceID = tablet.getDeviceID(i); - final SessionConnection connection = getSessionConnection(iDeviceID); + final SessionConnection connection = getSessionConnection(databaseToEndpoint, iDeviceID); int finalI = i; relationalTabletGroup.compute( connection, @@ -2975,13 +3040,19 @@ private void insertRelationalTabletWithLeaderCache(Tablet tablet) } } if (relationalTabletGroup.size() == 1) { - insertRelationalTabletOnce(relationalTabletGroup); + insertRelationalTabletOnce( + relationalTabletGroup, database, routeDatabase, databaseToEndpoint); } else { - insertRelationalTabletByGroup(relationalTabletGroup); + insertRelationalTabletByGroup( + relationalTabletGroup, database, routeDatabase, databaseToEndpoint); } } - private void insertRelationalTabletOnce(Map relationalTabletGroup) + private void insertRelationalTabletOnce( + Map relationalTabletGroup, + String database, + String routeDatabase, + Map databaseToEndpoint) throws IoTDBConnectionException, StatementExecutionException { Map.Entry entry = relationalTabletGroup.entrySet().iterator().next(); SessionConnection connection = entry.getKey(); @@ -2996,6 +3067,7 @@ private void insertRelationalTabletOnce(Map relationa return; } request.setWriteToTable(true); + setRelationalRequestDatabase(request, database); try { connection.insertTablet(request); } catch (RedirectException e) { @@ -3006,7 +3078,9 @@ private void insertRelationalTabletOnce(Map relationa endPointMap.put(tablet.getDeviceID(i), endPointList.get(i)); } } - endPointMap.forEach(this::handleRedirection); + endPointMap.forEach( + (deviceId, endpoint) -> + handleRedirection(routeDatabase, databaseToEndpoint, deviceId, endpoint)); } catch (IoTDBConnectionException e) { if (endPointToSessionConnection != null && endPointToSessionConnection.size() > 1) { // remove the broken session @@ -3024,7 +3098,11 @@ private void insertRelationalTabletOnce(Map relationa @SuppressWarnings({ "squid:S3776" }) // ignore Cognitive Complexity of methods should not be too high - private void insertRelationalTabletByGroup(Map relationalTabletGroup) + private void insertRelationalTabletByGroup( + Map relationalTabletGroup, + String database, + String routeDatabase, + Map databaseToEndpoint) throws IoTDBConnectionException, StatementExecutionException { List> completableFutures = relationalTabletGroup.entrySet().stream() @@ -3044,6 +3122,7 @@ private void insertRelationalTabletByGroup(Map relati return; } request.setWriteToTable(true); + setRelationalRequestDatabase(request, database); InsertConsumer insertConsumer = SessionConnection::insertTablet; try { @@ -3056,7 +3135,10 @@ private void insertRelationalTabletByGroup(Map relati endPointMap.put(subTablet.getDeviceID(i), endPointList.get(i)); } } - endPointMap.forEach(this::handleRedirection); + endPointMap.forEach( + (deviceId, endpoint) -> + handleRedirection( + routeDatabase, databaseToEndpoint, deviceId, endpoint)); } catch (StatementExecutionException e) { throw new CompletionException(e); } catch (IoTDBConnectionException e) { @@ -3101,6 +3183,12 @@ private void insertRelationalTabletByGroup(Map relati } } + private static void setRelationalRequestDatabase(TSInsertTabletReq request, String database) { + if (database != null) { + request.setDatabase(database); + } + } + /** * insert the aligned timeseries data of a device. For each timestamp, the number of measurements * is the same. diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java index ca8862cf1c989..fb0a2256cf314 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@ -129,6 +129,8 @@ public class SessionConnection { // ms is 1_000, us is 1_000_000, ns is 1_000_000_000 private int timeFactor = 1_000; + private boolean supportsDatabaseScopedTablet; + // TestOnly public SessionConnection(String sqlDialect) { availableNodes = Collections::emptyList; @@ -266,6 +268,10 @@ private void init( TSOpenSessionResp openResp = client.openSession(openReq); RpcUtils.verifySuccess(openResp.getStatus()); + supportsDatabaseScopedTablet = + openResp.getConfiguration() != null + && Boolean.parseBoolean( + openResp.getConfiguration().get(SessionConfig.SUPPORTS_DATABASE_SCOPED_TABLET)); this.timeFactor = RpcUtils.getTimeFactor(openResp); if (Session.protocolVersion.getValue() != openResp.getServerProtocolVersion().getValue()) { logger.warn( @@ -901,6 +907,12 @@ protected void insertTablet(TSInsertTabletReq request) } private TSStatus insertTabletInternal(TSInsertTabletReq request) throws TException { + if (request.isSetDatabase() && !supportsDatabaseScopedTablet) { + return new TSStatus(TSStatusCode.UNSUPPORTED_OPERATION.getStatusCode()) + .setMessage( + SessionMessages + .EXCEPTION_SERVER_DOES_NOT_SUPPORT_DATABASE_SCOPED_TABLET_INSERTION_01CFFAF8); + } request.setSessionId(sessionId); return client.insertTablet(request); } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java index 059fdea346948..b51a3a59831d8 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java @@ -39,6 +39,12 @@ public void insert(Tablet tablet) throws StatementExecutionException, IoTDBConne session.insertRelationalTablet(tablet); } + @Override + public void insert(Tablet tablet, String database) + throws StatementExecutionException, IoTDBConnectionException { + session.insertRelationalTablet(tablet, database); + } + @Override public void executeNonQueryStatement(String sql) throws IoTDBConnectionException, StatementExecutionException { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java index 9585d65c3902e..9ab73fdd8f20a 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java @@ -139,7 +139,7 @@ public class SessionPool implements ISessionPool { private boolean enableQueryRedirection = false; private Map deviceIdToEndpoint; - private Map tableModelDeviceIdToEndpoint; + private Map> tableModelDeviceIdToEndpoint; private int thriftDefaultBufferSize; private int thriftMaxFrameSize; @@ -774,7 +774,7 @@ private ISession getSession() throws IoTDBConnectionException { session = constructNewSession(); try { - session.open( + session.openWithTableModelDatabaseCache( enableThriftCompression, connectionTimeoutInMs, deviceIdToEndpoint, @@ -890,7 +890,7 @@ public void closeResultSet(SessionDataSetWrapper wrapper) { private void tryConstructNewSession() { Session session = constructNewSession(); try { - session.open( + session.openWithTableModelDatabaseCache( enableThriftCompression, connectionTimeoutInMs, deviceIdToEndpoint, diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java index 4cf7ca980f347..5cb7a89fc0eea 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java @@ -58,8 +58,14 @@ protected TableSessionWrapper(Session session, SessionPool sessionPool) { @Override public void insert(Tablet tablet) throws StatementExecutionException, IoTDBConnectionException { + insert(tablet, null); + } + + @Override + public void insert(Tablet tablet, String database) + throws StatementExecutionException, IoTDBConnectionException { try { - session.insertRelationalTablet(tablet); + session.insertRelationalTablet(tablet, database); } catch (IoTDBConnectionException e) { sessionPool.cleanSessionAndMayThrowConnectionException(session); closed.set(true); diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionCacheLeaderTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionCacheLeaderTest.java index 46ec3ee1f5d21..7940b33fc0496 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionCacheLeaderTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionCacheLeaderTest.java @@ -47,11 +47,13 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -704,11 +706,70 @@ public void testInsertRelationalTablet() tablet.reset(); } - assertEquals(4, session.tableModelDeviceIdToEndpoint.size()); + assertEquals(1, session.tableModelDeviceIdToEndpoint.size()); + assertEquals(4, session.tableModelDeviceIdToEndpoint.values().iterator().next().size()); assertEquals(4, session.endPointToSessionConnection.size()); session.close(); } + @Test + public void testRelationalTabletRedirectCacheIsScopedByDatabase() + throws IoTDBConnectionException, StatementExecutionException { + final TEndPoint db1Endpoint = endpoints.get(1); + final TEndPoint db2Endpoint = endpoints.get(2); + final AtomicInteger db1AcceptedWrites = new AtomicInteger(); + final AtomicInteger db2AcceptedWrites = new AtomicInteger(); + session = + new MockSession("127.0.0.1", 55560, true, "table") { + @Override + public SessionConnection constructSessionConnection( + Session owner, TEndPoint endpoint, ZoneId zoneId) { + final TEndPoint connectionEndpoint = endpoint; + return new MockSessionConnection(owner, endpoint, zoneId) { + @Override + protected void insertTablet(TSInsertTabletReq request) throws RedirectException { + TEndPoint leader = "db1".equals(request.getDatabase()) ? db1Endpoint : db2Endpoint; + if (connectionEndpoint.equals(leader)) { + if (leader.equals(db1Endpoint)) { + db1AcceptedWrites.incrementAndGet(); + } else { + db2AcceptedWrites.incrementAndGet(); + } + return; + } + throw new RedirectException(Collections.singletonList(leader)); + } + }; + } + }; + session.open(); + + Tablet tablet = + new Tablet( + "table1", + Arrays.asList("id", "field"), + Arrays.asList(TSDataType.STRING, TSDataType.INT64), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD), + 1); + tablet.addTimestamp(0, 1); + tablet.addValue("id", 0, "id0"); + tablet.addValue("field", 0, 1L); + + session.insertRelationalTablet(tablet, "DB1"); + session.insertRelationalTablet(tablet, "db2"); + session.insertRelationalTablet(tablet, "db1"); + session.insertRelationalTablet(tablet, "DB2"); + + assertEquals(1, db1AcceptedWrites.get()); + assertEquals(1, db2AcceptedWrites.get()); + assertEquals(2, session.tableModelDeviceIdToEndpoint.size()); + assertEquals( + db1Endpoint, session.tableModelDeviceIdToEndpoint.get("db1").get(tablet.getDeviceID(0))); + assertEquals( + db2Endpoint, session.tableModelDeviceIdToEndpoint.get("db2").get(tablet.getDeviceID(0))); + session.close(); + } + @Test public void testGroupedRelationalTabletPreservesStatementFailures() throws Exception { AtomicBoolean failWrites = new AtomicBoolean(); @@ -750,7 +811,8 @@ protected void insertTablet(TSInsertTabletReq request) tablet.addValue("field", row, 1L); } session.insertRelationalTablet(tablet); - assertEquals(4, session.tableModelDeviceIdToEndpoint.size()); + assertEquals(1, session.tableModelDeviceIdToEndpoint.size()); + assertEquals(4, session.tableModelDeviceIdToEndpoint.values().iterator().next().size()); failWrites.set(true); tablet.reset(); @@ -1217,9 +1279,11 @@ public void testInsertRelationalTabletWithSessionBroken() throws StatementExecut timestamp++; } - assertEquals(3, session.tableModelDeviceIdToEndpoint.size()); - for (Map.Entry endPointEntry : - session.tableModelDeviceIdToEndpoint.entrySet()) { + assertEquals(1, session.tableModelDeviceIdToEndpoint.size()); + Map tableModelDeviceRoutes = + session.tableModelDeviceIdToEndpoint.values().iterator().next(); + assertEquals(3, tableModelDeviceRoutes.size()); + for (Map.Entry endPointEntry : tableModelDeviceRoutes.entrySet()) { assertEquals(getDeviceIdBelongedEndpoint(endPointEntry.getKey()), endPointEntry.getValue()); } assertEquals(3, session.endPointToSessionConnection.size()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index e6eedc4b636a9..96f468ee68f10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -142,6 +142,7 @@ import org.apache.iotdb.db.utils.SchemaUtils; import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.db.utils.TypeServices; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.rpc.stmt.PreparedParameterSerde; @@ -1639,6 +1640,7 @@ public TSOpenSessionResp openSession(TSOpenSessionReq req) throws TException { Map configuration = new HashMap<>(); configuration.put( TIME_PRECISION, CommonDescriptor.getInstance().getConfig().getTimestampPrecision()); + configuration.put(SessionConfig.SUPPORTS_DATABASE_SCOPED_TABLET, Boolean.TRUE.toString()); return resp.setSessionId(openSessionResp.getSessionId()).setConfiguration(configuration); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 7a00d5dfd9aa4..262cdd5fbc742 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -57,7 +57,6 @@ import org.apache.iotdb.db.audit.PasswordChangeAuditTask; import org.apache.iotdb.db.audit.UserRoleModificationAuditContext; import org.apache.iotdb.db.auth.AuthorityChecker; -import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -269,6 +268,7 @@ import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterParser; import org.apache.iotdb.db.subscription.tagfilter.TagFilterParser; +import org.apache.iotdb.isession.util.TableModelDatabaseUtils; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.rpc.subscription.config.TopicConstant; @@ -310,7 +310,6 @@ import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.MAX_SCHEMA_REGION_GROUP_NUM_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.TIME_PARTITION_INTERVAL_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.TTL_KEY; -import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; public class TableConfigTaskVisitor implements AstVisitor { @@ -951,14 +950,11 @@ public IConfigTask visitSetColumnComment( } public static void validateDatabaseName(final String dbName) throws SemanticException { - // Check database length here - if (dbName.contains(PATH_SEPARATOR) - || !IoTDBConfig.DATABASE_PATTERN.matcher(dbName).matches() - || dbName.length() > MAX_DATABASE_NAME_LENGTH) { + if (!TableModelDatabaseUtils.isValidDatabaseName(dbName, MAX_DATABASE_NAME_LENGTH)) { throw new SemanticException( new IllegalPathException( dbName, - dbName.length() > MAX_DATABASE_NAME_LENGTH + dbName != null && dbName.length() > MAX_DATABASE_NAME_LENGTH ? DataNodeQueryMessages.THE_LENGTH_OF_DATABASE_NAME_SHALL_NOT_EXCEED + MAX_DATABASE_NAME_LENGTH : DataNodeQueryMessages diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java index 046a94c1fa243..c13bd39e51c1b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.qp.sql.IoTDBSqlParser; import org.apache.iotdb.db.qp.sql.SqlLexer; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; +import org.apache.iotdb.db.queryengine.plan.execution.config.TableConfigTaskVisitor; import org.apache.iotdb.db.queryengine.plan.expression.binary.GreaterEqualExpression; import org.apache.iotdb.db.queryengine.plan.expression.binary.LessThanExpression; import org.apache.iotdb.db.queryengine.plan.expression.binary.LogicAndExpression; @@ -116,6 +117,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; @@ -385,6 +387,11 @@ public static InsertTabletStatement createStatement(TSInsertTabletReq insertTabl ColumnCategory.values()[insertTabletReq.columnCategories.get(i).intValue()]); } insertStatement.setColumnCategories(columnCategories); + if (insertTabletReq.isSetDatabase()) { + String database = insertTabletReq.getDatabase().toLowerCase(Locale.ENGLISH); + TableConfigTaskVisitor.validateDatabaseName(database); + insertStatement.setDatabaseName(database); + } } PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return insertStatement; diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index 7ff12ceb02154..4ac00b7e270b1 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -274,6 +274,7 @@ struct TSInsertTabletReq { 11: optional bool isCompressed 12: optional list encodingTypes 13: optional byte compressType + 14: optional string database } struct TSInsertTabletsReq { @@ -705,4 +706,4 @@ service IClientRPCService { /** For other node's call */ common.TSStatus testConnectionEmptyRPC(); -} \ No newline at end of file +}