From beb050c5e7add46d6a96fc2d4402c97d43637c0a Mon Sep 17 00:00:00 2001 From: Yongzao <532741407@qq.com> Date: Mon, 27 Jul 2026 16:13:35 +0800 Subject: [PATCH] Support safe multi-node removal --- ...RegionOperationReliabilityITFramework.java | 15 +++++ .../IoTDBRemoveConfigNodeITFramework.java | 29 ++++++++ .../IoTDBRemoveConfigNodeNormalIT.java | 5 ++ .../IoTDBRemoveDataNodeNormalIT.java | 65 +++++++++++++----- .../IoTDBRemoveDataNodeUtils.java | 67 +++++++++++++++++-- .../confignode/i18n/ManagerMessages.java | 6 ++ .../confignode/i18n/ProcedureMessages.java | 3 + .../confignode/i18n/ManagerMessages.java | 6 ++ .../confignode/i18n/ProcedureMessages.java | 3 + .../confignode/manager/ConfigManager.java | 2 +- .../confignode/manager/ProcedureManager.java | 32 +++------ .../confignode/manager/node/NodeManager.java | 51 +++++++++++--- .../manager/node/NodeRemovalSafety.java | 38 +++++++++++ .../procedure/env/RemoveDataNodeHandler.java | 47 +++++++++++-- .../impl/node/RemoveConfigNodeProcedure.java | 4 ++ .../impl/node/RemoveDataNodesProcedure.java | 23 ++++--- .../manager/ProcedureManagerTest.java | 28 -------- .../manager/node/NodeRemovalSafetyTest.java | 45 +++++++++++++ 18 files changed, 372 insertions(+), 97 deletions(-) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeRemovalSafety.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeRemovalSafetyTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionOperationReliabilityITFramework.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionOperationReliabilityITFramework.java index de096a95712b3..60d05695a92e6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionOperationReliabilityITFramework.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionOperationReliabilityITFramework.java @@ -620,6 +620,21 @@ public static Map> getDataRegionMap(Statement statement) t return regionMap; } + public static Map> getSchemaRegionMap(Statement statement) + throws Exception { + ResultSet showRegionsResult = statement.executeQuery(SHOW_REGIONS); + Map> regionMap = new HashMap<>(); + while (showRegionsResult.next()) { + if (String.valueOf(TConsensusGroupType.SchemaRegion) + .equals(showRegionsResult.getString(ColumnHeaderConstant.TYPE))) { + int regionId = showRegionsResult.getInt(ColumnHeaderConstant.REGION_ID); + int dataNodeId = showRegionsResult.getInt(ColumnHeaderConstant.DATA_NODE_ID); + regionMap.computeIfAbsent(regionId, id -> new HashSet<>()).add(dataNodeId); + } + } + return regionMap; + } + public static Map>> getDataRegionMapWithLeader( Statement statement) throws Exception { ResultSet showRegionsResult = statement.executeQuery(SHOW_REGIONS); diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java index 6112fea7f5255..254361f0fd1cc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java @@ -152,6 +152,35 @@ public void testRemoveConfigNode( } } + public void testRejectRemoveConfigNodeFromTwoReplicaGroup(final SQLModel model) throws Exception { + EnvFactory.getEnv().initClusterEnvironment(2, 1); + + try (final Connection connection = makeItCloseQuietly(getConnectionWithSQLType(model)); + final Statement statement = makeItCloseQuietly(connection.createStatement())) { + final ResultSet result = statement.executeQuery(SHOW_CONFIGNODES); + final Set allConfigNodeIds = new HashSet<>(); + while (result.next()) { + allConfigNodeIds.add(result.getInt(ColumnHeaderConstant.NODE_ID)); + } + Assert.assertEquals(2, allConfigNodeIds.size()); + + final int removeConfigNodeId = allConfigNodeIds.iterator().next(); + try { + statement.execute(generateRemoveString(removeConfigNodeId)); + Assert.fail(); + } catch (IoTDBSQLException expected) { + // Removing one peer from a two-replica Ratis group would lose the majority. + } + + final ResultSet remainingResult = statement.executeQuery(SHOW_CONFIGNODES); + int remainingConfigNodeCount = 0; + while (remainingResult.next()) { + remainingConfigNodeCount++; + } + Assert.assertEquals(2, remainingConfigNodeCount); + } + } + private static void awaitUntilSuccess(Statement statement, int removeConfigNodeId) { AtomicReference> lastTimeConfigNodes = new AtomicReference<>(); AtomicReference lastException = new AtomicReference<>(); diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeNormalIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeNormalIT.java index 2b5026f9ad903..a49ebf75d733f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeNormalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeNormalIT.java @@ -39,4 +39,9 @@ public void test3C1DUseTreeSQL() throws Exception { public void test3C1DUseTableSQL() throws Exception { testRemoveConfigNode(1, 1, 3, 1, 2, SQLModel.TABLE_MODEL_SQL); } + + @Test + public void testRejectRemovingOneOfTwoConfigNodes() throws Exception { + testRejectRemoveConfigNodeFromTwoReplicaGroup(SQLModel.TREE_MODEL_SQL); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeNormalIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeNormalIT.java index bf0a6ce55272c..ca500f97ab4ee 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeNormalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeNormalIT.java @@ -58,11 +58,13 @@ import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getAllRegionMap; import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getDataRegionMap; +import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getSchemaRegionMap; import static org.apache.iotdb.confignode.it.removedatanode.IoTDBRemoveDataNodeUtils.awaitUntilSuccess; import static org.apache.iotdb.confignode.it.removedatanode.IoTDBRemoveDataNodeUtils.generateRemoveString; import static org.apache.iotdb.confignode.it.removedatanode.IoTDBRemoveDataNodeUtils.getConnectionWithSQLType; import static org.apache.iotdb.confignode.it.removedatanode.IoTDBRemoveDataNodeUtils.restartDataNodes; import static org.apache.iotdb.confignode.it.removedatanode.IoTDBRemoveDataNodeUtils.selectRemoveDataNodes; +import static org.apache.iotdb.confignode.it.removedatanode.IoTDBRemoveDataNodeUtils.selectRemoveDataNodesSharingWeakRegionSafely; import static org.apache.iotdb.confignode.it.removedatanode.IoTDBRemoveDataNodeUtils.selectRemoveDataNodesWithoutRegionConflict; import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly; @@ -158,10 +160,11 @@ public void success1C4DIoTTestUseTableSQL() throws Exception { } @Test - public void success1C5DRemoveTwoDataNodesUseSQL() throws Exception { - // Setup 1C5D, and remove 2D in a single "remove datanode a, b" statement; 3 DataNodes remain - // which is enough to keep both the data (factor 2) and schema (factor 3) replicas. - successTest(2, 3, 1, 5, 2, 2, true, SQLModel.TREE_MODEL_SQL, ConsensusFactory.IOT_CONSENSUS); + public void success1C5DRemoveTwoDataNodesSharingRegionUseTableSQL() throws Exception { + // IoTConsensus is weakly consistent, so two replicas of a three-replica DataRegion can be + // removed by one statement. The procedure migrates the two replicas in separate rounds. + successTestWithDataRegionConflict( + 3, 3, 1, 5, 2, 2, true, SQLModel.TABLE_MODEL_SQL, ConsensusFactory.IOT_CONSENSUS); } // @Test @@ -209,7 +212,33 @@ private void successTest( true, rejoinRemovedDataNode, model, - dataRegionGroupConsensusProtocol); + dataRegionGroupConsensusProtocol, + false); + } + + private void successTestWithDataRegionConflict( + final int dataReplicateFactor, + final int schemaReplicationFactor, + final int configNodeNum, + final int dataNodeNum, + final int removeDataNodeNum, + final int dataRegionPerDataNode, + final boolean rejoinRemovedDataNode, + final SQLModel model, + final String dataRegionGroupConsensusProtocol) + throws Exception { + testRemoveDataNode( + dataReplicateFactor, + schemaReplicationFactor, + configNodeNum, + dataNodeNum, + removeDataNodeNum, + dataRegionPerDataNode, + true, + rejoinRemovedDataNode, + model, + dataRegionGroupConsensusProtocol, + true); } private void failTest( @@ -233,7 +262,8 @@ private void failTest( false, rejoinRemovedDataNode, model, - dataRegionGroupConsensusProtocol); + dataRegionGroupConsensusProtocol, + false); } public void testRemoveDataNode( @@ -246,7 +276,8 @@ public void testRemoveDataNode( final boolean expectRemoveSuccess, final boolean rejoinRemovedDataNode, final SQLModel model, - final String dataRegionGroupConsensusProtocol) + final String dataRegionGroupConsensusProtocol, + final boolean selectWithDataRegionConflict) throws Exception { // Set up specific environment EnvFactory.getEnv() @@ -288,17 +319,17 @@ public void testRemoveDataNode( allDataNodeId.add(result.getInt(ColumnHeaderConstant.NODE_ID)); } - // Select data nodes to remove. When removing more than one DataNode we must avoid picking two - // DataNodes that host replicas of the same consensus group, otherwise the - // RemoveDataNodesProcedure would try to migrate two replicas of one group at once, which the - // ConfigNode rejects ("Only one replica of the same consensus group is allowed to be migrated - // at the same time."). Selecting purely at random therefore makes this test flaky, so use a - // conflict-free selection. + // Most multi-node cases use a conflict-free selection so strong-consistency RegionGroups do + // not exceed their removal threshold. The targeted weak-consistency case deliberately picks + // replicas from the same DataRegion to verify round-based migration orchestration. final Set removeDataNodes = - removeDataNodeNum > 1 - ? selectRemoveDataNodesWithoutRegionConflict( - allDataNodeId, removeDataNodeNum, getAllRegionMap(statement)) - : selectRemoveDataNodes(allDataNodeId, removeDataNodeNum); + selectWithDataRegionConflict + ? selectRemoveDataNodesSharingWeakRegionSafely( + removeDataNodeNum, regionMap, getSchemaRegionMap(statement)) + : removeDataNodeNum > 1 + ? selectRemoveDataNodesWithoutRegionConflict( + allDataNodeId, removeDataNodeNum, getAllRegionMap(statement)) + : selectRemoveDataNodes(allDataNodeId, removeDataNodeNum); List removeDataNodeWrappers = removeDataNodes.stream() diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeUtils.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeUtils.java index 2205554b1e9c0..98806c3e4ab04 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeUtils.java @@ -76,15 +76,68 @@ public static Set selectRemoveDataNodes( return new HashSet<>(shuffledDataNodeIds.subList(0, removeDataNodeNum)); } + /** + * Select DataNodes that share a weak-consistency RegionGroup while keeping every affected weak + * and strong RegionGroup within its removal-safety threshold. + */ + public static Set selectRemoveDataNodesSharingWeakRegionSafely( + int removeDataNodeNum, + Map> weakRegionMap, + Map> strongRegionMap) { + for (Set weakRegionDataNodes : weakRegionMap.values()) { + Set selected = new LinkedHashSet<>(); + if (searchSafeSharedRegionSelection( + new ArrayList<>(weakRegionDataNodes), + 0, + removeDataNodeNum, + selected, + weakRegionMap, + strongRegionMap)) { + return selected; + } + } + throw new IllegalStateException( + String.format( + "Cannot select %d DataNodes that safely share a weak-consistency RegionGroup. " + + "weakRegionMap=%s, strongRegionMap=%s", + removeDataNodeNum, weakRegionMap, strongRegionMap)); + } + + private static boolean searchSafeSharedRegionSelection( + List candidates, + int start, + int need, + Set selected, + Map> weakRegionMap, + Map> strongRegionMap) { + if (selected.size() == need) { + return weakRegionMap.values().stream() + .allMatch(region -> countSelectedReplicas(selected, region) < region.size()) + && strongRegionMap.values().stream() + .allMatch(region -> 2L * countSelectedReplicas(selected, region) < region.size()); + } + for (int i = start; i < candidates.size(); i++) { + selected.add(candidates.get(i)); + if (searchSafeSharedRegionSelection( + candidates, i + 1, need, selected, weakRegionMap, strongRegionMap)) { + return true; + } + selected.remove(candidates.get(i)); + } + return false; + } + + private static long countSelectedReplicas(Set selected, Set regionDataNodes) { + return selected.stream().filter(regionDataNodes::contains).count(); + } + /** * Select {@code removeDataNodeNum} DataNodes to remove such that no two of them host a replica of - * the same consensus group. Removing two DataNodes that share a region group would make the - * generated {@code RemoveDataNodesProcedure} try to migrate two replicas of the same group at - * once, which the ConfigNode rejects ("Only one replica of the same consensus group is allowed to - * be migrated at the same time."). Randomly picking DataNodes (see {@link - * #selectRemoveDataNodes}) therefore makes multi-DataNode-remove tests flaky; this method instead - * searches exhaustively (with backtracking) for a conflict-free selection, so it fails only when - * no such selection exists rather than when an unlucky pick order dead-ends. + * the same consensus group. Most removal tests are not intended to exercise the per-RegionGroup + * safety threshold, so randomly picking DataNodes (see {@link #selectRemoveDataNodes}) can make + * them fail when the selected nodes contain too many replicas of a strong-consistency group. This + * method instead searches exhaustively (with backtracking) for a conflict-free selection, so it + * fails only when no such selection exists rather than when an unlucky pick order dead-ends. * * @param allDataNodeId all registered DataNode ids * @param removeDataNodeNum how many DataNodes to remove diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 20316edc39713..d93852acf376c 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -660,6 +660,12 @@ private ManagerMessages() {} "Remove ConfigNode failed because there is no other ConfigNode in Running status in current" + " Cluster."; public static final String MESSAGE_REMOVE_CONFIGNODE_FAILED_BECAUSE_CONFIGNODEGROUP_LEADER_ELECTION_PLEASE_RETRY_3EE602F6 = "Remove ConfigNode failed because the ConfigNodeGroup is on leader election, please retry."; + public static final String + MESSAGE_CANNOT_REMOVE_CONFIGNODE_ARG_BECAUSE_IT_IS_ALREADY_BEING_REMOVED_EEE0128D = + "Cannot remove ConfigNode %d because it is already being removed."; + public static final String + MESSAGE_CANNOT_REMOVE_CONFIGNODE_ARG_REMOVING_ARG_CONFIGNODES_AT_THE_SAME_TIME_FROM_A_ARG_REPLICA_CONFIGNODEGROUP_USING_ARG_WOULD_VIOLATE_THE_SAFETY_CONDITION_ARG_REMOVINGCONFIGNODECOUNT_REPLICACOUNT_A64BF23B = + "Cannot remove ConfigNode %d: removing %d ConfigNodes at the same time from a %d-replica ConfigNodeGroup using %s would violate the safety condition %d * removingConfigNodeCount < replicaCount."; public static final String MESSAGE_TRANSFER_CONFIGNODE_LEADER_FAILED_BECAUSE_CAN_NOT_FIND_ANY_RUNNING_1FE4F96D = "Transfer ConfigNode leader failed because can not find any running ConfigNode."; public static final String MESSAGE_CONFIGNODE_REMOVED_LEADER_ALREADY_TRANSFER_LEADER_FA6D1603 = "The ConfigNode to be removed is leader, already transfer Leader to "; public static final String MESSAGE_TARGET_DATANODE_NOT_EXISTED_PLEASE_ENSURE_YOUR_INPUT_QUERYID_CORRECT_AB84CCDF = "The target DataNode is not existed, please ensure your input is correct"; diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index 8c0be448f7f8b..33213f9feb2b0 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -460,6 +460,9 @@ public final class ProcedureMessages { "Cannot remove %d DataNode(s): the cluster has %d available DataNode(s) and must retain at least %d of them (max(schema_replication_factor=%d, data_replication_factor=%d)) so that every region keeps enough replicas, but this request would leave only %d."; public static final String FAILED_TO_REMOVE_DATA_NODE_SINGLE_REPLICA_HINT = " With a single replica there is nowhere to migrate regions to, so at least one DataNode must always remain."; + public static final String + MESSAGE_CANNOT_REMOVE_ARG_REPLICAS_FROM_REGION_ARG_WITH_ARG_REPLICAS_USING_ARG_BECAUSE_THE_REMOVAL_SAFETY_CONDITION_ARG_REMOVEDREPLICACOUNT_REPLICACOUNT_IS_NOT_SATISFIED_4CF41C73 = + "Cannot remove %d replicas from region %s with %d replicas using %s because the removal safety condition %d * removedReplicaCount < replicaCount is not satisfied."; public static final String FAILED_TO_ROLLBACK_ALTER_PIPE_DETAILS_METADATA_WILL_BE_SYNCHRONIZED = "Failed to rollback alter pipe {}, details: {}, metadata will be synchronized later."; public static final String FAILED_TO_ROLLBACK_COMMIT_SET_TEMPLATE_ON_PATH_DUE_TO = diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 8dc9afd1e8bf0..ee1e9d3d1ab39 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -640,6 +640,12 @@ private ManagerMessages() {} public static final String MESSAGE_REMOVE_CONFIGNODE_FAILED_BECAUSE_THERE_ONLY_ONE_CONFIGNODE_CURRENT_CLUSTER_D1273758 = "移除 ConfigNode 失败，原因：当前集群中只有一个 ConfigNode。"; public static final String MESSAGE_REMOVE_CONFIGNODE_FAILED_BECAUSE_THERE_NO_OTHER_CONFIGNODE_RUNNING_STATUS_C9C43315 = "移除 ConfigNode 失败，原因：当前集群中没有其他处于 Running 状态的 ConfigNode。"; public static final String MESSAGE_REMOVE_CONFIGNODE_FAILED_BECAUSE_CONFIGNODEGROUP_LEADER_ELECTION_PLEASE_RETRY_3EE602F6 = "移除 ConfigNode 失败，原因：ConfigNodeGroup 正在进行 leader 选举，请重试。"; + public static final String + MESSAGE_CANNOT_REMOVE_CONFIGNODE_ARG_BECAUSE_IT_IS_ALREADY_BEING_REMOVED_EEE0128D = + "无法移除 ConfigNode %d，因为该节点已处于移除流程中。"; + public static final String + MESSAGE_CANNOT_REMOVE_CONFIGNODE_ARG_REMOVING_ARG_CONFIGNODES_AT_THE_SAME_TIME_FROM_A_ARG_REPLICA_CONFIGNODEGROUP_USING_ARG_WOULD_VIOLATE_THE_SAFETY_CONDITION_ARG_REMOVINGCONFIGNODECOUNT_REPLICACOUNT_A64BF23B = + "无法移除 ConfigNode %d：同时移除 %d 个 ConfigNode（ConfigNodeGroup 当前为 %d 副本并使用 %s）将违反安全条件：%d * removingConfigNodeCount < replicaCount。"; public static final String MESSAGE_TRANSFER_CONFIGNODE_LEADER_FAILED_BECAUSE_CAN_NOT_FIND_ANY_RUNNING_1FE4F96D = "转移 ConfigNode leader 失败，原因：找不到任何正在运行的 ConfigNode。"; public static final String MESSAGE_CONFIGNODE_REMOVED_LEADER_ALREADY_TRANSFER_LEADER_FA6D1603 = "待移除的 ConfigNode 是 leader，已将 Leader 转移到 "; public static final String MESSAGE_TARGET_DATANODE_NOT_EXISTED_PLEASE_ENSURE_YOUR_INPUT_QUERYID_CORRECT_AB84CCDF = "目标 DataNode 不存在，请确保输入的 正确"; diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index 91bf304b3266f..b380b2e530e59 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -446,6 +446,9 @@ public final class ProcedureMessages { "无法移除 %d 个 DataNode：集群当前有 %d 个可用 DataNode，且至少需保留 %d 个（max(schema_replication_factor=%d, data_replication_factor=%d)），以保证每个 Region 仍有足够的副本；但本次请求执行后将只剩 %d 个。"; public static final String FAILED_TO_REMOVE_DATA_NODE_SINGLE_REPLICA_HINT = " 单副本下没有其它节点可供迁移 Region，因此必须始终保留至少一个 DataNode。"; + public static final String + MESSAGE_CANNOT_REMOVE_ARG_REPLICAS_FROM_REGION_ARG_WITH_ARG_REPLICAS_USING_ARG_BECAUSE_THE_REMOVAL_SAFETY_CONDITION_ARG_REMOVEDREPLICACOUNT_REPLICACOUNT_IS_NOT_SATISFIED_4CF41C73 = + "无法移除 %d 个副本：Region %s 当前有 %d 个副本并使用 %s，未满足移除安全条件：%d * removedReplicaCount < replicaCount。"; public static final String FAILED_TO_ROLLBACK_ALTER_PIPE_DETAILS_METADATA_WILL_BE_SYNCHRONIZED = "回滚修改 pipe {} 失败，详情：{}，元数据将稍后同步。"; public static final String FAILED_TO_ROLLBACK_COMMIT_SET_TEMPLATE_ON_PATH_DUE_TO = diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index fee78f5568a04..5ba5ff7000e12 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -1606,7 +1606,7 @@ public TSStatus createPeerForConsensusGroup(List configNode } @Override - public TSStatus removeConfigNode(RemoveConfigNodePlan removeConfigNodePlan) { + public synchronized TSStatus removeConfigNode(RemoveConfigNodePlan removeConfigNodePlan) { TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 8f5b27b7e2fa4..dcf2b00a2edb8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -21,6 +21,7 @@ import org.apache.iotdb.calc.utils.constant.SqlConstant; import org.apache.iotdb.common.rpc.thrift.Model; +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; @@ -88,7 +89,6 @@ import org.apache.iotdb.confignode.procedure.impl.region.CreateRegionGroupsProcedure; import org.apache.iotdb.confignode.procedure.impl.region.ReconstructRegionProcedure; import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrateProcedure; -import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrationPlan; import org.apache.iotdb.confignode.procedure.impl.region.RegionOperationProcedure; import org.apache.iotdb.confignode.procedure.impl.region.RemoveRegionPeerProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.AlterEncodingCompressorProcedure; @@ -666,6 +666,15 @@ public void removeConfigNode(RemoveConfigNodePlan removeConfigNodePlan) { ManagerMessages.SUBMIT_REMOVECONFIGNODEPROCEDURE_SUCCESSFULLY, removeConfigNodePlan); } + public List getRemovingConfigNodes() { + return getExecutor().getProcedures().values().stream() + .filter(procedure -> procedure instanceof RemoveConfigNodeProcedure) + .filter(procedure -> !procedure.isFinished()) + .map(procedure -> ((RemoveConfigNodeProcedure) procedure).getRemovedConfigNode()) + .distinct() + .collect(Collectors.toList()); + } + /** * Generate {@link RemoveDataNodesProcedure}s, and serially execute all the {@link * RemoveDataNodesProcedure}s. @@ -752,26 +761,7 @@ public TSStatus checkRemoveDataNodes(List dataNodeLocations) ((RegionMigrateProcedure) conflictRegionMigrateProcedure.get()).getDestDataNode(), conflictRegionMigrateProcedure.get().getProcId()); } - // 3. Check if the RegionMigrateProcedure generated by RemoveDataNodesProcedure conflicts with - // each other - List regionMigrationPlans = - getEnv().getRemoveDataNodeHandler().getRegionMigrationPlans(dataNodeLocations); - removedDataNodesRegionSet.clear(); - for (RegionMigrationPlan regionMigrationPlan : regionMigrationPlans) { - if (removedDataNodesRegionSet.contains(regionMigrationPlan.getRegionId())) { - failMessage = - String.format( - "Submit RemoveDataNodesProcedure failed, " - + "because the RegionMigrateProcedure generated by this RemoveDataNodesProcedure conflicts with each other. " - + "Only one replica of the same consensus group is allowed to be migrated at the same time." - + "The conflict region id is %s . ", - regionMigrationPlan.getRegionId()); - break; - } - removedDataNodesRegionSet.add(regionMigrationPlan.getRegionId()); - } - - // 4. Check if there are any other unknown or readonly DataNodes in the consensus group that are + // 3. Check if there are any other unknown or readonly DataNodes in the consensus group that are // not the remove DataNodes for (TDataNodeLocation removeDataNode : dataNodeLocations) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 293f1942ff5a9..aa5eedc8bb65e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -101,6 +101,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TRatisConfig; import org.apache.iotdb.confignode.rpc.thrift.TRuntimeConfiguration; import org.apache.iotdb.confignode.rpc.thrift.TSetDataNodeStatusReq; +import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.common.Peer; import org.apache.iotdb.consensus.exception.ConsensusException; @@ -922,14 +923,54 @@ public void applyConfigNode( public TSStatus checkConfigNodeBeforeRemove(RemoveConfigNodePlan removeConfigNodePlan) { removeConfigNodeLock.lock(); try { + final List registeredConfigNodes = getRegisteredConfigNodes(); + // Check ConfigNodes number - if (getRegisteredConfigNodes().size() <= 1) { + if (registeredConfigNodes.size() <= 1) { return new TSStatus(TSStatusCode.REMOVE_CONFIGNODE_ERROR.getStatusCode()) .setMessage( ManagerMessages .MESSAGE_REMOVE_CONFIGNODE_FAILED_BECAUSE_THERE_ONLY_ONE_CONFIGNODE_CURRENT_CLUSTER_D1273758); } + // Check whether the registeredConfigNodes contain the ConfigNode to be removed. + if (!registeredConfigNodes.contains(removeConfigNodePlan.getConfigNodeLocation())) { + return new TSStatus(TSStatusCode.REMOVE_CONFIGNODE_ERROR.getStatusCode()) + .setMessage( + ManagerMessages + .REMOVE_CONFIGNODE_FAILED_BECAUSE_THE_CONFIGNODE_NOT_IN_CURRENT_CLUSTER); + } + + final List removingConfigNodes = + configManager.getProcedureManager().getRemovingConfigNodes().stream() + .filter(registeredConfigNodes::contains) + .collect(Collectors.toList()); + if (removingConfigNodes.contains(removeConfigNodePlan.getConfigNodeLocation())) { + return new TSStatus(TSStatusCode.REMOVE_CONFIGNODE_ERROR.getStatusCode()) + .setMessage( + String.format( + ManagerMessages + .MESSAGE_CANNOT_REMOVE_CONFIGNODE_ARG_BECAUSE_IT_IS_ALREADY_BEING_REMOVED_EEE0128D, + removeConfigNodePlan.getConfigNodeLocation().getConfigNodeId())); + } + + final boolean strongConsistency = + ConsensusFactory.RATIS_CONSENSUS.equals(CONF.getConfigNodeConsensusProtocolClass()); + final int removingConfigNodeCount = removingConfigNodes.size() + 1; + if (!NodeRemovalSafety.isSafe( + removingConfigNodeCount, registeredConfigNodes.size(), strongConsistency)) { + return new TSStatus(TSStatusCode.REMOVE_CONFIGNODE_ERROR.getStatusCode()) + .setMessage( + String.format( + ManagerMessages + .MESSAGE_CANNOT_REMOVE_CONFIGNODE_ARG_REMOVING_ARG_CONFIGNODES_AT_THE_SAME_TIME_FROM_A_ARG_REPLICA_CONFIGNODEGROUP_USING_ARG_WOULD_VIOLATE_THE_SAFETY_CONDITION_ARG_REMOVINGCONFIGNODECOUNT_REPLICACOUNT_A64BF23B, + removeConfigNodePlan.getConfigNodeLocation().getConfigNodeId(), + removingConfigNodeCount, + registeredConfigNodes.size(), + CONF.getConfigNodeConsensusProtocolClass(), + NodeRemovalSafety.getSafetyMultiplier(strongConsistency))); + } + // Check OnlineConfigNodes number final long deadline = System.nanoTime() @@ -951,14 +992,6 @@ public TSStatus checkConfigNodeBeforeRemove(RemoveConfigNodePlan removeConfigNod } } - // Check whether the registeredConfigNodes contain the ConfigNode to be removed. - if (!getRegisteredConfigNodes().contains(removeConfigNodePlan.getConfigNodeLocation())) { - return new TSStatus(TSStatusCode.REMOVE_CONFIGNODE_ERROR.getStatusCode()) - .setMessage( - ManagerMessages - .REMOVE_CONFIGNODE_FAILED_BECAUSE_THE_CONFIGNODE_NOT_IN_CURRENT_CLUSTER); - } - // Check whether the remove ConfigNode is leader TConfigNodeLocation leader = getConsensusManager().getLeaderLocation(); if (leader == null) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeRemovalSafety.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeRemovalSafety.java new file mode 100644 index 0000000000000..cf95f93563745 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeRemovalSafety.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.node; + +/** Safety rules shared by ConfigNode and DataNode removal validation. */ +public final class NodeRemovalSafety { + + private static final int WEAK_CONSISTENCY_MULTIPLIER = 1; + private static final int STRONG_CONSISTENCY_MULTIPLIER = 2; + + private NodeRemovalSafety() {} + + public static boolean isSafe( + int removingReplicaCount, int replicaCount, boolean strongConsistency) { + return (long) getSafetyMultiplier(strongConsistency) * removingReplicaCount < replicaCount; + } + + public static int getSafetyMultiplier(boolean strongConsistency) { + return strongConsistency ? STRONG_CONSISTENCY_MULTIPLIER : WEAK_CONSISTENCY_MULTIPLIER; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java index 9455d34a0ddb4..6e359fca30171 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java @@ -45,6 +45,7 @@ import org.apache.iotdb.confignode.manager.load.balancer.region.IRegionGroupAllocator; import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample; import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample; +import org.apache.iotdb.confignode.manager.node.NodeRemovalSafety; import org.apache.iotdb.confignode.manager.partition.PartitionMetrics; import org.apache.iotdb.confignode.persistence.node.NodeInfo; import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrationPlan; @@ -217,8 +218,9 @@ public List getRegionMigrationPlans( } /** - * Retrieves all region migration plans for the specified removed DataNodes and selects the - * destination. + * Selects one migration plan per affected RegionGroup for the specified removed DataNodes. The + * caller invokes this method again after the returned migrations finish, so multiple replicas of + * the same RegionGroup are migrated in separate rounds. * * @param removedDataNodes the list of DataNodes from which to obtain migration plans * @return a list of region migration plans associated with the removed DataNodes @@ -260,7 +262,7 @@ public List selectMigrationPlans( List allocatedReplicaSets = configManager.getPartitionManager().getAllReplicaSets(consensusGroupType); - // Step 1: Identify affected replica sets and record the removed DataNode for each replica set + // Step 1: Identify affected replica sets and select one removed DataNode for each replica set Map removedNodeMap = new HashMap<>(); Set affectedReplicaSets = identifyAffectedReplicaSets(allocatedReplicaSets, removedDataNodes, removedNodeMap); @@ -318,7 +320,7 @@ public List selectMigrationPlans( /** * Identifies affected replica sets from allocatedReplicaSets that contain any DataNode in - * removedDataNodes, and records the removed DataNode for each replica set. + * removedDataNodes, and selects one removed DataNode for each replica set in this round. */ private Set identifyAffectedReplicaSets( List allocatedReplicaSets, @@ -334,7 +336,7 @@ private Set identifyAffectedReplicaSets( .filter(replicaSet -> replicaSet.getDataNodeLocations().contains(removedNode)) .forEach( replicaSet -> { - removedNodeMap.put(replicaSet.getRegionId(), removedNode); + removedNodeMap.putIfAbsent(replicaSet.getRegionId(), removedNode); affectedReplicaSets.add(replicaSet); }); } @@ -621,6 +623,41 @@ public TSStatus checkRegionReplication(RemoveDataNodePlan removeDataNodePlan) { message += ProcedureMessages.FAILED_TO_REMOVE_DATA_NODE_SINGLE_REPLICA_HINT; } status.setMessage(message); + return status; + } + + final Set removedDataNodeIds = + removedDataNodes.stream().map(TDataNodeLocation::getDataNodeId).collect(Collectors.toSet()); + for (TRegionReplicaSet replicaSet : configManager.getPartitionManager().getAllReplicaSets()) { + final int removedReplicaCount = + (int) + replicaSet.getDataNodeLocations().stream() + .filter(location -> removedDataNodeIds.contains(location.getDataNodeId())) + .count(); + if (removedReplicaCount == 0) { + continue; + } + + final boolean strongConsistency = + CONF.isConsensusGroupStrongConsistency(replicaSet.getRegionId()); + final int replicaCount = replicaSet.getDataNodeLocationsSize(); + if (!NodeRemovalSafety.isSafe(removedReplicaCount, replicaCount, strongConsistency)) { + final String consensusProtocolClass = + TConsensusGroupType.SchemaRegion.equals(replicaSet.getRegionId().getType()) + ? CONF.getSchemaRegionConsensusProtocolClass() + : CONF.getDataRegionConsensusProtocolClass(); + status.setCode(TSStatusCode.REMOVE_DATANODE_ERROR.getStatusCode()); + status.setMessage( + String.format( + ProcedureMessages + .MESSAGE_CANNOT_REMOVE_ARG_REPLICAS_FROM_REGION_ARG_WITH_ARG_REPLICAS_USING_ARG_BECAUSE_THE_REMOVAL_SAFETY_CONDITION_ARG_REMOVEDREPLICACOUNT_REPLICACOUNT_IS_NOT_SATISFIED_4CF41C73, + removedReplicaCount, + replicaSet.getRegionId(), + replicaCount, + consensusProtocolClass, + NodeRemovalSafety.getSafetyMultiplier(strongConsistency))); + return status; + } } return status; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveConfigNodeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveConfigNodeProcedure.java index bbd408fac8b6e..dd85b42cf133e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveConfigNodeProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveConfigNodeProcedure.java @@ -153,4 +153,8 @@ public boolean equals(Object that) { public int hashCode() { return Objects.hash(this.removedConfigNode); } + + public TConfigNodeLocation getRemovedConfigNode() { + return removedConfigNode; + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedure.java index 400d8f8facd66..670867196d665 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedure.java @@ -125,12 +125,6 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, RemoveDataNodeState removedDataNodes.forEach( dataNode -> removedNodeStatusMap.put(dataNode.getDataNodeId(), NodeStatus.Removing)); removeDataNodeHandler.changeDataNodeStatus(removedDataNodes, removedNodeStatusMap); - regionMigrationPlans = - removeDataNodeHandler.selectedRegionMigrationPlans(removedDataNodes); - LOG.info( - ProcedureMessages.LOG_ARG_DATANODE_REGIONS_REMOVED_ARG_216A7DC7, - REMOVE_DATANODE_PROCESS, - regionMigrationPlans); setNextState(RemoveDataNodeState.BROADCAST_DISABLE_DATA_NODE); break; case BROADCAST_DISABLE_DATA_NODE: @@ -138,11 +132,22 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, RemoveDataNodeState setNextState(RemoveDataNodeState.SUBMIT_REGION_MIGRATE); break; case SUBMIT_REGION_MIGRATE: - // Avoid re-submit region-migration when leader change or ConfigNode reboot - if (!isStateDeserialized()) { + // Generate plans from the latest partition table on every round. At most one replica of + // each RegionGroup is migrated in a round, while different RegionGroups are migrated in + // parallel. Recomputing also makes recovery idempotent: completed migrations disappear + // from the next plan, and persisted child procedures keep the parent waiting. + regionMigrationPlans = + removeDataNodeHandler.selectedRegionMigrationPlans(removedDataNodes); + if (regionMigrationPlans.isEmpty()) { + setNextState(RemoveDataNodeState.STOP_DATA_NODE); + } else { + LOG.info( + ProcedureMessages.LOG_ARG_DATANODE_REGIONS_REMOVED_ARG_216A7DC7, + REMOVE_DATANODE_PROCESS, + regionMigrationPlans); submitChildRegionMigrate(env); + setNextState(RemoveDataNodeState.SUBMIT_REGION_MIGRATE); } - setNextState(RemoveDataNodeState.STOP_DATA_NODE); break; case STOP_DATA_NODE: checkRegionStatusAndStopDataNode(env); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java index 4de883b9c158f..4bb885c90404b 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java @@ -35,7 +35,6 @@ import org.apache.iotdb.confignode.procedure.env.RemoveDataNodeHandler; import org.apache.iotdb.confignode.procedure.impl.node.RemoveDataNodesProcedure; import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrateProcedure; -import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrationPlan; import org.junit.Assert; import org.junit.BeforeClass; @@ -87,15 +86,6 @@ public class ProcedureManagerTest { new TEndPoint("127.0.0.1", 6680), new TEndPoint("127.0.0.1", 6681)); - private final TDataNodeLocation toDataNodeLocation = - new TDataNodeLocation( - 12, - new TEndPoint("127.0.0.1", 6687), - new TEndPoint("127.0.0.1", 6688), - new TEndPoint("127.0.0.1", 6689), - new TEndPoint("127.0.0.1", 6690), - new TEndPoint("127.0.0.1", 6691)); - private final TDataNodeLocation coordinatorDataNodeLocation = new TDataNodeLocation( 13, @@ -163,24 +153,6 @@ public void testCheckRemoveDataNodeWithConflictRegionMigrateProcedure() { Assert.assertTrue(isFailed(status)); } - @Test - public void testCheckRemoveDataNodeWithRegionMigrateProcedureConflictsWithEachOther() { - RegionMigrationPlan regionMigrationPlanA = - new RegionMigrationPlan(consensusGroupId, removeDataNodeLocationA); - regionMigrationPlanA.setToDataNode(toDataNodeLocation); - RegionMigrationPlan regionMigrationPlanB = - new RegionMigrationPlan(consensusGroupId, removeDataNodeLocationB); - regionMigrationPlanB.setToDataNode(toDataNodeLocation); - - List regionMigrationPlans = - new ArrayList<>(Arrays.asList(regionMigrationPlanA, regionMigrationPlanB)); - when(REMOVE_DATA_NODE_HANDLER.getRegionMigrationPlans(removedDataNodes)) - .thenReturn(regionMigrationPlans); - - TSStatus status = PROCEDURE_MANAGER.checkRemoveDataNodes(removedDataNodes); - Assert.assertTrue(isFailed(status)); - } - @Test public void testCheckRemoveDataNodeWithAnotherUnknownDataNode() { Set relatedDataNodes = new HashSet<>(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeRemovalSafetyTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeRemovalSafetyTest.java new file mode 100644 index 0000000000000..58e71b2826b43 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeRemovalSafetyTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.node; + +import org.junit.Assert; +import org.junit.Test; + +public class NodeRemovalSafetyTest { + + @Test + public void testWeakConsistencyRemovalSafety() { + Assert.assertFalse(NodeRemovalSafety.isSafe(1, 1, false)); + Assert.assertTrue(NodeRemovalSafety.isSafe(1, 2, false)); + Assert.assertTrue(NodeRemovalSafety.isSafe(2, 3, false)); + Assert.assertFalse(NodeRemovalSafety.isSafe(3, 3, false)); + } + + @Test + public void testStrongConsistencyRemovalSafety() { + Assert.assertFalse(NodeRemovalSafety.isSafe(1, 1, true)); + Assert.assertFalse(NodeRemovalSafety.isSafe(1, 2, true)); + Assert.assertTrue(NodeRemovalSafety.isSafe(1, 3, true)); + Assert.assertTrue(NodeRemovalSafety.isSafe(1, 4, true)); + Assert.assertFalse(NodeRemovalSafety.isSafe(2, 4, true)); + Assert.assertTrue(NodeRemovalSafety.isSafe(2, 5, true)); + Assert.assertFalse(NodeRemovalSafety.isSafe(3, 5, true)); + } +}