diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/PipeSchemaRegionSnapshotRecoveryIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/PipeSchemaRegionSnapshotRecoveryIT.java new file mode 100644 index 0000000000000..c327b2d806d11 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/PipeSchemaRegionSnapshotRecoveryIT.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.pipe.it.single; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.consensus.ConsensusGroupId; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.consensus.IConsensus; +import org.apache.iotdb.consensus.common.Peer; +import org.apache.iotdb.consensus.config.ConsensusConfig; +import org.apache.iotdb.consensus.config.RatisConfig; +import org.apache.iotdb.db.consensus.statemachine.schemaregion.SchemaRegionStateMachine; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; +import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeOperateSchemaQueueNode; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.apache.ratis.util.ExitUtils; +import org.apache.ratis.util.TimeDuration; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.awaitility.Awaitility; +import org.junit.After; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Proxy; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +@Category({LocalStandaloneIT.class}) +public class PipeSchemaRegionSnapshotRecoveryIT { + + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @After + public void clearRatisExitState() { + ExitUtils.clear(); + } + + @Test + public void testInstallingSnapshotWithoutListenerClosesStaleQueue() throws Exception { + ExitUtils.disableSystemExit(); + + final SchemaRegionId schemaRegionId = new SchemaRegionId(Integer.MAX_VALUE - 17666); + final int leaderPort = freePort(); + int followerPort = freePort(); + while (followerPort == leaderPort) { + followerPort = freePort(); + } + final Peer leaderPeer = new Peer(schemaRegionId, 0, new TEndPoint("127.0.0.1", leaderPort)); + final Peer followerPeer = new Peer(schemaRegionId, 1, new TEndPoint("127.0.0.1", followerPort)); + final CountDownLatch leaderSnapshotCreated = new CountDownLatch(1); + final CountDownLatch followerSnapshotLoaded = new CountDownLatch(1); + final AtomicReference leaderSnapshotDir = new AtomicReference<>(); + final AtomicBoolean followerSnapshotLoadSucceeded = new AtomicBoolean(); + final AtomicReference staleQueueRef = new AtomicReference<>(); + final ISchemaRegion schemaRegion = + newSchemaRegion(schemaRegionId, leaderSnapshotDir, leaderSnapshotCreated); + + final RatisConfig ratisConfig = + RatisConfig.newBuilder() + .setRpc( + RatisConfig.Rpc.newBuilder() + .setFirstElectionTimeoutMin(TimeDuration.valueOf(1, TimeUnit.SECONDS)) + .setFirstElectionTimeoutMax(TimeDuration.valueOf(4, TimeUnit.SECONDS)) + .setTimeoutMin(TimeDuration.valueOf(1, TimeUnit.SECONDS)) + .setTimeoutMax(TimeDuration.valueOf(4, TimeUnit.SECONDS)) + .build()) + .setLog(RatisConfig.Log.newBuilder().setPurgeUptoSnapshotIndex(true).build()) + .build(); + final IConsensus leader = + newConsensus( + leaderPeer, + ratisConfig, + schemaRegion, + new File(temporaryFolder.getRoot(), "leader"), + null, + null); + final IConsensus follower = + newConsensus( + followerPeer, + ratisConfig, + schemaRegion, + new File(temporaryFolder.getRoot(), "follower"), + followerSnapshotLoaded, + followerSnapshotLoadSucceeded); + boolean leaderStarted = false; + boolean followerStarted = false; + + try { + leader.start(); + leaderStarted = true; + follower.start(); + followerStarted = true; + + leader.createLocalPeer(schemaRegionId, Collections.singletonList(leaderPeer)); + Awaitility.await() + .atMost(30, TimeUnit.SECONDS) + .until(() -> leader.isLeaderReady(schemaRegionId)); + + for (int i = 0; i < 10; i++) { + Assert.assertEquals( + 200, + leader + .write( + schemaRegionId, + new PipeOperateSchemaQueueNode(new PlanNodeId("close" + i), false)) + .getCode()); + } + leader.triggerSnapshot(schemaRegionId, true); + Assert.assertTrue(leaderSnapshotCreated.await(30, TimeUnit.SECONDS)); + Assert.assertFalse(SchemaRegionListeningQueue.hasSnapshot(leaderSnapshotDir.get())); + + final SchemaRegionListeningQueue staleQueue = + PipeDataNodeAgent.runtime().schemaListener(schemaRegionId); + staleQueueRef.set(staleQueue); + PipeDataNodeAgent.runtime().increaseAndGetSchemaListenerReferenceCount(schemaRegionId); + staleQueue.open(); + staleQueue.tryListenToNode( + new CreateTimeSeriesNode( + new PlanNodeId("stale"), + new MeasurementPath("root.db.d1.s1"), + TSDataType.INT32, + TSEncoding.PLAIN, + CompressionType.GZIP, + null, + null, + null, + "alias")); + Assert.assertEquals(1, staleQueue.getSize()); + + follower.createLocalPeer(schemaRegionId, Collections.emptyList()); + leader.addRemotePeer(schemaRegionId, followerPeer); + + Assert.assertTrue(followerSnapshotLoaded.await(30, TimeUnit.SECONDS)); + Assert.assertTrue(followerSnapshotLoadSucceeded.get()); + Assert.assertFalse(staleQueue.isOpened()); + Assert.assertEquals(0, staleQueue.getSize()); + Assert.assertSame( + staleQueue, PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + Assert.assertEquals( + 0, + PipeDataNodeAgent.runtime().decreaseAndGetSchemaListenerReferenceCount(schemaRegionId)); + Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + } finally { + if (followerStarted) { + follower.stop(); + } + if (leaderStarted) { + leader.stop(); + } + if (staleQueueRef.get() != null) { + staleQueueRef.get().close(); + } + PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId); + } + } + + private ISchemaRegion newSchemaRegion( + final SchemaRegionId schemaRegionId, + final AtomicReference snapshotDir, + final CountDownLatch snapshotCreated) { + return (ISchemaRegion) + Proxy.newProxyInstance( + ISchemaRegion.class.getClassLoader(), + new Class[] {ISchemaRegion.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "getSchemaRegionId": + return schemaRegionId; + case "createSnapshot": + final File dir = (File) args[0]; + Files.createDirectories(dir.toPath()); + Files.writeString(new File(dir, "schema.snapshot").toPath(), "schema"); + snapshotDir.set(dir); + snapshotCreated.countDown(); + return true; + case "loadSnapshot": + return new File((File) args[0], "schema.snapshot").isFile(); + default: + throw new UnsupportedOperationException(method.toString()); + } + }); + } + + private IConsensus newConsensus( + final Peer peer, + final RatisConfig ratisConfig, + final ISchemaRegion schemaRegion, + final File storageDir, + final CountDownLatch snapshotLoaded, + final AtomicBoolean snapshotLoadSucceeded) { + return ConsensusFactory.getConsensusImpl( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusConfig.newBuilder() + .setThisNodeId(peer.getNodeId()) + .setThisNode(peer.getEndpoint()) + .setRatisConfig(ratisConfig) + .setStorageDir(storageDir.getAbsolutePath()) + .setConsensusGroupType(TConsensusGroupType.SchemaRegion) + .build(), + ignored -> + new QuietSchemaRegionStateMachine( + schemaRegion, snapshotLoaded, snapshotLoadSucceeded)) + .orElseThrow(IllegalStateException::new); + } + + private int freePort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } + + private static class QuietSchemaRegionStateMachine extends SchemaRegionStateMachine { + + private final CountDownLatch snapshotLoaded; + private final AtomicBoolean snapshotLoadSucceeded; + + private QuietSchemaRegionStateMachine( + final ISchemaRegion schemaRegion, + final CountDownLatch snapshotLoaded, + final AtomicBoolean snapshotLoadSucceeded) { + super(schemaRegion); + this.snapshotLoaded = snapshotLoaded; + this.snapshotLoadSucceeded = snapshotLoadSucceeded; + } + + @Override + public boolean loadSnapshot(final File snapshotDir) { + final boolean succeeded = super.loadSnapshot(snapshotDir); + if (snapshotLoaded != null) { + snapshotLoadSucceeded.set(succeeded); + snapshotLoaded.countDown(); + } + return succeeded; + } + + @Override + public void notifyLeaderChanged(final ConsensusGroupId groupId, final int newLeaderId) {} + + @Override + public void notifyLeaderReady() {} + + @Override + public void notifyNotLeader() {} + + @Override + public void stop() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index a77b0895e4c0b..6a92d2c51c314 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -850,13 +850,20 @@ public TSStatus visitPipeEnrichedNonWritePlanNode( public TSStatus visitPipeOperateSchemaQueueNode( final PipeOperateSchemaQueueNode node, final ISchemaRegion schemaRegion) { final SchemaRegionId id = schemaRegion.getSchemaRegionId(); - final SchemaRegionListeningQueue queue = PipeDataNodeAgent.runtime().schemaListener(id); - if (node.isOpen() && !queue.isOpened()) { - logger.info(DataNodeMiscMessages.OPENED_PIPE_LISTENING_QUEUE, id); - queue.open(); - } else if (!node.isOpen() && queue.isOpened()) { - logger.info(DataNodeMiscMessages.CLOSED_PIPE_LISTENING_QUEUE, id); - queue.close(); + if (node.isOpen()) { + final SchemaRegionListeningQueue queue = PipeDataNodeAgent.runtime().schemaListener(id); + if (!queue.isOpened()) { + logger.info(DataNodeMiscMessages.OPENED_PIPE_LISTENING_QUEUE, id); + queue.open(); + } + } else { + final SchemaRegionListeningQueue queue = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(id); + if (queue != null && queue.isOpened()) { + logger.info(DataNodeMiscMessages.CLOSED_PIPE_LISTENING_QUEUE, id); + queue.close(); + PipeDataNodeAgent.runtime().cleanupSchemaListenerIfUnused(id); + } } return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java index 9396d8cc35438..38e04e7769bc5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java @@ -128,14 +128,21 @@ public boolean isReadOnly() { @Override public boolean takeSnapshot(final File snapshotDir) { - if (schemaRegion.createSnapshot(snapshotDir) - && PipeDataNodeAgent.runtime() - .schemaListener(schemaRegion.getSchemaRegionId()) - .createSnapshot(snapshotDir)) { - listen2Snapshot4PipeListener(true); + if (!schemaRegion.createSnapshot(snapshotDir)) { + return false; + } + + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId()); + if (listener == null) { return true; } - return false; + + if (!listener.createSnapshot(snapshotDir)) { + return false; + } + listen2Snapshot4PipeListener(true); + return true; } @Override @@ -145,12 +152,16 @@ public boolean loadSnapshot(final File latestSnapshotRootDir) { // callers (e.g. the AddPeer flow and the Ratis snapshot-install path) can detect a real // failure instead of treating a fallback-to-empty load as success. final boolean loadSucceeded = schemaRegion.loadSnapshot(latestSnapshotRootDir); - PipeDataNodeAgent.runtime() - .schemaListener(schemaRegion.getSchemaRegionId()) - .loadSnapshot(latestSnapshotRootDir); - // We recompute the snapshot for pipe listener when loading snapshot - // to recover the newest snapshot in cache - listen2Snapshot4PipeListener(false); + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime() + .schemaListenerForSnapshotLoad( + schemaRegion.getSchemaRegionId(), latestSnapshotRootDir); + if (listener != null) { + listener.loadSnapshot(latestSnapshotRootDir); + // We recompute the snapshot for pipe listener when loading snapshot + // to recover the newest snapshot in cache + listen2Snapshot4PipeListener(false); + } return loadSucceeded; } catch (Exception e) { logger.error( @@ -169,7 +180,10 @@ public void listen2Snapshot4PipeListener(final boolean isTmp) { .toString(), isTmp); final SchemaRegionListeningQueue listener = - PipeDataNodeAgent.runtime().schemaListener(schemaRegion.getSchemaRegionId()); + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId()); + if (listener == null) { + return; + } if (Objects.isNull(snapshotPaths) || Objects.isNull(snapshotPaths.get(0))) { if (listener.isOpened()) { logger.warn( @@ -197,9 +211,11 @@ public TSStatus write(final IConsensusRequest request) { final TSStatus result = ((PlanNode) request).accept(new SchemaExecutionVisitor(), schemaRegion); if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - PipeDataNodeAgent.runtime() - .schemaListener(schemaRegion.getSchemaRegionId()) - .tryListenToNode((PlanNode) request); + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId()); + if (listener != null) { + listener.tryListenToNode((PlanNode) request); + } } return result; } catch (final IllegalArgumentException e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java index 8ca2ee1f52f64..cf29cb7a93347 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java @@ -57,6 +57,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; @@ -174,6 +175,15 @@ public SchemaRegionListeningQueue schemaListener(SchemaRegionId schemaRegionId) return regionListenerManager.listener(schemaRegionId); } + public SchemaRegionListeningQueue schemaListenerIfPresent(SchemaRegionId schemaRegionId) { + return regionListenerManager.listenerIfPresent(schemaRegionId); + } + + public SchemaRegionListeningQueue schemaListenerForSnapshotLoad( + SchemaRegionId schemaRegionId, File snapshotDir) { + return regionListenerManager.listenerForSnapshotLoad(schemaRegionId, snapshotDir); + } + public int increaseAndGetSchemaListenerReferenceCount(SchemaRegionId schemaRegionId) { return regionListenerManager.increaseAndGetReferenceCount(schemaRegionId); } @@ -182,6 +192,14 @@ public int decreaseAndGetSchemaListenerReferenceCount(SchemaRegionId schemaRegio return regionListenerManager.decreaseAndGetReferenceCount(schemaRegionId); } + public void cleanupSchemaListenerIfUnused(SchemaRegionId schemaRegionId) { + regionListenerManager.cleanupListenerIfUnused(schemaRegionId); + } + + public void clearSchemaRegionState(SchemaRegionId schemaRegionId) { + regionListenerManager.clearSchemaRegionState(schemaRegionId); + } + public void notifySchemaLeaderReady(SchemaRegionId schemaRegionId) { regionListenerManager.notifyLeaderReady(schemaRegionId); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java index d28929b36a216..61a904c339cc6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java @@ -20,10 +20,10 @@ package org.apache.iotdb.db.pipe.agent.runtime; import org.apache.iotdb.commons.consensus.SchemaRegionId; -import org.apache.iotdb.commons.pipe.agent.task.PipeTask; import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionListenerMetrics; import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; +import java.io.File; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -34,6 +34,7 @@ public class PipeSchemaRegionListenerManager { private final Map id2ListenerMap = new ConcurrentHashMap<>(); + private final Map id2LeaderReadyMap = new ConcurrentHashMap<>(); public synchronized Set regionIds() { return id2ListenerMap.keySet(); @@ -44,6 +45,27 @@ public synchronized SchemaRegionListeningQueue listener(final SchemaRegionId sch .listeningQueue; } + public synchronized SchemaRegionListeningQueue listenerIfPresent( + final SchemaRegionId schemaRegionId) { + final PipeSchemaRegionListener listener = id2ListenerMap.get(schemaRegionId); + return listener == null ? null : listener.listeningQueue; + } + + public synchronized SchemaRegionListeningQueue listenerForSnapshotLoad( + final SchemaRegionId schemaRegionId, final File snapshotDir) { + final PipeSchemaRegionListener listener = id2ListenerMap.get(schemaRegionId); + if (!SchemaRegionListeningQueue.hasSnapshot(snapshotDir)) { + // The snapshot can replace a follower that missed the queue-close log entry. Absence of a + // queue snapshot represents a closed queue, so discard the follower's previous contents. + if (listener != null) { + listener.listeningQueue.close(); + cleanupListenerIfUnused(schemaRegionId, listener); + } + return null; + } + return listener == null ? listener(schemaRegionId) : listener.listeningQueue; + } + public synchronized int increaseAndGetReferenceCount(final SchemaRegionId schemaRegionId) { return id2ListenerMap .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new) @@ -52,28 +74,55 @@ public synchronized int increaseAndGetReferenceCount(final SchemaRegionId schema } public synchronized int decreaseAndGetReferenceCount(final SchemaRegionId schemaRegionId) { - return id2ListenerMap - .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new) - .listeningQueueReferenceCount - .updateAndGet(v -> v > 0 ? v - 1 : 0); + final PipeSchemaRegionListener listener = id2ListenerMap.get(schemaRegionId); + if (listener == null) { + return 0; + } + + final int referenceCount = + listener.listeningQueueReferenceCount.updateAndGet(v -> v > 0 ? v - 1 : 0); + if (referenceCount == 0 && !listener.listeningQueue.isOpened()) { + cleanupListenerIfUnused(schemaRegionId, listener); + } + return referenceCount; } public synchronized void notifyLeaderReady(final SchemaRegionId schemaRegionId) { - id2ListenerMap - .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new) - .notifyLeaderReady(); + id2LeaderReadyMap.computeIfAbsent(schemaRegionId, id -> new AtomicBoolean()).set(true); } public synchronized void notifyLeaderUnavailable(final SchemaRegionId schemaRegionId) { - id2ListenerMap - .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new) - .notifyLeaderUnavailable(); + id2LeaderReadyMap.computeIfAbsent(schemaRegionId, id -> new AtomicBoolean()).set(false); } public synchronized boolean isLeaderReady(final SchemaRegionId schemaRegionId) { - return id2ListenerMap - .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new) - .isLeaderReady(); + final AtomicBoolean isLeaderReady = id2LeaderReadyMap.get(schemaRegionId); + return isLeaderReady != null && isLeaderReady.get(); + } + + public synchronized void cleanupListenerIfUnused(final SchemaRegionId schemaRegionId) { + final PipeSchemaRegionListener listener = id2ListenerMap.get(schemaRegionId); + if (listener != null) { + cleanupListenerIfUnused(schemaRegionId, listener); + } + } + + public synchronized void clearSchemaRegionState(final SchemaRegionId schemaRegionId) { + final PipeSchemaRegionListener listener = id2ListenerMap.remove(schemaRegionId); + if (listener != null) { + PipeSchemaRegionListenerMetrics.getInstance().deregister(schemaRegionId.getId()); + } + id2LeaderReadyMap.remove(schemaRegionId); + } + + private void cleanupListenerIfUnused( + final SchemaRegionId schemaRegionId, final PipeSchemaRegionListener listener) { + if (listener.listeningQueueReferenceCount.get() > 0 || listener.listeningQueue.isOpened()) { + return; + } + if (id2ListenerMap.remove(schemaRegionId, listener)) { + PipeSchemaRegionListenerMetrics.getInstance().deregister(schemaRegionId.getId()); + } } private static class PipeSchemaRegionListener { @@ -81,33 +130,9 @@ private static class PipeSchemaRegionListener { private final SchemaRegionListeningQueue listeningQueue = new SchemaRegionListeningQueue(); private final AtomicInteger listeningQueueReferenceCount = new AtomicInteger(0); - private final AtomicBoolean isLeaderReady = new AtomicBoolean(false); - protected PipeSchemaRegionListener(final SchemaRegionId schemaRegionId) { PipeSchemaRegionListenerMetrics.getInstance() .register(listeningQueue, schemaRegionId.getId()); } - - /** - * Get leader ready state, DO NOT use consensus layer's leader ready flag because - * SimpleConsensus' ready flag is always {@code true}. Note that this flag has nothing to do - * with listening and a {@link PipeTask} starts only iff the current node is a leader and ready. - * - * @return {@code true} iff the current node is a leader and ready - */ - private boolean isLeaderReady() { - return isLeaderReady.get(); - } - - // Leader ready flag has the following effect - // 1. The linked list starts serving only after leader gets ready - // 2. Config pipe task is only created after leader gets ready - private void notifyLeaderReady() { - isLeaderReady.set(true); - } - - private void notifyLeaderUnavailable() { - isLeaderReady.set(false); - } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 359bdd79c50ef..1b54f1a0742cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -69,6 +69,7 @@ import org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener; import org.apache.iotdb.db.pipe.source.schemaregion.IoTDBSchemaRegionSource; import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningFilter; +import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; @@ -351,10 +352,14 @@ private Set clearSchemaRegionListeningQueueIfNecessary( } schemaRegionId2ListeningQueueNewFirstIndex.forEach( - (schemaRegionId, listeningQueueNewFirstIndex) -> - PipeDataNodeAgent.runtime() - .schemaListener(new SchemaRegionId(schemaRegionId)) - .removeBefore(listeningQueueNewFirstIndex)); + (schemaRegionId, listeningQueueNewFirstIndex) -> { + final SchemaRegionListeningQueue listeningQueue = + PipeDataNodeAgent.runtime() + .schemaListenerIfPresent(new SchemaRegionId(schemaRegionId)); + if (listeningQueue != null) { + listeningQueue.removeBefore(listeningQueueNewFirstIndex); + } + }); return schemaRegionId2ListeningQueueNewFirstIndex.keySet(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java index 10c48f7d9eaac..92d8499fa0511 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java @@ -162,12 +162,14 @@ public synchronized void unbindFrom(final AbstractMetricService metricService) { } private void removeMetrics(final String pipeID) { - removeAutoGauge(pipeID); - } - - private void removeAutoGauge(final String pipeID) { final PipeDataNodeRemainingEventAndTimeOperator operator = remainingEventAndTimeOperatorMap.get(pipeID); + if (Objects.nonNull(operator) && Objects.nonNull(metricService)) { + removeAutoGauge(operator); + } + } + + private void removeAutoGauge(final PipeDataNodeRemainingEventAndTimeOperator operator) { metricService.remove( MetricType.AUTO_GAUGE, Metric.PIPE_DATANODE_REMAINING_EVENT_COUNT.toString(), @@ -182,7 +184,8 @@ private void removeAutoGauge(final String pipeID) { operator.getPipeName(), Tag.CREATION_TIME.toString(), String.valueOf(operator.getCreationTime())); - if (completionOperatorMap.containsKey(pipeID)) { + if (completionOperatorMap.containsKey( + generatePipeID(operator.getPipeName(), operator.getCreationTime()))) { metricService.remove( MetricType.AUTO_GAUGE, Metric.PIPE_DATANODE_COMPLETION_READY.toString(), @@ -226,9 +229,40 @@ private void removeAutoGauge(final String pipeID) { //////////////////////////// register & deregister (pipe integration) //////////////////////////// + private static String generatePipeID(final String pipeName, final long creationTime) { + return pipeName + "_" + creationTime; + } + + private boolean isPipeAlive(final String pipeName, final long creationTime) { + return PipeDataNodeAgent.task().isPipeAlive(pipeName, creationTime); + } + + private PipeDataNodeRemainingEventAndTimeOperator getOrCreateOperatorIfPipeAlive( + final String pipeName, final long creationTime) { + final String pipeID = generatePipeID(pipeName, creationTime); + PipeDataNodeRemainingEventAndTimeOperator operator = + remainingEventAndTimeOperatorMap.get(pipeID); + if (Objects.nonNull(operator)) { + return operator; + } + + // Serialize creation with deregistration so a late update cannot restore a removed operator. + synchronized (this) { + operator = remainingEventAndTimeOperatorMap.get(pipeID); + if (Objects.nonNull(operator)) { + return operator; + } + if (!isPipeAlive(pipeName, creationTime)) { + return null; + } + return remainingEventAndTimeOperatorMap.computeIfAbsent( + pipeID, key -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)); + } + } + public synchronized void register(final IoTDBDataRegionSource source) { // The metric is global thus the regionId is omitted - final String pipeID = source.getPipeName() + "_" + source.getCreationTime(); + final String pipeID = generatePipeID(source.getPipeName(), source.getCreationTime()); final PipeDataNodeRemainingEventAndTimeOperator remainingOperator = remainingEventAndTimeOperatorMap.computeIfAbsent( pipeID, @@ -318,7 +352,7 @@ public void markDataRegionCompletionInvalid( public synchronized void register(final IoTDBSchemaRegionSource source) { // The metric is global thus the regionId is omitted - final String pipeID = source.getPipeName() + "_" + source.getCreationTime(); + final String pipeID = generatePipeID(source.getPipeName(), source.getCreationTime()); remainingEventAndTimeOperatorMap .computeIfAbsent( pipeID, @@ -332,19 +366,20 @@ public synchronized void register(final IoTDBSchemaRegionSource source) { } public void increaseInsertNodeEventCount(final String pipeName, final long creationTime) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .increaseInsertNodeEventCount(); + final PipeDataNodeRemainingEventAndTimeOperator operator = + getOrCreateOperatorIfPipeAlive(pipeName, creationTime); + if (Objects.nonNull(operator)) { + operator.increaseInsertNodeEventCount(); + } } public void decreaseInsertNodeEventCount( final String pipeName, final long creationTime, final long transferTime) { final PipeDataNodeRemainingEventAndTimeOperator operator = - remainingEventAndTimeOperatorMap.computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)); + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.isNull(operator)) { + return; + } operator.decreaseInsertNodeEventCount(); @@ -355,46 +390,44 @@ public void decreaseInsertNodeEventCount( public void updateInsertNodeTransferTimer( final String pipeName, final long creationTime, final long transferTime) { - if (transferTime > 0) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .getInsertNodeTransferTimer() - .update(transferTime, TimeUnit.NANOSECONDS); + final PipeDataNodeRemainingEventAndTimeOperator operator = + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.nonNull(operator) && transferTime > 0) { + operator.getInsertNodeTransferTimer().update(transferTime, TimeUnit.NANOSECONDS); } } public void increaseRawTabletEventCount(final String pipeName, final long creationTime) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .increaseRawTabletEventCount(); + final PipeDataNodeRemainingEventAndTimeOperator operator = + getOrCreateOperatorIfPipeAlive(pipeName, creationTime); + if (Objects.nonNull(operator)) { + operator.increaseRawTabletEventCount(); + } } public void decreaseRawTabletEventCount(final String pipeName, final long creationTime) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .decreaseRawTabletEventCount(); + final PipeDataNodeRemainingEventAndTimeOperator operator = + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.nonNull(operator)) { + operator.decreaseRawTabletEventCount(); + } } public void increaseTsFileEventCount(final String pipeName, final long creationTime) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .increaseTsFileEventCount(); + final PipeDataNodeRemainingEventAndTimeOperator operator = + getOrCreateOperatorIfPipeAlive(pipeName, creationTime); + if (Objects.nonNull(operator)) { + operator.increaseTsFileEventCount(); + } } public void decreaseTsFileEventCount( final String pipeName, final long creationTime, final long transferTime) { final PipeDataNodeRemainingEventAndTimeOperator operator = - remainingEventAndTimeOperatorMap.computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)); + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.isNull(operator)) { + return; + } operator.decreaseTsFileEventCount(); @@ -405,28 +438,25 @@ public void decreaseTsFileEventCount( public void updateTsFileTransferTimer( final String pipeName, final long creationTime, final long transferTime) { - if (transferTime > 0) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .getTsFileTransferTimer() - .update(transferTime, TimeUnit.NANOSECONDS); + final PipeDataNodeRemainingEventAndTimeOperator operator = + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.nonNull(operator) && transferTime > 0) { + operator.getTsFileTransferTimer().update(transferTime, TimeUnit.NANOSECONDS); } } public void increaseHeartbeatEventCount(final String pipeName, final long creationTime) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .increaseHeartbeatEventCount(); + final PipeDataNodeRemainingEventAndTimeOperator operator = + getOrCreateOperatorIfPipeAlive(pipeName, creationTime); + if (Objects.nonNull(operator)) { + operator.increaseHeartbeatEventCount(); + } } public void decreaseHeartbeatEventCount(final String pipeName, final long creationTime) { final PipeDataNodeRemainingEventAndTimeOperator operator = - remainingEventAndTimeOperatorMap.get(pipeName + "_" + creationTime); - if (operator != null) { + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.nonNull(operator)) { operator.decreaseHeartbeatEventCount(); } } @@ -452,12 +482,16 @@ public void freezeRate(final String pipeID) { } public synchronized void deregister(final String pipeID) { - if (!remainingEventAndTimeOperatorMap.containsKey(pipeID)) { + final PipeDataNodeRemainingEventAndTimeOperator operator = + remainingEventAndTimeOperatorMap.remove(pipeID); + if (Objects.isNull(operator)) { LOGGER.warn(DataNodePipeMessages.FAILED_TO_DEREGISTER_PIPE_REMAINING_EVENT_AND, pipeID); - } else if (Objects.nonNull(metricService)) { - removeMetrics(pipeID); + completionOperatorMap.remove(pipeID); + return; + } + if (Objects.nonNull(metricService)) { + removeAutoGauge(operator); } - remainingEventAndTimeOperatorMap.remove(pipeID); completionOperatorMap.remove(pipeID); } @@ -500,9 +534,10 @@ public void markTsFileCollectInvocationCount( public Pair getRemainingEventAndTime( final String pipeName, final long creationTime) { final PipeDataNodeRemainingEventAndTimeOperator operator = - remainingEventAndTimeOperatorMap.computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)); + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.isNull(operator)) { + return new Pair<>(0L, 0.0); + } return new Pair<>(operator.getRemainingNonHeartbeatEvents(), operator.getRemainingTime()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java index 39aef7896ebcc..90878ab3c4dec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java @@ -109,39 +109,47 @@ public synchronized void unbindFrom(final AbstractMetricService metricService) { } private void removeMetrics(final String pipeID) { + pipeTimerMap.remove(pipeID); metricService.remove( MetricType.TIMER, Metric.PIPE_TSFILE_TO_TABLETS_TIME.toString(), Tag.NAME.toString(), pipeID); - pipeTimerMap.remove(pipeID); + pipeRateMap.remove(pipeID); metricService.remove( MetricType.RATE, Metric.PIPE_TSFILE_TO_TABLETS_RATE.toString(), Tag.NAME.toString(), pipeID); - pipeRateMap.remove(pipeID); + pipeTabletCountMap.remove(pipeID); metricService.remove( MetricType.COUNTER, Metric.PIPE_TSFILE_TO_TABLETS_COUNT.toString(), Tag.NAME.toString(), pipeID); - pipeTabletCountMap.remove(pipeID); + pipeTabletMemoryMap.remove(pipeID); metricService.remove( MetricType.COUNTER, Metric.PIPE_TSFILE_TO_TABLETS_TOTAL_MEMORY.toString(), Tag.NAME.toString(), pipeID); - pipeTabletMemoryMap.remove(pipeID); + pipeParseFileCountMap.remove(pipeID); metricService.remove( MetricType.COUNTER, Metric.PIPE_TSFILE_PARSE_FILE_COUNT.toString(), Tag.NAME.toString(), pipeID); + } + + private void removePipeStats(final String pipeID) { + pipeTimerMap.remove(pipeID); + pipeRateMap.remove(pipeID); + pipeTabletCountMap.remove(pipeID); + pipeTabletMemoryMap.remove(pipeID); pipeParseFileCountMap.remove(pipeID); } @@ -163,6 +171,8 @@ public synchronized void deregister(final String pipeID) { try { if (Objects.nonNull(metricService)) { removeMetrics(pipeID); + } else { + removePipeStats(pipeID); } } finally { pipe.remove(pipeID); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java index f549ad182d950..4d58accebdccc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java @@ -753,12 +753,20 @@ private static Type getOutputTabletType(final TSDataType dataType) { @Override public void close() throws Exception { - if (Objects.nonNull(pipeName) - && pipeName2referenceCountMap.compute( - pipeName, (name, count) -> Objects.nonNull(count) ? count - 1 : 0) - == 0) { - pipeName2timeSeries2TimeSeriesRuntimeStateMap.get(pipeName).clear(); - pipeName2timeSeries2TimeSeriesRuntimeStateMap.remove(pipeName); + boolean isLastReference = false; + if (Objects.nonNull(pipeName)) { + isLastReference = + pipeName2referenceCountMap.computeIfPresent( + pipeName, (name, count) -> count > 1 ? count - 1 : null) + == null; + } + if (isLastReference) { + final ConcurrentMap> + timeSeries2RuntimeStateMap = + pipeName2timeSeries2TimeSeriesRuntimeStateMap.remove(pipeName); + if (timeSeries2RuntimeStateMap != null) { + timeSeries2RuntimeStateMap.clear(); + } pipeName2LastValueReceiveTimeMap.remove(pipeName); } if (Objects.nonNull(windowingProcessor)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java index 64f2591975026..a72b6efbe0ef9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java @@ -1368,6 +1368,7 @@ private static String format(final TEndPoint endPoint) { @Override public synchronized void discardEventsOfPipe( final String pipeNameToDrop, final long creationTimeToDrop, final int regionId) { + super.discardEventsOfPipe(pipeNameToDrop, creationTimeToDrop, regionId); discardEventsOfPipe(new CommitterKey(pipeNameToDrop, creationTimeToDrop, regionId, -1)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java index 444a374992a21..d78a8961c576d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java @@ -806,6 +806,7 @@ public TPipeTransferReq compressIfNeeded(final TPipeTransferReq req) throws IOEx @Override public synchronized void discardEventsOfPipe( final String pipeNameToDrop, final long creationTimeToDrop, final int regionId) { + super.discardEventsOfPipe(pipeNameToDrop, creationTimeToDrop, regionId); discardEventsOfPipe(new CommitterKey(pipeNameToDrop, creationTimeToDrop, regionId, -1)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java index 8e33be13bdbe1..05b3d29f39828 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java @@ -329,6 +329,16 @@ protected AbstractPipeListeningQueue getListeningQueue() { return PipeDataNodeAgent.runtime().schemaListener(schemaRegionId); } + @Override + public long getUnTransferredEventCount() { + if (hasBeenClosed.get() || Objects.isNull(pipeTaskMeta)) { + return 0L; + } + // Metric callbacks can outlive region cleanup and must not recreate a listener. + return getUnTransferredEventCount( + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + } + @Override protected boolean isTypeListened(final PipeWritePlanEvent event) { final PlanNode planNode = ((PipeSchemaRegionWritePlanEvent) event).getPlanNode(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/SchemaRegionListeningQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/SchemaRegionListeningQueue.java index 3ae8bff32bb0b..aec6565e6af48 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/SchemaRegionListeningQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/SchemaRegionListeningQueue.java @@ -114,6 +114,10 @@ public synchronized boolean createSnapshot(final File snapshotDir) { } } + public static boolean hasSnapshot(final File snapshotDir) { + return snapshotDir != null && new File(snapshotDir, SNAPSHOT_FILE_NAME).isFile(); + } + public synchronized void loadSnapshot(final File snapshotDir) { try { super.deserializeFromFile(new File(snapshotDir, SNAPSHOT_FILE_NAME)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java index 9501c8e677da2..357044efe5c37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java @@ -38,6 +38,7 @@ import org.apache.iotdb.db.exception.DataRegionException; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; +import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; import org.apache.iotdb.db.schemaengine.SchemaEngine; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.rpc.RpcUtils; @@ -220,7 +221,12 @@ public TSStatus deleteSchemaRegion(SchemaRegionId schemaRegionId) { if (!schemaEngine.deleteSchemaRegion(schemaRegionId)) { return RpcUtils.getStatus(TSStatusCode.REGION_NOT_EXIST); } - PipeDataNodeAgent.runtime().schemaListener(schemaRegionId).close(); + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId); + if (listener != null) { + listener.close(); + } + PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId); schemaRegionLockMap.remove(schemaRegionId); return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } catch (MetadataException e) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java new file mode 100644 index 0000000000000..aec6094935f6c --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.agent.runtime; + +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.consensus.statemachine.schemaregion.SchemaRegionStateMachine; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; +import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionListenerMetrics; +import org.apache.iotdb.db.pipe.source.schemaregion.IoTDBSchemaRegionSource; +import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Map; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PipeSchemaRegionListenerManagerTest { + + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testLeaderStateDoesNotCreateListener() { + final PipeSchemaRegionListenerManager manager = new PipeSchemaRegionListenerManager(); + final SchemaRegionId schemaRegionId = new SchemaRegionId(1); + + manager.notifyLeaderReady(schemaRegionId); + + Assert.assertTrue(manager.isLeaderReady(schemaRegionId)); + Assert.assertTrue(manager.regionIds().isEmpty()); + Assert.assertNull(manager.listenerIfPresent(schemaRegionId)); + } + + @Test + public void testCleanupUnusedClosedListener() { + final PipeSchemaRegionListenerManager manager = new PipeSchemaRegionListenerManager(); + final SchemaRegionId schemaRegionId = new SchemaRegionId(2); + + final SchemaRegionListeningQueue listeningQueue = manager.listener(schemaRegionId); + Assert.assertEquals(1, manager.increaseAndGetReferenceCount(schemaRegionId)); + + listeningQueue.open(); + Assert.assertEquals(0, manager.decreaseAndGetReferenceCount(schemaRegionId)); + Assert.assertNotNull(manager.listenerIfPresent(schemaRegionId)); + + listeningQueue.close(); + manager.cleanupListenerIfUnused(schemaRegionId); + + Assert.assertNull(manager.listenerIfPresent(schemaRegionId)); + Assert.assertTrue(manager.regionIds().isEmpty()); + } + + @Test + public void testAutoCleanupAfterLastReferenceReleased() { + final PipeSchemaRegionListenerManager manager = new PipeSchemaRegionListenerManager(); + final SchemaRegionId schemaRegionId = new SchemaRegionId(3); + + manager.increaseAndGetReferenceCount(schemaRegionId); + + Assert.assertEquals(0, manager.decreaseAndGetReferenceCount(schemaRegionId)); + Assert.assertNull(manager.listenerIfPresent(schemaRegionId)); + Assert.assertTrue(manager.regionIds().isEmpty()); + } + + @Test + public void testMetricsDoNotRecreateListenerAfterRegionCleanup() throws Exception { + final SchemaRegionId schemaRegionId = new SchemaRegionId(Integer.MAX_VALUE - 10); + final TestSchemaRegionSource source = new TestSchemaRegionSource(); + final Field schemaRegionIdField = + IoTDBSchemaRegionSource.class.getDeclaredField("schemaRegionId"); + schemaRegionIdField.setAccessible(true); + schemaRegionIdField.set(source, schemaRegionId); + + try { + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime().schemaListener(schemaRegionId); + listener.open(); + listener.close(); + PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId); + + Assert.assertEquals(0L, source.getUnTransferredEventCount()); + Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + + source.close(); + Assert.assertEquals(0L, source.getUnTransferredEventCount()); + Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + } finally { + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId); + if (listener != null) { + listener.close(); + } + PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId); + } + } + + @Test + public void testCreateListenerForSnapshotLoadOnlyWhenSnapshotExists() throws IOException { + final PipeSchemaRegionListenerManager manager = new PipeSchemaRegionListenerManager(); + final SchemaRegionId schemaRegionId = new SchemaRegionId(4); + final File snapshotDir = temporaryFolder.newFolder(); + + Assert.assertNull(manager.listenerForSnapshotLoad(schemaRegionId, snapshotDir)); + Assert.assertTrue(manager.regionIds().isEmpty()); + + final SchemaRegionListeningQueue originalListener = manager.listener(schemaRegionId); + originalListener.open(); + Assert.assertTrue(originalListener.createSnapshot(snapshotDir)); + originalListener.close(); + manager.clearSchemaRegionState(schemaRegionId); + + final SchemaRegionListeningQueue restoredListener = + manager.listenerForSnapshotLoad(schemaRegionId, snapshotDir); + Assert.assertNotNull(restoredListener); + restoredListener.loadSnapshot(snapshotDir); + Assert.assertTrue(restoredListener.isOpened()); + + restoredListener.close(); + manager.clearSchemaRegionState(schemaRegionId); + } + + @Test + public void testSnapshotWithoutListenerClearsStaleFollowerQueue() throws Exception { + assertSnapshotWithoutListenerClearsStaleQueue(false); + } + + @Test + public void testSnapshotWithoutListenerRetainsReferencedClosedQueue() throws Exception { + assertSnapshotWithoutListenerClearsStaleQueue(true); + } + + private void assertSnapshotWithoutListenerClearsStaleQueue(final boolean hasLocalReference) + throws Exception { + final SchemaRegionId schemaRegionId = + new SchemaRegionId(Integer.MAX_VALUE - (hasLocalReference ? 12 : 11)); + final File snapshotDir = temporaryFolder.newFolder(); + final ISchemaRegion schemaRegion = mock(ISchemaRegion.class); + when(schemaRegion.getSchemaRegionId()).thenReturn(schemaRegionId); + when(schemaRegion.createSnapshot(snapshotDir)).thenReturn(true); + when(schemaRegion.loadSnapshot(snapshotDir)).thenReturn(true); + final SchemaRegionStateMachine stateMachine = new SchemaRegionStateMachine(schemaRegion); + final Field queueMapField = + PipeSchemaRegionListenerMetrics.class.getDeclaredField("listeningQueueMap"); + queueMapField.setAccessible(true); + final Map metricQueues = + (Map) queueMapField.get(PipeSchemaRegionListenerMetrics.getInstance()); + + try { + // The leader has applied queue-close and cleaned its listener before taking this snapshot. + Assert.assertTrue(stateMachine.takeSnapshot(snapshotDir)); + Assert.assertFalse(SchemaRegionListeningQueue.hasSnapshot(snapshotDir)); + Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + + // A lagging follower can still have the queue from before the close log entry. + final SchemaRegionListeningQueue staleQueue = + PipeDataNodeAgent.runtime().schemaListener(schemaRegionId); + if (hasLocalReference) { + PipeDataNodeAgent.runtime().increaseAndGetSchemaListenerReferenceCount(schemaRegionId); + } + staleQueue.open(); + final PlanNode node = mock(PlanNode.class); + when(node.getType()).thenReturn(PlanNodeType.CREATE_TIME_SERIES); + staleQueue.tryListenToNode(node); + Assert.assertEquals(1L, staleQueue.getSize()); + + Assert.assertTrue(stateMachine.loadSnapshot(snapshotDir)); + + Assert.assertFalse(staleQueue.isOpened()); + Assert.assertEquals(0L, staleQueue.getSize()); + if (hasLocalReference) { + Assert.assertSame( + staleQueue, PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + Assert.assertTrue(metricQueues.containsKey(schemaRegionId.getId())); + Assert.assertEquals( + 0, + PipeDataNodeAgent.runtime().decreaseAndGetSchemaListenerReferenceCount(schemaRegionId)); + } + Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + Assert.assertFalse(metricQueues.containsKey(schemaRegionId.getId())); + } finally { + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId); + if (listener != null) { + listener.close(); + } + PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId); + } + } + + private static class TestSchemaRegionSource extends IoTDBSchemaRegionSource { + + private TestSchemaRegionSource() { + pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 0); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/PipeMetricsRestartTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/PipeMetricsRestartTest.java index ab982c78af0aa..79277b651af0d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/PipeMetricsRestartTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/PipeMetricsRestartTest.java @@ -131,6 +131,30 @@ public void testSinglePipeMetricsKeepOperators() { assertTrue(metrics.remainingEventAndTimeOperatorMap.isEmpty()); } + @Test + public void testSinglePipeMetricsIgnoreLateEventsAfterDeregister() { + final PipeDataNodeSinglePipeMetrics metrics = PipeDataNodeSinglePipeMetrics.getInstance(); + bind(metrics); + metrics.register(mockDataRegionSource()); + metrics.deregister(PIPE_ID); + + metrics.increaseInsertNodeEventCount(PIPE, CREATION_TIME); + metrics.decreaseInsertNodeEventCount(PIPE, CREATION_TIME, 10); + metrics.updateInsertNodeTransferTimer(PIPE, CREATION_TIME, 10); + metrics.increaseRawTabletEventCount(PIPE, CREATION_TIME); + metrics.decreaseRawTabletEventCount(PIPE, CREATION_TIME); + metrics.increaseTsFileEventCount(PIPE, CREATION_TIME); + metrics.decreaseTsFileEventCount(PIPE, CREATION_TIME, 10); + metrics.updateTsFileTransferTimer(PIPE, CREATION_TIME, 10); + metrics.increaseHeartbeatEventCount(PIPE, CREATION_TIME); + metrics.decreaseHeartbeatEventCount(PIPE, CREATION_TIME); + assertEquals(0L, (long) metrics.getRemainingEventAndTime(PIPE, CREATION_TIME).getLeft()); + + assertTrue(metrics.remainingEventAndTimeOperatorMap.isEmpty()); + service.restartService(); + assertEquals(0, count(Metric.PIPE_DATANODE_REMAINING_EVENT_COUNT)); + } + @Test public void testDataRegionSourceMetrics() { final PipeDataRegionSourceMetrics metrics = PipeDataRegionSourceMetrics.getInstance(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java index 8246f9080d38c..702d2e1dc209d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java @@ -20,10 +20,18 @@ package org.apache.iotdb.db.pipe.metric.overview; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; +import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMetaKeeper; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent; import org.apache.iotdb.db.pipe.source.dataregion.IoTDBDataRegionSource; import org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionSource; @@ -37,10 +45,20 @@ import org.mockito.invocation.Invocation; import java.lang.reflect.Field; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockingDetails; @@ -154,6 +172,177 @@ public void testDeregisterClearsOperatorsWhenMetricsAreUnbound() throws Exceptio } } + @Test + public void testMetricCreationUsesExactPipeInstanceAcrossModels() throws Exception { + final PipeDataNodeSinglePipeMetrics metrics = PipeDataNodeSinglePipeMetrics.getInstance(); + final Field metricServiceField = + PipeDataNodeSinglePipeMetrics.class.getDeclaredField("metricService"); + metricServiceField.setAccessible(true); + reset(metrics, metricServiceField); + + final String pipeName = "shared_model_metric_" + System.nanoTime(); + final long treeCreationTime = System.nanoTime(); + final long tableCreationTime = treeCreationTime + 1; + final PipeMeta treePipe = + createPipeMeta(pipeName, treeCreationTime, SystemConstant.SQL_DIALECT_TREE_VALUE); + final PipeMeta tablePipe = + createPipeMeta(pipeName, tableCreationTime, SystemConstant.SQL_DIALECT_TABLE_VALUE); + final PipeMetaKeeper pipeMetaKeeper = getPipeMetaKeeper(); + + try { + pipeMetaKeeper.addPipeMeta(treePipe); + pipeMetaKeeper.addPipeMeta(tablePipe); + + assertEquals(treeCreationTime, PipeDataNodeAgent.task().getPipeCreationTime(pipeName)); + assertTrue(PipeDataNodeAgent.task().isPipeExisted(pipeName, tableCreationTime)); + + metrics.increaseTsFileEventCount(pipeName, tableCreationTime); + + assertTrue( + metrics.remainingEventAndTimeOperatorMap.containsKey(pipeName + "_" + tableCreationTime)); + } finally { + metrics.deregister(pipeName + "_" + tableCreationTime); + pipeMetaKeeper.removePipeMeta(treePipe.getStaticMeta()); + pipeMetaKeeper.removePipeMeta(tablePipe.getStaticMeta()); + reset(metrics, metricServiceField); + } + } + + @Test + public void testMetricCreationSerializesWithDeregister() throws Exception { + final PipeDataNodeSinglePipeMetrics metrics = PipeDataNodeSinglePipeMetrics.getInstance(); + final Field metricServiceField = + PipeDataNodeSinglePipeMetrics.class.getDeclaredField("metricService"); + metricServiceField.setAccessible(true); + reset(metrics, metricServiceField); + + final String pipeName = "metric_drop_race_" + System.nanoTime(); + final long creationTime = System.nanoTime(); + final String pipeID = pipeName + "_" + creationTime; + final PipeMeta pipeMeta = + createPipeMeta(pipeName, creationTime, SystemConstant.SQL_DIALECT_TREE_VALUE); + final PipeMetaKeeper pipeMetaKeeper = getPipeMetaKeeper(); + final Field pipeMetaMapField = PipeMetaKeeper.class.getDeclaredField("pipeNameToPipeMetaMap"); + pipeMetaMapField.setAccessible(true); + @SuppressWarnings("unchecked") + final Map originalPipeMetaMap = + (Map) pipeMetaMapField.get(pipeMetaKeeper); + final BlockingPipeMetaMap blockingPipeMetaMap = + new BlockingPipeMetaMap(originalPipeMetaMap, pipeName, creationTime); + pipeMetaMapField.set(pipeMetaKeeper, blockingPipeMetaMap); + pipeMetaKeeper.addPipeMeta(pipeMeta); + + final CountDownLatch metadataRemoved = new CountDownLatch(1); + final AtomicReference backgroundFailure = new AtomicReference<>(); + final Thread updateThread = + new Thread( + () -> { + try { + metrics.increaseTsFileEventCount(pipeName, creationTime); + } catch (final Throwable e) { + backgroundFailure.compareAndSet(null, e); + } + }); + final Thread deregisterThread = + new Thread( + () -> { + try { + pipeMetaKeeper.removePipeMeta(pipeMeta.getStaticMeta()); + metadataRemoved.countDown(); + metrics.deregister(pipeID); + } catch (final Throwable e) { + backgroundFailure.compareAndSet(null, e); + } + }); + + try { + try { + updateThread.start(); + assertTrue(blockingPipeMetaMap.lookupPaused.await(10, TimeUnit.SECONDS)); + deregisterThread.start(); + assertTrue(metadataRemoved.await(10, TimeUnit.SECONDS)); + assertEquals(Thread.State.BLOCKED, waitForBlockedOrTerminated(deregisterThread)); + } finally { + blockingPipeMetaMap.resumeLookup.countDown(); + updateThread.join(TimeUnit.SECONDS.toMillis(10)); + deregisterThread.join(TimeUnit.SECONDS.toMillis(10)); + } + + assertFalse(updateThread.isAlive()); + assertFalse(deregisterThread.isAlive()); + assertNull(backgroundFailure.get()); + assertFalse(metrics.remainingEventAndTimeOperatorMap.containsKey(pipeID)); + } finally { + pipeMetaKeeper.removePipeMeta(pipeMeta.getStaticMeta()); + pipeMetaMapField.set(pipeMetaKeeper, originalPipeMetaMap); + metrics.deregister(pipeID); + reset(metrics, metricServiceField); + } + } + + @Test + public void testLateMetricUpdateAfterDropDeregisterDoesNotRecreateOperator() throws Exception { + final PipeDataNodeSinglePipeMetrics metrics = PipeDataNodeSinglePipeMetrics.getInstance(); + final Field metricServiceField = + PipeDataNodeSinglePipeMetrics.class.getDeclaredField("metricService"); + metricServiceField.setAccessible(true); + reset(metrics, metricServiceField); + + final String pipeName = "metric_late_update_after_drop_" + System.nanoTime(); + final long creationTime = System.nanoTime(); + final String pipeID = pipeName + "_" + creationTime; + final PipeMeta pipeMeta = + createPipeMeta(pipeName, creationTime, SystemConstant.SQL_DIALECT_TREE_VALUE); + final PipeMetaKeeper pipeMetaKeeper = getPipeMetaKeeper(); + + try { + pipeMetaKeeper.addPipeMeta(pipeMeta); + metrics.increaseTsFileEventCount(pipeName, creationTime); + assertTrue(metrics.remainingEventAndTimeOperatorMap.containsKey(pipeID)); + + // dropPipe marks the metadata dropped before task deregistration and removes it afterward. + pipeMeta.getRuntimeMeta().getStatus().set(PipeStatus.DROPPED); + metrics.deregister(pipeID); + + metrics.increaseTsFileEventCount(pipeName, creationTime); + + assertFalse(metrics.remainingEventAndTimeOperatorMap.containsKey(pipeID)); + } finally { + pipeMetaKeeper.removePipeMeta(pipeMeta.getStaticMeta()); + metrics.deregister(pipeID); + reset(metrics, metricServiceField); + } + } + + private static PipeMeta createPipeMeta( + final String pipeName, final long creationTime, final String sqlDialect) { + final Map sourceAttributes = new HashMap<>(); + sourceAttributes.put(SystemConstant.SQL_DIALECT_KEY, sqlDialect); + return new PipeMeta( + new PipeStaticMeta( + pipeName, creationTime, sourceAttributes, new HashMap<>(), new HashMap<>()), + new PipeRuntimeMeta(new ConcurrentHashMap<>())); + } + + private static PipeMetaKeeper getPipeMetaKeeper() throws Exception { + final Field pipeMetaKeeperField = PipeTaskAgent.class.getDeclaredField("pipeMetaKeeper"); + pipeMetaKeeperField.setAccessible(true); + return (PipeMetaKeeper) pipeMetaKeeperField.get(PipeDataNodeAgent.task()); + } + + private static Thread.State waitForBlockedOrTerminated(final Thread thread) + throws InterruptedException { + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + Thread.State state = thread.getState(); + while (state != Thread.State.BLOCKED + && state != Thread.State.TERMINATED + && System.nanoTime() < deadline) { + Thread.sleep(1); + state = thread.getState(); + } + return state; + } + private static boolean hasCompletionGaugeCreation( final AbstractMetricService metricService, final String pipeName, final long creationTime) { for (final Invocation invocation : mockingDetails(metricService).getInvocations()) { @@ -185,6 +374,62 @@ private static void reset( metricServiceField.set(metrics, null); } + private static class BlockingPipeMetaMap extends ConcurrentHashMap { + + private static final long serialVersionUID = 1L; + + private final String pipeName; + private final long creationTime; + private final AtomicBoolean pauseNextLookup = new AtomicBoolean(true); + private final CountDownLatch lookupPaused = new CountDownLatch(1); + private final CountDownLatch resumeLookup = new CountDownLatch(1); + + private BlockingPipeMetaMap( + final Map originalMap, final String pipeName, final long creationTime) { + putAll(originalMap); + this.pipeName = pipeName; + this.creationTime = creationTime; + } + + @Override + public PipeMeta get(final Object key) { + final PipeMeta pipeMeta = super.get(key); + pauseIfMatching(pipeMeta); + return pipeMeta; + } + + @Override + public Collection values() { + final Collection values = super.values(); + final ArrayList snapshot = new ArrayList<>(values); + if (snapshot.stream().anyMatch(this::matches)) { + pauseIfMatching(snapshot.stream().filter(this::matches).findFirst().orElse(null)); + return snapshot; + } + return values; + } + + private boolean matches(final PipeMeta pipeMeta) { + return pipeMeta != null + && pipeName.equals(pipeMeta.getStaticMeta().getPipeName()) + && creationTime == pipeMeta.getStaticMeta().getCreationTime(); + } + + private void pauseIfMatching(final PipeMeta pipeMeta) { + if (matches(pipeMeta) && pauseNextLookup.compareAndSet(true, false)) { + lookupPaused.countDown(); + try { + if (!resumeLookup.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("Timed out waiting to resume pipe metadata lookup"); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + } + } + private static class TestSource extends PipeRealtimeDataRegionSource { private TestSource( diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index 3bb68a29562c3..c1b896cb54c80 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -177,6 +177,11 @@ reflections test + + ch.qos.logback + logback-classic + test + org.mockito mockito-core diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index 34bad05803fc5..355708481ac75 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -39,6 +39,7 @@ import org.apache.iotdb.commons.pipe.resource.PipeResourceFailureType; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.commons.pipe.sink.limiter.PipeEndPointRateLimiter; +import org.apache.iotdb.commons.pipe.sink.protocol.IoTDBSink; import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq; import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaRespExceptionMessage; @@ -61,6 +62,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -101,10 +103,14 @@ public abstract class PipeTaskAgent { protected final PipeMetaKeeper pipeMetaKeeper; protected final PipeTaskManager pipeTaskManager; + private final Map pipeNameWithCreationTime2FloatingMemoryUsageInByteMap; + private final Map> pipeName2CreationTimeSetMap; protected PipeTaskAgent() { pipeMetaKeeper = new PipeMetaKeeper(); pipeTaskManager = new PipeTaskManager(); + pipeNameWithCreationTime2FloatingMemoryUsageInByteMap = new ConcurrentHashMap<>(); + pipeName2CreationTimeSetMap = new ConcurrentHashMap<>(); // Help PipeEndPointRateLimiter to check if the pipe is still alive PipeEndPointRateLimiter.setTaskAgent(this); @@ -655,6 +661,8 @@ protected boolean dropPipe(final String pipeName, final long creationTime) { // Remove pipe meta from pipe meta keeper pipeMetaKeeper.removePipeMeta(existedPipeMeta.getStaticMeta()); + cleanupFloatingMemoryUsageCounterIfNecessary(pipeName, creationTime); + clearPipeRuntimeState(pipeName, creationTime); return true; } @@ -697,6 +705,9 @@ protected boolean dropPipe(final String pipeName) { // Remove pipe meta from pipe meta keeper pipeMetaKeeper.removePipeMeta(existedPipeMeta.getStaticMeta()); + cleanupFloatingMemoryUsageCounterIfNecessary( + pipeName, existedPipeMeta.getStaticMeta().getCreationTime()); + clearPipeRuntimeState(pipeName, existedPipeMeta.getStaticMeta().getCreationTime()); return true; } @@ -1283,6 +1294,41 @@ public static boolean isSnapshotMode(final PipeParameters parameters) { ///////////////////////// Maintain meta info ///////////////////////// + private static String generatePipeNameWithCreationTime( + final String pipeName, final long creationTime) { + return pipeName + "_" + creationTime; + } + + private AtomicLong cleanupFloatingMemoryUsageCounterIfUnused( + final String pipeName, final long creationTime, final AtomicLong counter) { + if (counter.get() != 0) { + return counter; + } + + // A missing counter also reports zero; a later acquisition recreates it without scanning all + // pipe metadata on the event-release path. + pipeName2CreationTimeSetMap.computeIfPresent( + pipeName, + (key, creationTimes) -> { + creationTimes.remove(creationTime); + return creationTimes.isEmpty() ? null : creationTimes; + }); + return null; + } + + private void cleanupFloatingMemoryUsageCounterIfNecessary( + final String pipeName, final long creationTime) { + pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.computeIfPresent( + generatePipeNameWithCreationTime(pipeName, creationTime), + (key, counter) -> + cleanupFloatingMemoryUsageCounterIfUnused(pipeName, creationTime, counter)); + } + + private void clearPipeRuntimeState(final String pipeName, final long creationTime) { + PipeEventCommitManager.getInstance().clear(pipeName, creationTime); + IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); + } + public long getPipeCreationTime(final String pipeName) { final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); return pipeMeta == null ? 0 : pipeMeta.getStaticMeta().getCreationTime(); @@ -1292,10 +1338,15 @@ public boolean isPipeExisted(final String pipeName, final long creationTime) { return pipeMetaKeeper.getPipeMeta(pipeName, creationTime) != null; } + public boolean isPipeAlive(final String pipeName, final long creationTime) { + final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); + return pipeMeta != null && pipeMeta.getRuntimeMeta().getStatus().get() != PipeStatus.DROPPED; + } + public String getPipeNameWithCreationTime(final String pipeName, final long creationTime) { final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); return pipeMeta == null - ? pipeName + "_" + creationTime + ? generatePipeNameWithCreationTime(pipeName, creationTime) : ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()).getPipeNameWithCreationTime(); } @@ -1309,49 +1360,60 @@ public CommitterKey getCommitterKey( } public long getAllFloatingMemoryUsageInByte() { - final AtomicLong bytes = new AtomicLong(0); - pipeMetaKeeper - .getPipeMetaList() - .forEach( - pipeMeta -> - bytes.addAndGet( - ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()) - .getFloatingMemoryUsageInByte())); - return bytes.get(); + return pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.values().stream() + .mapToLong(AtomicLong::get) + .sum(); } public long getFloatingMemoryUsageInByte(final String pipeName) { - final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); - return pipeMeta == null - ? 0 - : ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()).getFloatingMemoryUsageInByte(); + final Set creationTimes = pipeName2CreationTimeSetMap.get(pipeName); + if (Objects.isNull(creationTimes)) { + return 0; + } + + return creationTimes.stream() + .map(creationTime -> generatePipeNameWithCreationTime(pipeName, creationTime)) + .map(pipeNameWithCreationTime2FloatingMemoryUsageInByteMap::get) + .filter(Objects::nonNull) + .mapToLong(AtomicLong::get) + .sum(); } public long getFloatingMemoryUsageInByte(final String pipeName, final long creationTime) { - final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); - return pipeMeta == null - ? 0 - : ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()).getFloatingMemoryUsageInByte(); + final AtomicLong counter = + pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.get( + generatePipeNameWithCreationTime(pipeName, creationTime)); + return Objects.isNull(counter) ? 0 : counter.get(); } public void addFloatingMemoryUsageInByte( final String pipeName, final long creationTime, final long sizeInByte) { - final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); - // To avoid stale pipe before alter - if (Objects.nonNull(pipeMeta) && pipeMeta.getStaticMeta().getCreationTime() == creationTime) { - ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()) - .addFloatingMemoryUsageInByte(sizeInByte); - } + // Late events still own memory after drop and must be paired with their eventual release. + pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.compute( + generatePipeNameWithCreationTime(pipeName, creationTime), + (key, counter) -> { + final AtomicLong updatedCounter = Objects.isNull(counter) ? new AtomicLong(0) : counter; + updatedCounter.addAndGet(sizeInByte); + pipeName2CreationTimeSetMap.compute( + pipeName, + (name, creationTimes) -> { + final Set updatedCreationTimes = + Objects.isNull(creationTimes) ? ConcurrentHashMap.newKeySet() : creationTimes; + updatedCreationTimes.add(creationTime); + return updatedCreationTimes; + }); + return updatedCounter; + }); } public void decreaseFloatingMemoryUsageInByte( final String pipeName, final long creationTime, final long sizeInByte) { - final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); - // To avoid stale pipe before alter - if (Objects.nonNull(pipeMeta) && pipeMeta.getStaticMeta().getCreationTime() == creationTime) { - ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()) - .decreaseFloatingMemoryUsageInByte(sizeInByte); - } + pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.computeIfPresent( + generatePipeNameWithCreationTime(pipeName, creationTime), + (key, counter) -> { + counter.addAndGet(-sizeInByte); + return cleanupFloatingMemoryUsageCounterIfUnused(pipeName, creationTime, counter); + }); } public void recordPipeResourceFailure( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java index b2654d1e8a755..a04f7519551d3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java @@ -77,6 +77,13 @@ public void deregister(final String pipeName, final long creationTime, final int LOGGER.info(PipeMessages.PIPE_COMMITTER_DEREGISTERED, committerKey); } + public void clear(final String pipeName, final long creationTime) { + eventCommitterMap.keySet().removeIf(key -> clearIfMatches(key, pipeName, creationTime, true)); + eventCommitterRestartTimesMap + .keySet() + .removeIf(key -> clearIfMatches(key, pipeName, creationTime, false)); + } + /** * Assign a commit id and a key for commit. Make sure {@code EnrichedEvent.pipeName} is set before * calling this. @@ -87,8 +94,15 @@ public void enrichWithCommitterKeyAndCommitId( return; } + final String pipeName = event.getPipeName(); + final CommitterKey restartTimesKey = + generateCommitterRestartTimesKey(pipeName, creationTime, regionId); final CommitterKey committerKey = - generateCommitterKey(event.getPipeName(), creationTime, regionId); + taskAgent.getCommitterKey( + pipeName, + creationTime, + regionId, + eventCommitterRestartTimesMap.getOrDefault(restartTimesKey, 0)); final PipeEventCommitter committer = eventCommitterMap.get(committerKey); if (committer == null) { return; @@ -143,10 +157,11 @@ private boolean commitSingleId( final PipeEventCommitter committer = eventCommitterMap.get(committerKey); if (committer == null) { - final int currentRestartTimes = - eventCommitterRestartTimesMap.computeIfAbsent( - generateCommitterRestartTimesKey(committerKey), k -> 0); if (LOGGER.isDebugEnabled()) { + // Late commits must not recreate restart state after the pipe has been cleared. + final int currentRestartTimes = + eventCommitterRestartTimesMap.getOrDefault( + generateCommitterRestartTimesKey(committerKey), 0); if (committerKey.getRestartTimes() < currentRestartTimes) { LOGGER.debug( PipeMessages.STALE_PIPE_EVENT_COMMITTER, @@ -179,13 +194,25 @@ public CommitterKey getCommitterKey( * older committer after the same Pipe/DataRegion task has been restarted. */ public boolean isCurrentCommitterKey(final CommitterKey committerKey) { - return committerKey != null - && eventCommitterMap.containsKey(committerKey) - && committerKey.equals( - generateCommitterKey( + if (committerKey == null) { + return false; + } + + final PipeEventCommitter committer = eventCommitterMap.get(committerKey); + if (committer == null) { + return false; + } + + final int restartTimes = + eventCommitterRestartTimesMap.getOrDefault( + generateCommitterRestartTimesKey(committerKey), 0); + return committerKey.equals( + taskAgent.getCommitterKey( committerKey.getPipeName(), committerKey.getCreationTime(), - committerKey.getRegionId())); + committerKey.getRegionId(), + restartTimes)) + && eventCommitterMap.get(committerKey) == committer; } private CommitterKey generateCommitterKey( @@ -216,6 +243,20 @@ public void setCommitRateMarker(final BiConsumer commitRateMark this.commitRateMarker = commitRateMarker; } + private boolean clearIfMatches( + final CommitterKey key, + final String pipeName, + final long creationTime, + final boolean shouldClearMetrics) { + if (!Objects.equals(key.getPipeName(), pipeName) || key.getCreationTime() != creationTime) { + return false; + } + if (shouldClearMetrics) { + PipeEventCommitMetrics.getInstance().deregister(key.stringify()); + } + return true; + } + //////////////////////////// singleton //////////////////////////// private PipeEventCommitManager() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java index 2701e5bb23702..fbbe95a703450 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java @@ -55,6 +55,11 @@ public static void setTaskAgent(final PipeTaskAgent taskAgent) { PipeEndPointRateLimiter.taskAgent = taskAgent; } + public static boolean isPipeAlive(final String pipeName, final long creationTime) { + final PipeTaskAgent finalTaskAgent = taskAgent; + return Objects.isNull(finalTaskAgent) || finalTaskAgent.isPipeAlive(pipeName, creationTime); + } + public void acquire(final TEndPoint endPoint, long bytes) { if (endPoint == null) { return; @@ -82,9 +87,7 @@ private boolean tryAcquireWithPipeCheck(final RateLimiter rateLimiter, final int bytes, PipeConfig.getInstance().getRateLimiterHotReloadCheckIntervalMs(), TimeUnit.MILLISECONDS)) { - final PipeTaskAgent finalTaskAgent = taskAgent; - if (Objects.nonNull(finalTaskAgent) - && !finalTaskAgent.isPipeExisted(pipeName, creationTime)) { + if (!isPipeAlive(pipeName, creationTime)) { return false; } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java index 601ad31a0f42e..6bbf9cc5b3ba0 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java @@ -677,25 +677,33 @@ public void rateLimitIfNeeded( final TEndPoint endPoint, final long bytesLength) { if (pipeName != null && endPointRateLimitBytesPerSecond > 0) { - PIPE_END_POINT_RATE_LIMITER_MAP - .computeIfAbsent( + final PipeEndPointRateLimiter limiter = + PIPE_END_POINT_RATE_LIMITER_MAP.computeIfAbsent( new Pair<>(pipeName, creationTime), endpoint -> - new PipeEndPointRateLimiter( - pipeName, creationTime, endPointRateLimitBytesPerSecond)) - .acquire(endPoint, bytesLength); + PipeEndPointRateLimiter.isPipeAlive(pipeName, creationTime) + ? new PipeEndPointRateLimiter( + pipeName, creationTime, endPointRateLimitBytesPerSecond) + : null); + if (limiter != null) { + limiter.acquire(endPoint, bytesLength); + } } GLOBAL_RATE_LIMITER.acquire(bytesLength); } + public static void clearPipeRateLimiter(final String pipeName, final long creationTime) { + PIPE_END_POINT_RATE_LIMITER_MAP.remove(new Pair<>(pipeName, creationTime)); + } + /** - * When a pipe is dropped, the connector maybe reused and will not be closed. We need to discard - * its batched or queued events in the output pipe connector. + * When a region task is dropped, the connector may be reused and will not be closed. Discard its + * batched or queued events in the output pipe connector. */ public synchronized void discardEventsOfPipe( final String pipeName, final long creationTime, final int regionId) { - // Do nothing by default + // Rate limiter state is pipe-wide and is cleared only after the whole pipe is dropped. } public PipeReceiverStatusHandler statusHandler() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java index d4d55ba7dac7a..c827fe2b98543 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java @@ -302,12 +302,20 @@ public long getUnTransferredEventCount() { if (Objects.isNull(pipeTaskMeta)) { return 0L; } + return getUnTransferredEventCount(getListeningQueue()); + } + + /** Calculate metrics from a captured queue without looking it up again during cleanup. */ + protected long getUnTransferredEventCount(final AbstractPipeListeningQueue listeningQueue) { + if (Objects.isNull(pipeTaskMeta) || Objects.isNull(listeningQueue)) { + return 0L; + } final ProgressIndex progressIndex = pipeTaskMeta.getProgressIndex(); warnIfUnexpectedHybridProgressIndex(progressIndex); final MetaProgressIndex metaProgressIndex = extractMetaProgressIndex(progressIndex); return Objects.nonNull(metaProgressIndex) - ? getListeningQueue().getTailIndex() - metaProgressIndex.getIndex() - 1 - : getListeningQueue().getSize() + historicalEventsCount; + ? listeningQueue.getTailIndex() - metaProgressIndex.getIndex() - 1 + : listeningQueue.getSize() + historicalEventsCount; } private static MetaProgressIndex extractMetaProgressIndex(final ProgressIndex progressIndex) { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java new file mode 100644 index 0000000000000..1f54a0d6d15ba --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.agent.task; + +import org.apache.iotdb.common.rpc.thrift.TPipeHeartbeatResp; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; +import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager; +import org.apache.iotdb.commons.pipe.sink.limiter.PipeEndPointRateLimiter; +import org.apache.iotdb.commons.pipe.sink.protocol.IoTDBSink; +import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq; + +import org.apache.thrift.TException; +import org.apache.tsfile.utils.Pair; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +public class PipeTaskAgentTest { + + @Test + public void testFloatingMemoryUsageSurvivesDropUntilLateRelease() throws IllegalPathException { + final DummyPipeTaskAgent agent = new DummyPipeTaskAgent(); + + agent.createPipeForTest(generatePipeMeta("pipe", 1L)); + agent.addFloatingMemoryUsageInByte("pipe", 1L, 100L); + Assert.assertEquals(100L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe")); + Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe", 1L)); + + agent.dropPipeForTest("pipe", 1L); + Assert.assertEquals(100L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe")); + Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe", 1L)); + + agent.createPipeForTest(generatePipeMeta("pipe", 2L)); + agent.addFloatingMemoryUsageInByte("pipe", 2L, 20L); + Assert.assertEquals(120L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(120L, agent.getFloatingMemoryUsageInByte("pipe")); + Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe", 1L)); + Assert.assertEquals(20L, agent.getFloatingMemoryUsageInByte("pipe", 2L)); + + agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 100L); + Assert.assertEquals(20L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(20L, agent.getFloatingMemoryUsageInByte("pipe")); + Assert.assertEquals(0L, agent.getFloatingMemoryUsageInByte("pipe", 1L)); + Assert.assertEquals(20L, agent.getFloatingMemoryUsageInByte("pipe", 2L)); + + agent.dropPipeForTest("pipe", 2L); + agent.decreaseFloatingMemoryUsageInByte("pipe", 2L, 20L); + Assert.assertEquals(0L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(0L, agent.getFloatingMemoryUsageInByte("pipe")); + } + + @Test + public void testZeroFloatingMemoryUsageCounterIsCleanedAfterDrop() throws Exception { + final DummyPipeTaskAgent agent = new DummyPipeTaskAgent(); + + agent.createPipeForTest(generatePipeMeta("pipe", 1L)); + agent.addFloatingMemoryUsageInByte("pipe", 1L, 100L); + agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 100L); + + Assert.assertTrue( + getMapField(agent, "pipeNameWithCreationTime2FloatingMemoryUsageInByteMap").isEmpty()); + Assert.assertTrue(getMapField(agent, "pipeName2CreationTimeSetMap").isEmpty()); + + agent.dropPipeForTest("pipe", 1L); + + Assert.assertTrue( + getMapField(agent, "pipeNameWithCreationTime2FloatingMemoryUsageInByteMap").isEmpty()); + Assert.assertTrue(getMapField(agent, "pipeName2CreationTimeSetMap").isEmpty()); + } + + @Test + public void testLateFloatingMemoryAcquisitionDoesNotConsumeOutstandingUsage() throws Exception { + final DummyPipeTaskAgent agent = new DummyPipeTaskAgent(); + agent.createPipeForTest(generatePipeMeta("pipe", 1L)); + agent.addFloatingMemoryUsageInByte("pipe", 1L, 100L); + agent.dropPipeForTest("pipe", 1L); + + // An assigner can retain a matched source across drop while other events still own memory. + agent.addFloatingMemoryUsageInByte("pipe", 1L, 20L); + Assert.assertEquals(120L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(120L, agent.getFloatingMemoryUsageInByte("pipe")); + agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 20L); + Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe", 1L)); + agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 100L); + + Assert.assertEquals(0L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertTrue( + getMapField(agent, "pipeNameWithCreationTime2FloatingMemoryUsageInByteMap").isEmpty()); + Assert.assertTrue(getMapField(agent, "pipeName2CreationTimeSetMap").isEmpty()); + } + + @Test + public void testLateFloatingMemoryAcquisitionAfterCounterCleanup() throws Exception { + final DummyPipeTaskAgent agent = new DummyPipeTaskAgent(); + agent.createPipeForTest(generatePipeMeta("pipe", 1L)); + agent.addFloatingMemoryUsageInByte("pipe", 1L, 100L); + agent.dropPipeForTest("pipe", 1L); + agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 100L); + + // Late acquisitions must also be tracked after the previous events released the last byte. + agent.addFloatingMemoryUsageInByte("pipe", 1L, 20L); + Assert.assertEquals(20L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(20L, agent.getFloatingMemoryUsageInByte("pipe")); + Assert.assertEquals(20L, agent.getFloatingMemoryUsageInByte("pipe", 1L)); + agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 20L); + + Assert.assertTrue( + getMapField(agent, "pipeNameWithCreationTime2FloatingMemoryUsageInByteMap").isEmpty()); + Assert.assertTrue(getMapField(agent, "pipeName2CreationTimeSetMap").isEmpty()); + } + + @Test + public void testFloatingMemoryIncrementIsNotLostDuringDropCleanup() throws Exception { + final DummyPipeTaskAgent agent = new DummyPipeTaskAgent(); + final PipeMeta pipeMeta = generatePipeMeta("concurrent-pipe", 1L); + agent.createPipeForTest(pipeMeta); + agent.addFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L); + agent.decreaseFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L); + + final Field counterMapField = + PipeTaskAgent.class.getDeclaredField( + "pipeNameWithCreationTime2FloatingMemoryUsageInByteMap"); + counterMapField.setAccessible(true); + @SuppressWarnings("unchecked") + final Map originalCounterMap = + (Map) counterMapField.get(agent); + final BlockingFloatingMemoryMap blockingCounterMap = + new BlockingFloatingMemoryMap(originalCounterMap, "concurrent-pipe_1"); + counterMapField.set(agent, blockingCounterMap); + + final AtomicReference backgroundFailure = new AtomicReference<>(); + final Thread increaseThread = + new Thread( + () -> + runAndRecordFailure( + () -> agent.addFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L), + backgroundFailure)); + final Thread dropThread = + new Thread( + () -> + runAndRecordFailure( + () -> agent.dropPipeForTest("concurrent-pipe", 1L), backgroundFailure)); + + try { + try { + blockingCounterMap.pauseNextUpdate(); + increaseThread.start(); + Assert.assertTrue(blockingCounterMap.updatePaused.await(10, TimeUnit.SECONDS)); + dropThread.start(); + Assert.assertTrue(waitForDrop(agent, "concurrent-pipe", 1L)); + } finally { + blockingCounterMap.resumeUpdate.countDown(); + increaseThread.join(TimeUnit.SECONDS.toMillis(10)); + dropThread.join(TimeUnit.SECONDS.toMillis(10)); + } + + Assert.assertFalse(increaseThread.isAlive()); + Assert.assertFalse(dropThread.isAlive()); + Assert.assertNull(backgroundFailure.get()); + Assert.assertEquals(100L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("concurrent-pipe")); + + agent.decreaseFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L); + Assert.assertEquals(0L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(0L, agent.getFloatingMemoryUsageInByte("concurrent-pipe")); + } finally { + counterMapField.set(agent, originalCounterMap); + } + } + + @Test + public void testPipeDropClearsSharedRuntimeState() throws Exception { + assertPipeDropClearsSharedRuntimeState(false); + assertPipeDropClearsSharedRuntimeState(true); + } + + private static void assertPipeDropClearsSharedRuntimeState(final boolean dropByName) + throws Exception { + final String pipeName = "runtime_state_cleanup_" + System.nanoTime(); + final long creationTime = System.nanoTime(); + final int regionId = 17; + final DummyPipeTaskAgent agent = new DummyPipeTaskAgent(); + final PipeEventCommitManager commitManager = PipeEventCommitManager.getInstance(); + final Map, PipeEndPointRateLimiter> rateLimiters = getRateLimiterMap(); + final Pair pipeKey = new Pair<>(pipeName, creationTime); + + try { + commitManager.register(pipeName, creationTime, regionId, "test-sink"); + final CommitterKey committerKey = + commitManager.getCommitterKey(pipeName, creationTime, regionId); + final CommitterKey restartTimesKey = new CommitterKey(pipeName, creationTime, regionId, -1); + rateLimiters.put(pipeKey, new PipeEndPointRateLimiter(pipeName, creationTime, 1.0)); + agent.createPipeForTest(generatePipeMeta(pipeName, creationTime)); + + if (dropByName) { + Assert.assertTrue(agent.dropPipeByNameForTest(pipeName)); + } else { + Assert.assertTrue(agent.dropPipeForTest(pipeName, creationTime)); + } + + Assert.assertFalse( + getCommitManagerMap(commitManager, "eventCommitterMap").containsKey(committerKey)); + Assert.assertFalse( + getCommitManagerMap(commitManager, "eventCommitterRestartTimesMap") + .containsKey(restartTimesKey)); + Assert.assertFalse(rateLimiters.containsKey(pipeKey)); + } finally { + commitManager.clear(pipeName, creationTime); + IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); + final PipeMeta pipeMeta = agent.getPipeMetaForTest(pipeName, creationTime); + if (pipeMeta != null) { + agent.dropPipeForTest(pipeName, creationTime); + } + } + } + + @SuppressWarnings("unchecked") + private static Map, PipeEndPointRateLimiter> getRateLimiterMap() + throws ReflectiveOperationException { + final Field field = IoTDBSink.class.getDeclaredField("PIPE_END_POINT_RATE_LIMITER_MAP"); + field.setAccessible(true); + return (Map, PipeEndPointRateLimiter>) field.get(null); + } + + private static Map getCommitManagerMap( + final PipeEventCommitManager commitManager, final String fieldName) + throws ReflectiveOperationException { + final Field field = PipeEventCommitManager.class.getDeclaredField(fieldName); + field.setAccessible(true); + return (Map) field.get(commitManager); + } + + @SuppressWarnings("unchecked") + private static Map getMapField(final PipeTaskAgent agent, final String fieldName) + throws NoSuchFieldException, IllegalAccessException { + final Field field = PipeTaskAgent.class.getDeclaredField(fieldName); + field.setAccessible(true); + return (Map) field.get(agent); + } + + private static PipeMeta generatePipeMeta(final String pipeName, final long creationTime) { + return new PipeMeta( + new PipeStaticMeta( + pipeName, creationTime, new HashMap<>(), new HashMap<>(), new HashMap<>()), + new PipeRuntimeMeta(new ConcurrentHashMap<>())); + } + + private static void runAndRecordFailure( + final Runnable operation, final AtomicReference failure) { + try { + operation.run(); + } catch (final Throwable e) { + failure.compareAndSet(null, e); + } + } + + private static boolean waitForDrop( + final DummyPipeTaskAgent agent, final String pipeName, final long creationTime) + throws InterruptedException { + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (agent.getPipeMetaForTest(pipeName, creationTime) == null) { + return true; + } + Thread.sleep(1); + } + return false; + } + + private static class DummyPipeTaskAgent extends PipeTaskAgent { + + private boolean createPipeForTest(final PipeMeta pipeMeta) throws IllegalPathException { + return createPipe(pipeMeta); + } + + private boolean dropPipeForTest(final String pipeName, final long creationTime) { + return dropPipe(pipeName, creationTime); + } + + private boolean dropPipeByNameForTest(final String pipeName) { + return dropPipe(pipeName); + } + + private PipeMeta getPipeMetaForTest(final String pipeName, final long creationTime) { + return pipeMetaKeeper.getPipeMeta(pipeName, creationTime); + } + + @Override + protected boolean isShutdown() { + return false; + } + + @Override + protected void thawRate(final String pipeName, final long creationTime) { + // Do nothing + } + + @Override + protected void freezeRate(final String pipeName, final long creationTime) { + // Do nothing + } + + @Override + protected Map buildPipeTasks(final PipeMeta pipeMetaFromCoordinator) { + return new HashMap<>(); + } + + @Override + protected void createPipeTask( + final int consensusGroupId, + final PipeStaticMeta pipeStaticMeta, + final org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta pipeTaskMeta) { + // Do nothing + } + + @Override + protected void collectPipeMetaListInternal( + final TPipeHeartbeatReq req, final TPipeHeartbeatResp resp) throws TException { + // Do nothing + } + + @Override + public void runPipeTasks( + final Collection pipeTasks, final Consumer runSingle) { + pipeTasks.forEach(runSingle); + } + } + + private static class BlockingFloatingMemoryMap extends ConcurrentHashMap { + + private static final long serialVersionUID = 1L; + + private final String pipeID; + private final AtomicBoolean shouldPause = new AtomicBoolean(); + private final CountDownLatch updatePaused = new CountDownLatch(1); + private final CountDownLatch resumeUpdate = new CountDownLatch(1); + + private BlockingFloatingMemoryMap( + final Map originalMap, final String pipeID) { + putAll(originalMap); + this.pipeID = pipeID; + } + + private void pauseNextUpdate() { + shouldPause.set(true); + } + + @Override + public AtomicLong compute( + final String key, + final BiFunction + remappingFunction) { + return super.compute( + key, + (currentKey, currentValue) -> { + final AtomicLong updatedValue = remappingFunction.apply(currentKey, currentValue); + pauseIfNecessary(currentKey); + return updatedValue; + }); + } + + @Override + public AtomicLong computeIfAbsent( + final String key, final Function mappingFunction) { + final AtomicLong value = super.computeIfAbsent(key, mappingFunction); + pauseIfNecessary(key); + return value; + } + + private void pauseIfNecessary(final String key) { + if (!pipeID.equals(key) || !shouldPause.compareAndSet(true, false)) { + return; + } + updatePaused.countDown(); + try { + if (!resumeUpdate.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("Timed out waiting to resume floating memory update"); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java new file mode 100644 index 0000000000000..2d81059d623e3 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.agent.task.progress; + +import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent; +import org.apache.iotdb.commons.pipe.event.EnrichedEvent; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +@RunWith(Parameterized.class) +public class PipeEventCommitManagerTest { + + private static final String PIPE_NAME = "late-commit-test"; + private static final long CREATION_TIME = 1L; + private static final int REGION_ID = 1; + + private final Level logLevel; + private final Logger logger = (Logger) LoggerFactory.getLogger(PipeEventCommitManager.class); + private Level originalLogLevel; + private PipeEventCommitManager manager; + + public PipeEventCommitManagerTest(final Level logLevel) { + this.logLevel = logLevel; + } + + @Parameterized.Parameters(name = "{0}") + public static Iterable logLevels() { + return Arrays.asList(Level.INFO, Level.DEBUG); + } + + @Before + public void setUp() throws Exception { + originalLogLevel = logger.getLevel(); + logger.setLevel(logLevel); + final Constructor constructor = + PipeEventCommitManager.class.getDeclaredConstructor(); + constructor.setAccessible(true); + manager = constructor.newInstance(); + + final PipeTaskAgent taskAgent = mock(PipeTaskAgent.class); + when(taskAgent.getCommitterKey(eq(PIPE_NAME), eq(CREATION_TIME), eq(REGION_ID), anyInt())) + .thenAnswer( + invocation -> + new CommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID, invocation.getArgument(3))); + manager.setTaskAgent(taskAgent); + manager.register(PIPE_NAME, CREATION_TIME, REGION_ID, "iotdb-thrift-sink"); + } + + @After + public void tearDown() { + try { + if (manager != null) { + manager.clear(PIPE_NAME, CREATION_TIME); + } + } finally { + logger.setLevel(originalLogLevel); + } + } + + @Test + public void testLateSingleCommitAfterClearDoesNotRestoreState() throws Exception { + final CommitterKey key = manager.getCommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID); + final Runnable onCommitted = mock(Runnable.class); + final EnrichedEvent event = createEvent(onCommitted); + manager.clear(PIPE_NAME, CREATION_TIME); + + manager.commit(event, key); + + assertTrue(getManagerMap("eventCommitterMap").isEmpty()); + assertTrue(getManagerMap("eventCommitterRestartTimesMap").isEmpty()); + verify(onCommitted, never()).run(); + } + + @Test + public void testLateMultipleCommitsAfterClearDoNotRestoreState() throws Exception { + final CommitterKey key = manager.getCommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID); + final Runnable onCommitted = mock(Runnable.class); + final EnrichedEvent event = createEvent(onCommitted); + when(event.hasMultipleCommitIds()).thenReturn(true); + when(event.getCommitIds()).thenReturn(Arrays.asList(1L, 2L, 3L)); + manager.clear(PIPE_NAME, CREATION_TIME); + + manager.commit(event, key); + + assertTrue(getManagerMap("eventCommitterMap").isEmpty()); + assertTrue(getManagerMap("eventCommitterRestartTimesMap").isEmpty()); + verify(onCommitted, never()).run(); + } + + @Test + public void testLateEventEnrichmentAfterClearDoesNotRestoreState() throws Exception { + final EnrichedEvent event = createEvent(mock(Runnable.class)); + manager.clear(PIPE_NAME, CREATION_TIME); + + manager.enrichWithCommitterKeyAndCommitId(event, CREATION_TIME, REGION_ID); + + assertTrue(getManagerMap("eventCommitterMap").isEmpty()); + assertTrue(getManagerMap("eventCommitterRestartTimesMap").isEmpty()); + } + + @Test + public void testLateCommitterCheckAfterClearDoesNotRestoreRestartState() throws Exception { + final CommitterKey key = manager.getCommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID); + final Field eventCommitterMapField = + PipeEventCommitManager.class.getDeclaredField("eventCommitterMap"); + eventCommitterMapField.setAccessible(true); + @SuppressWarnings("unchecked") + final Map originalEventCommitterMap = + (Map) eventCommitterMapField.get(manager); + final BlockingCommitterMap blockingMap = new BlockingCommitterMap(originalEventCommitterMap); + eventCommitterMapField.set(manager, blockingMap); + + final AtomicReference result = new AtomicReference<>(); + final AtomicReference backgroundFailure = new AtomicReference<>(); + final Thread checkThread = + new Thread( + () -> { + try { + result.set(manager.isCurrentCommitterKey(key)); + } catch (final Throwable e) { + backgroundFailure.compareAndSet(null, e); + } + }); + + try { + blockingMap.pauseNextLookup(); + checkThread.start(); + assertTrue(blockingMap.lookupPaused.await(10, TimeUnit.SECONDS)); + + manager.clear(PIPE_NAME, CREATION_TIME); + } finally { + blockingMap.resumeLookup.countDown(); + checkThread.join(TimeUnit.SECONDS.toMillis(10)); + eventCommitterMapField.set(manager, originalEventCommitterMap); + } + + assertFalse(checkThread.isAlive()); + assertTrue(backgroundFailure.get() == null); + assertFalse(result.get()); + assertTrue(getManagerMap("eventCommitterRestartTimesMap").isEmpty()); + } + + @Test + public void testRestartRejectsStaleCommitAndAcceptsCurrentCommit() throws Exception { + final CommitterKey oldKey = manager.getCommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID); + final Runnable oldOnCommitted = mock(Runnable.class); + manager.deregister(PIPE_NAME, CREATION_TIME, REGION_ID); + + manager.commit(createEvent(oldOnCommitted), oldKey); + assertEquals( + Integer.valueOf(1), + getManagerMap("eventCommitterRestartTimesMap") + .get(new CommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID))); + + manager.register(PIPE_NAME, CREATION_TIME, REGION_ID, "iotdb-thrift-sink"); + final CommitterKey newKey = manager.getCommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID); + assertEquals(1, newKey.getRestartTimes()); + final Runnable newOnCommitted = mock(Runnable.class); + manager.commit(createEvent(oldOnCommitted), oldKey); + manager.commit(createEvent(newOnCommitted), newKey); + + verify(oldOnCommitted, never()).run(); + verify(newOnCommitted).run(); + assertEquals(1, getManagerMap("eventCommitterMap").size()); + assertEquals(1, getManagerMap("eventCommitterRestartTimesMap").size()); + } + + private static EnrichedEvent createEvent(final Runnable onCommitted) { + final EnrichedEvent event = + mock( + EnrichedEvent.class, + withSettings() + .useConstructor( + PIPE_NAME, + CREATION_TIME, + null, + null, + null, + null, + null, + null, + false, + Long.MIN_VALUE, + Long.MAX_VALUE)); + when(event.needToCommit()).thenReturn(true); + when(event.getCommitId()).thenReturn(1L); + when(event.getOnCommittedHooks()).thenReturn(Collections.singletonList(onCommitted)); + return event; + } + + private Map getManagerMap(final String fieldName) throws Exception { + final Field field = PipeEventCommitManager.class.getDeclaredField(fieldName); + field.setAccessible(true); + return (Map) field.get(manager); + } + + private static class BlockingCommitterMap + extends ConcurrentHashMap { + + private static final long serialVersionUID = 1L; + + private final AtomicBoolean shouldPause = new AtomicBoolean(); + private final CountDownLatch lookupPaused = new CountDownLatch(1); + private final CountDownLatch resumeLookup = new CountDownLatch(1); + + private BlockingCommitterMap(final Map originalMap) { + putAll(originalMap); + } + + private void pauseNextLookup() { + shouldPause.set(true); + } + + @Override + public PipeEventCommitter get(final Object key) { + final PipeEventCommitter committer = super.get(key); + if (committer != null && shouldPause.compareAndSet(true, false)) { + lookupPaused.countDown(); + try { + if (!resumeLookup.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("Timed out waiting to resume committer lookup"); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + return committer; + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java new file mode 100644 index 0000000000000..a98a3a59b3ce4 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.protocol; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent; +import org.apache.iotdb.commons.pipe.sink.limiter.PipeEndPointRateLimiter; + +import org.apache.tsfile.utils.Pair; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Map; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class IoTDBSinkRateLimiterTest { + + @Test + public void testRegionTaskDiscardKeepsPipeWideRateLimiter() throws Exception { + final String pipeName = "rate_limiter_lifecycle_test"; + final long creationTime = System.nanoTime(); + final Pair pipeKey = new Pair<>(pipeName, creationTime); + final Map, PipeEndPointRateLimiter> limiters = getRateLimiterMap(); + final PipeEndPointRateLimiter limiter = mock(PipeEndPointRateLimiter.class); + final IoTDBSink sink = mock(IoTDBSink.class, CALLS_REAL_METHODS); + limiters.put(pipeKey, limiter); + + try { + sink.discardEventsOfPipe(pipeName, creationTime, 1); + + assertSame(limiter, limiters.get(pipeKey)); + + IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); + + assertFalse(limiters.containsKey(pipeKey)); + } finally { + IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); + } + } + + @Test + public void testLateTransferDoesNotRecreateRateLimiterAfterDrop() throws Exception { + final String pipeName = "late_rate_limiter_lifecycle_test"; + final long creationTime = System.nanoTime(); + final Pair pipeKey = new Pair<>(pipeName, creationTime); + final Map, PipeEndPointRateLimiter> limiters = getRateLimiterMap(); + final IoTDBSink sink = mock(IoTDBSink.class, CALLS_REAL_METHODS); + final PipeTaskAgent taskAgent = mock(PipeTaskAgent.class); + when(taskAgent.isPipeAlive(pipeName, creationTime)).thenReturn(false); + + final Field taskAgentField = PipeEndPointRateLimiter.class.getDeclaredField("taskAgent"); + taskAgentField.setAccessible(true); + final PipeTaskAgent originalTaskAgent = (PipeTaskAgent) taskAgentField.get(null); + final Field rateLimitField = + IoTDBSink.class.getDeclaredField("endPointRateLimitBytesPerSecond"); + rateLimitField.setAccessible(true); + rateLimitField.setDouble(sink, 1.0); + + PipeEndPointRateLimiter.setTaskAgent(taskAgent); + try { + sink.rateLimitIfNeeded(pipeName, creationTime, new TEndPoint("localhost", 1), 0); + + assertFalse(limiters.containsKey(pipeKey)); + } finally { + IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); + PipeEndPointRateLimiter.setTaskAgent(originalTaskAgent); + } + } + + @SuppressWarnings("unchecked") + private static Map, PipeEndPointRateLimiter> getRateLimiterMap() + throws ReflectiveOperationException { + final Field field = IoTDBSink.class.getDeclaredField("PIPE_END_POINT_RATE_LIMITER_MAP"); + field.setAccessible(true); + return (Map, PipeEndPointRateLimiter>) field.get(null); + } +}