From 3167d75f51b61eaf707602253572903c5ff07b92 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 13 May 2026 18:54:58 +0800 Subject: [PATCH 01/14] Fix --- .../schemaregion/SchemaExecutionVisitor.java | 20 +++-- .../SchemaRegionStateMachine.java | 46 ++++++---- .../runtime/PipeDataNodeRuntimeAgent.java | 12 +++ .../PipeSchemaRegionListenerManager.java | 85 ++++++++++--------- .../agent/task/PipeDataNodeTaskAgent.java | 15 +++- .../aggregate/AggregateProcessor.java | 19 +++-- .../async/IoTDBDataRegionAsyncSink.java | 1 + .../thrift/sync/IoTDBDataRegionSyncSink.java | 1 + .../thrift/impl/DataNodeRegionManager.java | 8 +- .../PipeSchemaRegionListenerManagerTest.java | 72 ++++++++++++++++ .../task/progress/PipeEventCommitManager.java | 21 +++++ .../commons/pipe/sink/protocol/IoTDBSink.java | 2 +- 12 files changed, 229 insertions(+), 73 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index f3ddfbab7b292..2fc2694c4c564 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -843,13 +843,19 @@ public TSStatus visitPipeEnrichedNonWritePlanNode( public TSStatus visitPipeOperateSchemaQueueNode( final PipeOperateSchemaQueueNode node, final ISchemaRegion schemaRegion) { final SchemaRegionId id = schemaRegion.getSchemaRegionId(); - final SchemaRegionListeningQueue queue = PipeDataNodeAgent.runtime().schemaListener(id); - if (node.isOpen() && !queue.isOpened()) { - logger.info("Opened pipe listening queue on schema region {}", id); - queue.open(); - } else if (!node.isOpen() && queue.isOpened()) { - logger.info("Closed pipe listening queue on schema region {}", id); - queue.close(); + if (node.isOpen()) { + final SchemaRegionListeningQueue queue = PipeDataNodeAgent.runtime().schemaListener(id); + if (!queue.isOpened()) { + logger.info("Opened pipe listening queue on schema region {}", id); + queue.open(); + } + } else { + final SchemaRegionListeningQueue queue = PipeDataNodeAgent.runtime().schemaListenerIfPresent(id); + if (queue != null && queue.isOpened()) { + logger.info("Closed pipe listening queue on schema region {}", id); + queue.close(); + PipeDataNodeAgent.runtime().cleanupSchemaListenerIfUnused(id); + } } return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java index edab5862d11e8..d904da7df315c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java @@ -125,25 +125,34 @@ public boolean isReadOnly() { @Override public boolean takeSnapshot(final File snapshotDir) { - if (schemaRegion.createSnapshot(snapshotDir) - && PipeDataNodeAgent.runtime() - .schemaListener(schemaRegion.getSchemaRegionId()) - .createSnapshot(snapshotDir)) { - listen2Snapshot4PipeListener(true); + if (!schemaRegion.createSnapshot(snapshotDir)) { + return false; + } + + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId()); + if (listener == null) { return true; } - return false; + + if (!listener.createSnapshot(snapshotDir)) { + return false; + } + listen2Snapshot4PipeListener(true); + return true; } @Override public void loadSnapshot(final File latestSnapshotRootDir) { schemaRegion.loadSnapshot(latestSnapshotRootDir); - PipeDataNodeAgent.runtime() - .schemaListener(schemaRegion.getSchemaRegionId()) - .loadSnapshot(latestSnapshotRootDir); - // We recompute the snapshot for pipe listener when loading snapshot - // to recover the newest snapshot in cache - listen2Snapshot4PipeListener(false); + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId()); + if (listener != null) { + listener.loadSnapshot(latestSnapshotRootDir); + // We recompute the snapshot for pipe listener when loading snapshot + // to recover the newest snapshot in cache + listen2Snapshot4PipeListener(false); + } } public void listen2Snapshot4PipeListener(final boolean isTmp) { @@ -154,7 +163,10 @@ public void listen2Snapshot4PipeListener(final boolean isTmp) { .toString(), isTmp); final SchemaRegionListeningQueue listener = - PipeDataNodeAgent.runtime().schemaListener(schemaRegion.getSchemaRegionId()); + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId()); + if (listener == null) { + return; + } if (Objects.isNull(snapshotPaths) || Objects.isNull(snapshotPaths.get(0))) { if (listener.isOpened()) { logger.warn( @@ -181,9 +193,11 @@ public TSStatus write(final IConsensusRequest request) { final TSStatus result = ((PlanNode) request).accept(new SchemaExecutionVisitor(), schemaRegion); if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - PipeDataNodeAgent.runtime() - .schemaListener(schemaRegion.getSchemaRegionId()) - .tryListenToNode((PlanNode) request); + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId()); + if (listener != null) { + listener.tryListenToNode((PlanNode) request); + } } return result; } catch (final IllegalArgumentException e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java index 8f75b7b188541..3d3547f649ba6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java @@ -148,6 +148,10 @@ public SchemaRegionListeningQueue schemaListener(SchemaRegionId schemaRegionId) return regionListenerManager.listener(schemaRegionId); } + public SchemaRegionListeningQueue schemaListenerIfPresent(SchemaRegionId schemaRegionId) { + return regionListenerManager.listenerIfPresent(schemaRegionId); + } + public int increaseAndGetSchemaListenerReferenceCount(SchemaRegionId schemaRegionId) { return regionListenerManager.increaseAndGetReferenceCount(schemaRegionId); } @@ -156,6 +160,14 @@ public int decreaseAndGetSchemaListenerReferenceCount(SchemaRegionId schemaRegio return regionListenerManager.decreaseAndGetReferenceCount(schemaRegionId); } + public void cleanupSchemaListenerIfUnused(SchemaRegionId schemaRegionId) { + regionListenerManager.cleanupListenerIfUnused(schemaRegionId); + } + + public void clearSchemaRegionState(SchemaRegionId schemaRegionId) { + regionListenerManager.clearSchemaRegionState(schemaRegionId); + } + public void notifySchemaLeaderReady(SchemaRegionId schemaRegionId) { regionListenerManager.notifyLeaderReady(schemaRegionId); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java index d28929b36a216..3b03146256bb5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.pipe.agent.runtime; import org.apache.iotdb.commons.consensus.SchemaRegionId; -import org.apache.iotdb.commons.pipe.agent.task.PipeTask; import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionListenerMetrics; import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; @@ -34,6 +33,7 @@ public class PipeSchemaRegionListenerManager { private final Map id2ListenerMap = new ConcurrentHashMap<>(); + private final Map id2LeaderReadyMap = new ConcurrentHashMap<>(); public synchronized Set regionIds() { return id2ListenerMap.keySet(); @@ -44,6 +44,12 @@ public synchronized SchemaRegionListeningQueue listener(final SchemaRegionId sch .listeningQueue; } + public synchronized SchemaRegionListeningQueue listenerIfPresent( + final SchemaRegionId schemaRegionId) { + final PipeSchemaRegionListener listener = id2ListenerMap.get(schemaRegionId); + return listener == null ? null : listener.listeningQueue; + } + public synchronized int increaseAndGetReferenceCount(final SchemaRegionId schemaRegionId) { return id2ListenerMap .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new) @@ -52,28 +58,55 @@ public synchronized int increaseAndGetReferenceCount(final SchemaRegionId schema } public synchronized int decreaseAndGetReferenceCount(final SchemaRegionId schemaRegionId) { - return id2ListenerMap - .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new) - .listeningQueueReferenceCount - .updateAndGet(v -> v > 0 ? v - 1 : 0); + final PipeSchemaRegionListener listener = id2ListenerMap.get(schemaRegionId); + if (listener == null) { + return 0; + } + + final int referenceCount = + listener.listeningQueueReferenceCount.updateAndGet(v -> v > 0 ? v - 1 : 0); + if (referenceCount == 0 && !listener.listeningQueue.isOpened()) { + cleanupListenerIfUnused(schemaRegionId, listener); + } + return referenceCount; } public synchronized void notifyLeaderReady(final SchemaRegionId schemaRegionId) { - id2ListenerMap - .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new) - .notifyLeaderReady(); + id2LeaderReadyMap.computeIfAbsent(schemaRegionId, id -> new AtomicBoolean()).set(true); } public synchronized void notifyLeaderUnavailable(final SchemaRegionId schemaRegionId) { - id2ListenerMap - .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new) - .notifyLeaderUnavailable(); + id2LeaderReadyMap.computeIfAbsent(schemaRegionId, id -> new AtomicBoolean()).set(false); } public synchronized boolean isLeaderReady(final SchemaRegionId schemaRegionId) { - return id2ListenerMap - .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new) - .isLeaderReady(); + final AtomicBoolean isLeaderReady = id2LeaderReadyMap.get(schemaRegionId); + return isLeaderReady != null && isLeaderReady.get(); + } + + public synchronized void cleanupListenerIfUnused(final SchemaRegionId schemaRegionId) { + final PipeSchemaRegionListener listener = id2ListenerMap.get(schemaRegionId); + if (listener != null) { + cleanupListenerIfUnused(schemaRegionId, listener); + } + } + + public synchronized void clearSchemaRegionState(final SchemaRegionId schemaRegionId) { + final PipeSchemaRegionListener listener = id2ListenerMap.remove(schemaRegionId); + if (listener != null) { + PipeSchemaRegionListenerMetrics.getInstance().deregister(schemaRegionId.getId()); + } + id2LeaderReadyMap.remove(schemaRegionId); + } + + private void cleanupListenerIfUnused( + final SchemaRegionId schemaRegionId, final PipeSchemaRegionListener listener) { + if (listener.listeningQueueReferenceCount.get() > 0 || listener.listeningQueue.isOpened()) { + return; + } + if (id2ListenerMap.remove(schemaRegionId, listener)) { + PipeSchemaRegionListenerMetrics.getInstance().deregister(schemaRegionId.getId()); + } } private static class PipeSchemaRegionListener { @@ -81,33 +114,9 @@ private static class PipeSchemaRegionListener { private final SchemaRegionListeningQueue listeningQueue = new SchemaRegionListeningQueue(); private final AtomicInteger listeningQueueReferenceCount = new AtomicInteger(0); - private final AtomicBoolean isLeaderReady = new AtomicBoolean(false); - protected PipeSchemaRegionListener(final SchemaRegionId schemaRegionId) { PipeSchemaRegionListenerMetrics.getInstance() .register(listeningQueue, schemaRegionId.getId()); } - - /** - * Get leader ready state, DO NOT use consensus layer's leader ready flag because - * SimpleConsensus' ready flag is always {@code true}. Note that this flag has nothing to do - * with listening and a {@link PipeTask} starts only iff the current node is a leader and ready. - * - * @return {@code true} iff the current node is a leader and ready - */ - private boolean isLeaderReady() { - return isLeaderReady.get(); - } - - // Leader ready flag has the following effect - // 1. The linked list starts serving only after leader gets ready - // 2. Config pipe task is only created after leader gets ready - private void notifyLeaderReady() { - isLeaderReady.set(true); - } - - private void notifyLeaderUnavailable() { - isLeaderReady.set(false); - } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 06d6a512b30a4..fd57495b14c47 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -39,6 +39,7 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeType; +import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; @@ -59,6 +60,7 @@ import org.apache.iotdb.db.pipe.source.dataregion.DataRegionListeningFilter; import org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener; import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningFilter; +import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; @@ -267,10 +269,13 @@ private Set clearSchemaRegionListeningQueueIfNecessary( } schemaRegionId2ListeningQueueNewFirstIndex.forEach( - (schemaRegionId, listeningQueueNewFirstIndex) -> - PipeDataNodeAgent.runtime() - .schemaListener(new SchemaRegionId(schemaRegionId)) - .removeBefore(listeningQueueNewFirstIndex)); + (schemaRegionId, listeningQueueNewFirstIndex) -> { + final SchemaRegionListeningQueue listeningQueue = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(new SchemaRegionId(schemaRegionId)); + if (listeningQueue != null) { + listeningQueue.removeBefore(listeningQueueNewFirstIndex); + } + }); return schemaRegionId2ListeningQueueNewFirstIndex.keySet(); } @@ -323,6 +328,7 @@ protected boolean dropPipe(final String pipeName, final long creationTime) { final String taskId = pipeName + "_" + creationTime; PipeTsFileToTabletsMetrics.getInstance().deregister(taskId); PipeDataNodeSinglePipeMetrics.getInstance().deregister(taskId); + PipeEventCommitManager.getInstance().clear(pipeName, creationTime); return true; } @@ -351,6 +357,7 @@ protected boolean dropPipe(final String pipeName) { final String taskId = pipeName + "_" + creationTime; PipeTsFileToTabletsMetrics.getInstance().deregister(taskId); PipeDataNodeSinglePipeMetrics.getInstance().deregister(taskId); + PipeEventCommitManager.getInstance().clear(pipeName, creationTime); // When the pipe contains no pipe tasks, there is no corresponding prefetching queue for the // subscribed pipe, so the subscription needs to be manually marked as completed. if (!hasPipeTasks && PipeStaticMeta.isSubscriptionPipe(pipeName)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java index 63bdbd1daec39..24c930a04a615 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java @@ -842,12 +842,19 @@ public void collectWindowOutputs( @Override public void close() throws Exception { - if (Objects.nonNull(pipeName) - && pipeName2referenceCountMap.compute( - pipeName, (name, count) -> Objects.nonNull(count) ? count - 1 : 0) - == 0) { - pipeName2timeSeries2TimeSeriesRuntimeStateMap.get(pipeName).clear(); - pipeName2timeSeries2TimeSeriesRuntimeStateMap.remove(pipeName); + boolean isLastReference = false; + if (Objects.nonNull(pipeName)) { + isLastReference = + pipeName2referenceCountMap.computeIfPresent( + pipeName, (name, count) -> count > 1 ? count - 1 : null) + == null; + } + if (isLastReference) { + final ConcurrentMap> + timeSeries2RuntimeStateMap = pipeName2timeSeries2TimeSeriesRuntimeStateMap.remove(pipeName); + if (timeSeries2RuntimeStateMap != null) { + timeSeries2RuntimeStateMap.clear(); + } pipeName2LastValueReceiveTimeMap.remove(pipeName); } if (Objects.nonNull(windowingProcessor)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java index 554c6c43dfbb0..4030ae580c717 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java @@ -743,6 +743,7 @@ public boolean isEnableSendTsFileLimit() { @Override public synchronized void discardEventsOfPipe( final String pipeNameToDrop, final long creationTimeToDrop, final int regionId) { + super.discardEventsOfPipe(pipeNameToDrop, creationTimeToDrop, regionId); droppedPipeTaskKeys.add(new Triple<>(pipeNameToDrop, creationTimeToDrop, regionId)); if (isTabletBatchModeEnabled && Objects.nonNull(tabletBatchBuilder)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java index e8c4420861c59..b255e67404edd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java @@ -601,6 +601,7 @@ public TPipeTransferReq compressIfNeeded(final TPipeTransferReq req) throws IOEx @Override public synchronized void discardEventsOfPipe( final String pipeNameToDrop, final long creationTimeToDrop, final int regionId) { + super.discardEventsOfPipe(pipeNameToDrop, creationTimeToDrop, regionId); if (Objects.nonNull(tabletBatchBuilder)) { tabletBatchBuilder.discardEventsOfPipe(pipeNameToDrop, creationTimeToDrop, regionId); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java index 4fc2cfbff9c68..1cd4996324063 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl; import org.apache.iotdb.db.exception.DataRegionException; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; +import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; import org.apache.iotdb.db.schemaengine.SchemaEngine; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.rpc.RpcUtils; @@ -203,7 +204,12 @@ public TSStatus deleteDataRegion(DataRegionId dataRegionId) { public TSStatus deleteSchemaRegion(SchemaRegionId schemaRegionId) { try { schemaEngine.deleteSchemaRegion(schemaRegionId); - PipeDataNodeAgent.runtime().schemaListener(schemaRegionId).close(); + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId); + if (listener != null) { + listener.close(); + } + PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId); schemaRegionLockMap.remove(schemaRegionId); return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully"); } catch (MetadataException e) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java new file mode 100644 index 0000000000000..3e08d23978b2c --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.agent.runtime; + +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; + +import org.junit.Assert; +import org.junit.Test; + +public class PipeSchemaRegionListenerManagerTest { + + @Test + public void testLeaderStateDoesNotCreateListener() { + final PipeSchemaRegionListenerManager manager = new PipeSchemaRegionListenerManager(); + final SchemaRegionId schemaRegionId = new SchemaRegionId(1); + + manager.notifyLeaderReady(schemaRegionId); + + Assert.assertTrue(manager.isLeaderReady(schemaRegionId)); + Assert.assertTrue(manager.regionIds().isEmpty()); + Assert.assertNull(manager.listenerIfPresent(schemaRegionId)); + } + + @Test + public void testCleanupUnusedClosedListener() { + final PipeSchemaRegionListenerManager manager = new PipeSchemaRegionListenerManager(); + final SchemaRegionId schemaRegionId = new SchemaRegionId(2); + + final SchemaRegionListeningQueue listeningQueue = manager.listener(schemaRegionId); + Assert.assertEquals(1, manager.increaseAndGetReferenceCount(schemaRegionId)); + + listeningQueue.open(); + Assert.assertEquals(0, manager.decreaseAndGetReferenceCount(schemaRegionId)); + Assert.assertNotNull(manager.listenerIfPresent(schemaRegionId)); + + listeningQueue.close(); + manager.cleanupListenerIfUnused(schemaRegionId); + + Assert.assertNull(manager.listenerIfPresent(schemaRegionId)); + Assert.assertTrue(manager.regionIds().isEmpty()); + } + + @Test + public void testAutoCleanupAfterLastReferenceReleased() { + final PipeSchemaRegionListenerManager manager = new PipeSchemaRegionListenerManager(); + final SchemaRegionId schemaRegionId = new SchemaRegionId(3); + + manager.increaseAndGetReferenceCount(schemaRegionId); + + Assert.assertEquals(0, manager.decreaseAndGetReferenceCount(schemaRegionId)); + Assert.assertNull(manager.listenerIfPresent(schemaRegionId)); + Assert.assertTrue(manager.regionIds().isEmpty()); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java index b47441753c40e..82ee87f6c8d76 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java @@ -78,6 +78,13 @@ public void deregister(final String pipeName, final long creationTime, final int LOGGER.info("Pipe committer deregistered for pipe on region: {}", committerKey); } + public void clear(final String pipeName, final long creationTime) { + eventCommitterMap.keySet().removeIf(key -> clearIfMatches(key, pipeName, creationTime, true)); + eventCommitterRestartTimesMap + .keySet() + .removeIf(key -> clearIfMatches(key, pipeName, creationTime, false)); + } + /** * Assign a commit id and a key for commit. Make sure {@code EnrichedEvent.pipeName} is set before * calling this. @@ -197,6 +204,20 @@ public void setCommitRateMarker(final BiConsumer commitRateMark this.commitRateMarker = commitRateMarker; } + private boolean clearIfMatches( + final CommitterKey key, + final String pipeName, + final long creationTime, + final boolean shouldClearMetrics) { + if (!Objects.equals(key.getPipeName(), pipeName) || key.getCreationTime() != creationTime) { + return false; + } + if (shouldClearMetrics) { + PipeEventCommitMetrics.getInstance().deregister(key.stringify()); + } + return true; + } + //////////////////////////// singleton //////////////////////////// private PipeEventCommitManager() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java index ebb3d90bc8cab..a7e603bd9bc40 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java @@ -643,7 +643,7 @@ public void rateLimitIfNeeded( */ public synchronized void discardEventsOfPipe( final String pipeName, final long creationTime, final int regionId) { - // Do nothing by default + PIPE_END_POINT_RATE_LIMITER_MAP.remove(new Pair<>(pipeName, creationTime)); } public PipeReceiverStatusHandler statusHandler() { From 29804ff25169f05d98f99d3591056b20a17836d8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 14 May 2026 15:33:50 +0800 Subject: [PATCH 02/14] leak_fix --- .../PipeDataNodeSinglePipeMetrics.java | 135 +++++++++------- .../overview/PipeTsFileToTabletsMetrics.java | 18 ++- .../pipe/agent/task/PipeTaskAgent.java | 102 +++++++++--- .../pipe/agent/task/PipeTaskAgentTest.java | 149 ++++++++++++++++++ 4 files changed, 318 insertions(+), 86 deletions(-) create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java index 6535d371a915c..b9d90fd8a68a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java @@ -139,12 +139,14 @@ public void unbindFrom(final AbstractMetricService metricService) { } private void removeMetrics(final String pipeID) { - removeAutoGauge(pipeID); + final PipeDataNodeRemainingEventAndTimeOperator operator = + remainingEventAndTimeOperatorMap.remove(pipeID); + if (Objects.nonNull(operator) && Objects.nonNull(metricService)) { + removeAutoGauge(operator); + } } - private void removeAutoGauge(final String pipeID) { - final PipeDataNodeRemainingEventAndTimeOperator operator = - remainingEventAndTimeOperatorMap.get(pipeID); + private void removeAutoGauge(final PipeDataNodeRemainingEventAndTimeOperator operator) { metricService.remove( MetricType.AUTO_GAUGE, Metric.PIPE_DATANODE_REMAINING_EVENT_COUNT.toString(), @@ -190,14 +192,31 @@ private void removeAutoGauge(final String pipeID) { Metric.PIPE_TSFILE_EVENT_TRANSFER_TIME.toString(), Tag.NAME.toString(), operator.getPipeName()); - remainingEventAndTimeOperatorMap.remove(pipeID); } //////////////////////////// register & deregister (pipe integration) //////////////////////////// + private static String generatePipeID(final String pipeName, final long creationTime) { + return pipeName + "_" + creationTime; + } + + private boolean isPipeAlive(final String pipeName, final long creationTime) { + return PipeDataNodeAgent.task().getPipeCreationTime(pipeName) == creationTime; + } + + private PipeDataNodeRemainingEventAndTimeOperator getOrCreateOperatorIfPipeAlive( + final String pipeName, final long creationTime) { + if (!isPipeAlive(pipeName, creationTime)) { + return null; + } + return remainingEventAndTimeOperatorMap.computeIfAbsent( + generatePipeID(pipeName, creationTime), + key -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)); + } + public void register(final IoTDBDataRegionSource source) { // The metric is global thus the regionId is omitted - final String pipeID = source.getPipeName() + "_" + source.getCreationTime(); + final String pipeID = generatePipeID(source.getPipeName(), source.getCreationTime()); remainingEventAndTimeOperatorMap.computeIfAbsent( pipeID, k -> @@ -210,7 +229,7 @@ public void register(final IoTDBDataRegionSource source) { public void register(final IoTDBSchemaRegionSource source) { // The metric is global thus the regionId is omitted - final String pipeID = source.getPipeName() + "_" + source.getCreationTime(); + final String pipeID = generatePipeID(source.getPipeName(), source.getCreationTime()); remainingEventAndTimeOperatorMap .computeIfAbsent( pipeID, @@ -224,19 +243,20 @@ public void register(final IoTDBSchemaRegionSource source) { } public void increaseInsertNodeEventCount(final String pipeName, final long creationTime) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .increaseInsertNodeEventCount(); + final PipeDataNodeRemainingEventAndTimeOperator operator = + getOrCreateOperatorIfPipeAlive(pipeName, creationTime); + if (Objects.nonNull(operator)) { + operator.increaseInsertNodeEventCount(); + } } public void decreaseInsertNodeEventCount( final String pipeName, final long creationTime, final long transferTime) { final PipeDataNodeRemainingEventAndTimeOperator operator = - remainingEventAndTimeOperatorMap.computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)); + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.isNull(operator)) { + return; + } operator.decreaseInsertNodeEventCount(); @@ -247,46 +267,44 @@ public void decreaseInsertNodeEventCount( public void updateInsertNodeTransferTimer( final String pipeName, final long creationTime, final long transferTime) { - if (transferTime > 0) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .getInsertNodeTransferTimer() - .update(transferTime, TimeUnit.NANOSECONDS); + final PipeDataNodeRemainingEventAndTimeOperator operator = + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.nonNull(operator) && transferTime > 0) { + operator.getInsertNodeTransferTimer().update(transferTime, TimeUnit.NANOSECONDS); } } public void increaseRawTabletEventCount(final String pipeName, final long creationTime) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .increaseRawTabletEventCount(); + final PipeDataNodeRemainingEventAndTimeOperator operator = + getOrCreateOperatorIfPipeAlive(pipeName, creationTime); + if (Objects.nonNull(operator)) { + operator.increaseRawTabletEventCount(); + } } public void decreaseRawTabletEventCount(final String pipeName, final long creationTime) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .decreaseRawTabletEventCount(); + final PipeDataNodeRemainingEventAndTimeOperator operator = + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.nonNull(operator)) { + operator.decreaseRawTabletEventCount(); + } } public void increaseTsFileEventCount(final String pipeName, final long creationTime) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .increaseTsFileEventCount(); + final PipeDataNodeRemainingEventAndTimeOperator operator = + getOrCreateOperatorIfPipeAlive(pipeName, creationTime); + if (Objects.nonNull(operator)) { + operator.increaseTsFileEventCount(); + } } public void decreaseTsFileEventCount( final String pipeName, final long creationTime, final long transferTime) { final PipeDataNodeRemainingEventAndTimeOperator operator = - remainingEventAndTimeOperatorMap.computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)); + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.isNull(operator)) { + return; + } operator.decreaseTsFileEventCount(); @@ -297,30 +315,27 @@ public void decreaseTsFileEventCount( public void updateTsFileTransferTimer( final String pipeName, final long creationTime, final long transferTime) { - if (transferTime > 0) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .getTsFileTransferTimer() - .update(transferTime, TimeUnit.NANOSECONDS); + final PipeDataNodeRemainingEventAndTimeOperator operator = + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.nonNull(operator) && transferTime > 0) { + operator.getTsFileTransferTimer().update(transferTime, TimeUnit.NANOSECONDS); } } public void increaseHeartbeatEventCount(final String pipeName, final long creationTime) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .increaseHeartbeatEventCount(); + final PipeDataNodeRemainingEventAndTimeOperator operator = + getOrCreateOperatorIfPipeAlive(pipeName, creationTime); + if (Objects.nonNull(operator)) { + operator.increaseHeartbeatEventCount(); + } } public void decreaseHeartbeatEventCount(final String pipeName, final long creationTime) { - remainingEventAndTimeOperatorMap - .computeIfAbsent( - pipeName + "_" + creationTime, - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)) - .decreaseHeartbeatEventCount(); + final PipeDataNodeRemainingEventAndTimeOperator operator = + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.nonNull(operator)) { + operator.decreaseHeartbeatEventCount(); + } } public void thawRate(final String pipeID) { @@ -350,9 +365,7 @@ public void deregister(final String pipeID) { pipeID); return; } - if (Objects.nonNull(metricService)) { - removeMetrics(pipeID); - } + removeMetrics(pipeID); } public void markRegionCommit(final String pipeID, final boolean isDataRegion) { @@ -395,7 +408,7 @@ public Pair getRemainingEventAndTime( final String pipeName, final long creationTime) { final PipeDataNodeRemainingEventAndTimeOperator operator = remainingEventAndTimeOperatorMap.computeIfAbsent( - pipeName + "_" + creationTime, + generatePipeID(pipeName, creationTime), k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)); return new Pair<>(operator.getRemainingNonHeartbeatEvents(), operator.getRemainingTime()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java index f9436377bb3c6..90260fd17fc5c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java @@ -110,39 +110,47 @@ public void unbindFrom(final AbstractMetricService metricService) { } private void removeMetrics(final String pipeID) { + pipeTimerMap.remove(pipeID); metricService.remove( MetricType.TIMER, Metric.PIPE_TSFILE_TO_TABLETS_TIME.toString(), Tag.NAME.toString(), pipeID); - pipeTimerMap.remove(pipeID); + pipeRateMap.remove(pipeID); metricService.remove( MetricType.RATE, Metric.PIPE_TSFILE_TO_TABLETS_RATE.toString(), Tag.NAME.toString(), pipeID); - pipeRateMap.remove(pipeID); + pipeTabletCountMap.remove(pipeID); metricService.remove( MetricType.COUNTER, Metric.PIPE_TSFILE_TO_TABLETS_COUNT.toString(), Tag.NAME.toString(), pipeID); - pipeTabletCountMap.remove(pipeID); + pipeTabletMemoryMap.remove(pipeID); metricService.remove( MetricType.COUNTER, Metric.PIPE_TSFILE_TO_TABLETS_TOTAL_MEMORY.toString(), Tag.NAME.toString(), pipeID); - pipeTabletMemoryMap.remove(pipeID); + pipeParseFileCountMap.remove(pipeID); metricService.remove( MetricType.COUNTER, Metric.PIPE_TSFILE_PARSE_FILE_COUNT.toString(), Tag.NAME.toString(), pipeID); + } + + private void removePipeStats(final String pipeID) { + pipeTimerMap.remove(pipeID); + pipeRateMap.remove(pipeID); + pipeTabletCountMap.remove(pipeID); + pipeTabletMemoryMap.remove(pipeID); pipeParseFileCountMap.remove(pipeID); } @@ -165,6 +173,8 @@ public void deregister(final String pipeID) { try { if (Objects.nonNull(metricService)) { removeMetrics(pipeID); + } else { + removePipeStats(pipeID); } } finally { pipe.remove(pipeID); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index 99d452987943e..41efbef9c047d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -60,6 +60,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -104,10 +105,14 @@ public abstract class PipeTaskAgent { protected final PipeMetaKeeper pipeMetaKeeper; protected final PipeTaskManager pipeTaskManager; + private final Map pipeNameWithCreationTime2FloatingMemoryUsageInByteMap; + private final Map> pipeName2CreationTimeSetMap; protected PipeTaskAgent() { pipeMetaKeeper = new PipeMetaKeeper(); pipeTaskManager = new PipeTaskManager(); + pipeNameWithCreationTime2FloatingMemoryUsageInByteMap = new ConcurrentHashMap<>(); + pipeName2CreationTimeSetMap = new ConcurrentHashMap<>(); // Help PipeEndPointRateLimiter to check if the pipe is still alive PipeEndPointRateLimiter.setTaskAgent(this); @@ -601,6 +606,7 @@ protected boolean dropPipe(final String pipeName, final long creationTime) { // Remove pipe meta from pipe meta keeper pipeMetaKeeper.removePipeMeta(pipeName); + cleanupFloatingMemoryUsageCounterIfNecessary(pipeName, creationTime); return true; } @@ -639,6 +645,8 @@ protected boolean dropPipe(final String pipeName) { // Remove pipe meta from pipe meta keeper pipeMetaKeeper.removePipeMeta(pipeName); + cleanupFloatingMemoryUsageCounterIfNecessary( + pipeName, existedPipeMeta.getStaticMeta().getCreationTime()); return true; } @@ -1170,6 +1178,57 @@ public static boolean isSnapshotMode(final PipeParameters parameters) { ///////////////////////// Maintain meta info ///////////////////////// + private static String generatePipeNameWithCreationTime( + final String pipeName, final long creationTime) { + return pipeName + "_" + creationTime; + } + + private AtomicLong getOrCreateFloatingMemoryUsageCounter( + final String pipeName, final long creationTime) { + pipeName2CreationTimeSetMap + .computeIfAbsent(pipeName, key -> ConcurrentHashMap.newKeySet()) + .add(creationTime); + return pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.computeIfAbsent( + generatePipeNameWithCreationTime(pipeName, creationTime), key -> new AtomicLong(0)); + } + + private void tryCleanupFloatingMemoryUsageCounter( + final String pipeName, final long creationTime, final AtomicLong counter) { + if (counter.get() != 0) { + return; + } + + final PipeMeta currentPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); + if (Objects.nonNull(currentPipeMeta) + && currentPipeMeta.getStaticMeta().getCreationTime() == creationTime) { + return; + } + + final String pipeNameWithCreationTime = + generatePipeNameWithCreationTime(pipeName, creationTime); + pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.remove(pipeNameWithCreationTime, counter); + + if (!pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.containsKey( + pipeNameWithCreationTime)) { + pipeName2CreationTimeSetMap.computeIfPresent( + pipeName, + (key, creationTimes) -> { + creationTimes.remove(creationTime); + return creationTimes.isEmpty() ? null : creationTimes; + }); + } + } + + private void cleanupFloatingMemoryUsageCounterIfNecessary( + final String pipeName, final long creationTime) { + final AtomicLong counter = + pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.get( + generatePipeNameWithCreationTime(pipeName, creationTime)); + if (Objects.nonNull(counter)) { + tryCleanupFloatingMemoryUsageCounter(pipeName, creationTime, counter); + } + } + public long getPipeCreationTime(final String pipeName) { final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); return pipeMeta == null ? 0 : pipeMeta.getStaticMeta().getCreationTime(); @@ -1178,7 +1237,7 @@ public long getPipeCreationTime(final String pipeName) { public String getPipeNameWithCreationTime(final String pipeName, final long creationTime) { final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); return pipeMeta == null - ? pipeName + "_" + creationTime + ? generatePipeNameWithCreationTime(pipeName, creationTime) : ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()).getPipeNameWithCreationTime(); } @@ -1192,22 +1251,23 @@ public CommitterKey getCommitterKey( } public long getAllFloatingMemoryUsageInByte() { - final AtomicLong bytes = new AtomicLong(0); - pipeMetaKeeper - .getPipeMetaList() - .forEach( - pipeMeta -> - bytes.addAndGet( - ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()) - .getFloatingMemoryUsageInByte())); - return bytes.get(); + return pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.values().stream() + .mapToLong(AtomicLong::get) + .sum(); } public long getFloatingMemoryUsageInByte(final String pipeName) { - final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); - return pipeMeta == null - ? 0 - : ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()).getFloatingMemoryUsageInByte(); + final Set creationTimes = pipeName2CreationTimeSetMap.get(pipeName); + if (Objects.isNull(creationTimes)) { + return 0; + } + + return creationTimes.stream() + .map(creationTime -> generatePipeNameWithCreationTime(pipeName, creationTime)) + .map(pipeNameWithCreationTime2FloatingMemoryUsageInByteMap::get) + .filter(Objects::nonNull) + .mapToLong(AtomicLong::get) + .sum(); } public void addFloatingMemoryUsageInByte( @@ -1215,18 +1275,18 @@ public void addFloatingMemoryUsageInByte( final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); // To avoid stale pipe before alter if (Objects.nonNull(pipeMeta) && pipeMeta.getStaticMeta().getCreationTime() == creationTime) { - ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()) - .addFloatingMemoryUsageInByte(sizeInByte); + getOrCreateFloatingMemoryUsageCounter(pipeName, creationTime).addAndGet(sizeInByte); } } public void decreaseFloatingMemoryUsageInByte( final String pipeName, final long creationTime, final long sizeInByte) { - final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); - // To avoid stale pipe before alter - if (Objects.nonNull(pipeMeta) && pipeMeta.getStaticMeta().getCreationTime() == creationTime) { - ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()) - .decreaseFloatingMemoryUsageInByte(sizeInByte); + final AtomicLong counter = + pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.get( + generatePipeNameWithCreationTime(pipeName, creationTime)); + if (Objects.nonNull(counter)) { + counter.addAndGet(-sizeInByte); + tryCleanupFloatingMemoryUsageCounter(pipeName, creationTime, counter); } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java new file mode 100644 index 0000000000000..50a485df9d260 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.agent.task; + +import org.apache.iotdb.common.rpc.thrift.TPipeHeartbeatResp; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq; + +import org.apache.thrift.TException; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; + +public class PipeTaskAgentTest { + + @Test + public void testFloatingMemoryUsageSurvivesDropUntilLateRelease() throws IllegalPathException { + final DummyPipeTaskAgent agent = new DummyPipeTaskAgent(); + + agent.createPipeForTest(generatePipeMeta("pipe", 1L)); + agent.addFloatingMemoryUsageInByte("pipe", 1L, 100L); + Assert.assertEquals(100L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe")); + + agent.dropPipeForTest("pipe", 1L); + Assert.assertEquals(100L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe")); + + agent.createPipeForTest(generatePipeMeta("pipe", 2L)); + agent.addFloatingMemoryUsageInByte("pipe", 2L, 20L); + Assert.assertEquals(120L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(120L, agent.getFloatingMemoryUsageInByte("pipe")); + + agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 100L); + Assert.assertEquals(20L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(20L, agent.getFloatingMemoryUsageInByte("pipe")); + + agent.dropPipeForTest("pipe", 2L); + agent.decreaseFloatingMemoryUsageInByte("pipe", 2L, 20L); + Assert.assertEquals(0L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(0L, agent.getFloatingMemoryUsageInByte("pipe")); + } + + @Test + public void testZeroFloatingMemoryUsageCounterIsCleanedAfterDrop() throws Exception { + final DummyPipeTaskAgent agent = new DummyPipeTaskAgent(); + + agent.createPipeForTest(generatePipeMeta("pipe", 1L)); + agent.addFloatingMemoryUsageInByte("pipe", 1L, 100L); + agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 100L); + agent.dropPipeForTest("pipe", 1L); + + Assert.assertTrue( + getMapField(agent, "pipeNameWithCreationTime2FloatingMemoryUsageInByteMap").isEmpty()); + Assert.assertTrue(getMapField(agent, "pipeName2CreationTimeSetMap").isEmpty()); + } + + @SuppressWarnings("unchecked") + private static Map getMapField(final PipeTaskAgent agent, final String fieldName) + throws NoSuchFieldException, IllegalAccessException { + final Field field = PipeTaskAgent.class.getDeclaredField(fieldName); + field.setAccessible(true); + return (Map) field.get(agent); + } + + private static PipeMeta generatePipeMeta(final String pipeName, final long creationTime) { + return new PipeMeta( + new PipeStaticMeta( + pipeName, creationTime, new HashMap<>(), new HashMap<>(), new HashMap<>()), + new PipeRuntimeMeta(new ConcurrentHashMap<>())); + } + + private static class DummyPipeTaskAgent extends PipeTaskAgent { + + private boolean createPipeForTest(final PipeMeta pipeMeta) throws IllegalPathException { + return createPipe(pipeMeta); + } + + private boolean dropPipeForTest(final String pipeName, final long creationTime) { + return dropPipe(pipeName, creationTime); + } + + @Override + protected boolean isShutdown() { + return false; + } + + @Override + protected void thawRate(final String pipeName, final long creationTime) { + // Do nothing + } + + @Override + protected void freezeRate(final String pipeName, final long creationTime) { + // Do nothing + } + + @Override + protected Map buildPipeTasks(final PipeMeta pipeMetaFromCoordinator) { + return new HashMap<>(); + } + + @Override + protected void createPipeTask( + final int consensusGroupId, + final PipeStaticMeta pipeStaticMeta, + final org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta pipeTaskMeta) { + // Do nothing + } + + @Override + protected void collectPipeMetaListInternal( + final TPipeHeartbeatReq req, final TPipeHeartbeatResp resp) throws TException { + // Do nothing + } + + @Override + public void runPipeTasks( + final Collection pipeTasks, final Consumer runSingle) { + pipeTasks.forEach(runSingle); + } + } +} From 1944c67b61d2e75c509b36697128c25f386a9ea6 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 14 May 2026 16:27:52 +0800 Subject: [PATCH 03/14] spotless --- .../statemachine/schemaregion/SchemaExecutionVisitor.java | 3 ++- .../apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java | 3 ++- .../iotdb/db/pipe/processor/aggregate/AggregateProcessor.java | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index 2fc2694c4c564..0319cf5be9039 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -850,7 +850,8 @@ public TSStatus visitPipeOperateSchemaQueueNode( queue.open(); } } else { - final SchemaRegionListeningQueue queue = PipeDataNodeAgent.runtime().schemaListenerIfPresent(id); + final SchemaRegionListeningQueue queue = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(id); if (queue != null && queue.isOpened()) { logger.info("Closed pipe listening queue on schema region {}", id); queue.close(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index fd57495b14c47..36e1fe1ae9fe6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -271,7 +271,8 @@ private Set clearSchemaRegionListeningQueueIfNecessary( schemaRegionId2ListeningQueueNewFirstIndex.forEach( (schemaRegionId, listeningQueueNewFirstIndex) -> { final SchemaRegionListeningQueue listeningQueue = - PipeDataNodeAgent.runtime().schemaListenerIfPresent(new SchemaRegionId(schemaRegionId)); + PipeDataNodeAgent.runtime() + .schemaListenerIfPresent(new SchemaRegionId(schemaRegionId)); if (listeningQueue != null) { listeningQueue.removeBefore(listeningQueueNewFirstIndex); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java index 24c930a04a615..b6531897ba302 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java @@ -851,7 +851,8 @@ public void close() throws Exception { } if (isLastReference) { final ConcurrentMap> - timeSeries2RuntimeStateMap = pipeName2timeSeries2TimeSeriesRuntimeStateMap.remove(pipeName); + timeSeries2RuntimeStateMap = + pipeName2timeSeries2TimeSeriesRuntimeStateMap.remove(pipeName); if (timeSeries2RuntimeStateMap != null) { timeSeries2RuntimeStateMap.clear(); } From a0d22e6899008ad23ef18004e646dc7f105544fd Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:26:55 +0800 Subject: [PATCH 04/14] Restore schema pipe queue from snapshots --- .../SchemaRegionStateMachine.java | 4 ++- .../runtime/PipeDataNodeRuntimeAgent.java | 6 ++++ .../PipeSchemaRegionListenerManager.java | 10 ++++++ .../SchemaRegionListeningQueue.java | 4 +++ .../PipeSchemaRegionListenerManagerTest.java | 32 +++++++++++++++++++ 5 files changed, 55 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java index 5f2ef32c9c415..38e04e7769bc5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java @@ -153,7 +153,9 @@ public boolean loadSnapshot(final File latestSnapshotRootDir) { // failure instead of treating a fallback-to-empty load as success. final boolean loadSucceeded = schemaRegion.loadSnapshot(latestSnapshotRootDir); final SchemaRegionListeningQueue listener = - PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId()); + PipeDataNodeAgent.runtime() + .schemaListenerForSnapshotLoad( + schemaRegion.getSchemaRegionId(), latestSnapshotRootDir); if (listener != null) { listener.loadSnapshot(latestSnapshotRootDir); // We recompute the snapshot for pipe listener when loading snapshot diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java index c934e9a907f9e..e0dbc0cd4b0d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java @@ -56,6 +56,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; @@ -173,6 +174,11 @@ public SchemaRegionListeningQueue schemaListenerIfPresent(SchemaRegionId schemaR return regionListenerManager.listenerIfPresent(schemaRegionId); } + public SchemaRegionListeningQueue schemaListenerForSnapshotLoad( + SchemaRegionId schemaRegionId, File snapshotDir) { + return regionListenerManager.listenerForSnapshotLoad(schemaRegionId, snapshotDir); + } + public int increaseAndGetSchemaListenerReferenceCount(SchemaRegionId schemaRegionId) { return regionListenerManager.increaseAndGetReferenceCount(schemaRegionId); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java index 3b03146256bb5..6d12e95979c61 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionListenerMetrics; import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; +import java.io.File; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -50,6 +51,15 @@ public synchronized SchemaRegionListeningQueue listenerIfPresent( return listener == null ? null : listener.listeningQueue; } + public synchronized SchemaRegionListeningQueue listenerForSnapshotLoad( + final SchemaRegionId schemaRegionId, final File snapshotDir) { + final PipeSchemaRegionListener listener = id2ListenerMap.get(schemaRegionId); + if (listener != null) { + return listener.listeningQueue; + } + return SchemaRegionListeningQueue.hasSnapshot(snapshotDir) ? listener(schemaRegionId) : null; + } + public synchronized int increaseAndGetReferenceCount(final SchemaRegionId schemaRegionId) { return id2ListenerMap .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/SchemaRegionListeningQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/SchemaRegionListeningQueue.java index 3ae8bff32bb0b..aec6565e6af48 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/SchemaRegionListeningQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/SchemaRegionListeningQueue.java @@ -114,6 +114,10 @@ public synchronized boolean createSnapshot(final File snapshotDir) { } } + public static boolean hasSnapshot(final File snapshotDir) { + return snapshotDir != null && new File(snapshotDir, SNAPSHOT_FILE_NAME).isFile(); + } + public synchronized void loadSnapshot(final File snapshotDir) { try { super.deserializeFromFile(new File(snapshotDir, SNAPSHOT_FILE_NAME)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java index 3e08d23978b2c..fd7106c74b5a2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java @@ -23,10 +23,17 @@ import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; public class PipeSchemaRegionListenerManagerTest { + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test public void testLeaderStateDoesNotCreateListener() { final PipeSchemaRegionListenerManager manager = new PipeSchemaRegionListenerManager(); @@ -69,4 +76,29 @@ public void testAutoCleanupAfterLastReferenceReleased() { Assert.assertNull(manager.listenerIfPresent(schemaRegionId)); Assert.assertTrue(manager.regionIds().isEmpty()); } + + @Test + public void testCreateListenerForSnapshotLoadOnlyWhenSnapshotExists() throws IOException { + final PipeSchemaRegionListenerManager manager = new PipeSchemaRegionListenerManager(); + final SchemaRegionId schemaRegionId = new SchemaRegionId(4); + final File snapshotDir = temporaryFolder.newFolder(); + + Assert.assertNull(manager.listenerForSnapshotLoad(schemaRegionId, snapshotDir)); + Assert.assertTrue(manager.regionIds().isEmpty()); + + final SchemaRegionListeningQueue originalListener = manager.listener(schemaRegionId); + originalListener.open(); + Assert.assertTrue(originalListener.createSnapshot(snapshotDir)); + originalListener.close(); + manager.clearSchemaRegionState(schemaRegionId); + + final SchemaRegionListeningQueue restoredListener = + manager.listenerForSnapshotLoad(schemaRegionId, snapshotDir); + Assert.assertNotNull(restoredListener); + restoredListener.loadSnapshot(snapshotDir); + Assert.assertTrue(restoredListener.isOpened()); + + restoredListener.close(); + manager.clearSchemaRegionState(schemaRegionId); + } } From 5f71bb0d0e602286e9fe05e78e698fc3325966c9 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:00:57 +0800 Subject: [PATCH 05/14] Pipe: Avoid recreating committer state for late commits --- .../task/progress/PipeEventCommitManager.java | 7 +- .../progress/PipeEventCommitManagerTest.java | 183 ++++++++++++++++++ 2 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java index 231e65bdf82cb..0d0ce8cc410bf 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java @@ -150,10 +150,11 @@ private boolean commitSingleId( final PipeEventCommitter committer = eventCommitterMap.get(committerKey); if (committer == null) { - final int currentRestartTimes = - eventCommitterRestartTimesMap.computeIfAbsent( - generateCommitterRestartTimesKey(committerKey), k -> 0); if (LOGGER.isDebugEnabled()) { + // Late commits must not recreate restart state after the pipe has been cleared. + final int currentRestartTimes = + eventCommitterRestartTimesMap.getOrDefault( + generateCommitterRestartTimesKey(committerKey), 0); if (committerKey.getRestartTimes() < currentRestartTimes) { LOGGER.debug( PipeMessages.STALE_PIPE_EVENT_COMMITTER, diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java new file mode 100644 index 0000000000000..1a639ea1116f2 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.agent.task.progress; + +import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent; +import org.apache.iotdb.commons.pipe.event.EnrichedEvent; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +@RunWith(Parameterized.class) +public class PipeEventCommitManagerTest { + + private static final String PIPE_NAME = "late-commit-test"; + private static final long CREATION_TIME = 1L; + private static final int REGION_ID = 1; + + private final Level logLevel; + private final Logger logger = (Logger) LoggerFactory.getLogger(PipeEventCommitManager.class); + private Level originalLogLevel; + private PipeEventCommitManager manager; + + public PipeEventCommitManagerTest(final Level logLevel) { + this.logLevel = logLevel; + } + + @Parameterized.Parameters(name = "{0}") + public static Iterable logLevels() { + return Arrays.asList(Level.INFO, Level.DEBUG); + } + + @Before + public void setUp() throws Exception { + originalLogLevel = logger.getLevel(); + logger.setLevel(logLevel); + final Constructor constructor = + PipeEventCommitManager.class.getDeclaredConstructor(); + constructor.setAccessible(true); + manager = constructor.newInstance(); + + final PipeTaskAgent taskAgent = mock(PipeTaskAgent.class); + when(taskAgent.getCommitterKey(eq(PIPE_NAME), eq(CREATION_TIME), eq(REGION_ID), anyInt())) + .thenAnswer( + invocation -> + new CommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID, invocation.getArgument(3))); + manager.setTaskAgent(taskAgent); + manager.register(PIPE_NAME, CREATION_TIME, REGION_ID, "iotdb-thrift-sink"); + } + + @After + public void tearDown() { + try { + if (manager != null) { + manager.clear(PIPE_NAME, CREATION_TIME); + } + } finally { + logger.setLevel(originalLogLevel); + } + } + + @Test + public void testLateSingleCommitAfterClearDoesNotRestoreState() throws Exception { + final CommitterKey key = manager.getCommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID); + final Runnable onCommitted = mock(Runnable.class); + final EnrichedEvent event = createEvent(onCommitted); + manager.clear(PIPE_NAME, CREATION_TIME); + + manager.commit(event, key); + + assertTrue(getManagerMap("eventCommitterMap").isEmpty()); + assertTrue(getManagerMap("eventCommitterRestartTimesMap").isEmpty()); + verify(onCommitted, never()).run(); + } + + @Test + public void testLateMultipleCommitsAfterClearDoNotRestoreState() throws Exception { + final CommitterKey key = manager.getCommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID); + final Runnable onCommitted = mock(Runnable.class); + final EnrichedEvent event = createEvent(onCommitted); + when(event.hasMultipleCommitIds()).thenReturn(true); + when(event.getCommitIds()).thenReturn(Arrays.asList(1L, 2L, 3L)); + manager.clear(PIPE_NAME, CREATION_TIME); + + manager.commit(event, key); + + assertTrue(getManagerMap("eventCommitterMap").isEmpty()); + assertTrue(getManagerMap("eventCommitterRestartTimesMap").isEmpty()); + verify(onCommitted, never()).run(); + } + + @Test + public void testRestartRejectsStaleCommitAndAcceptsCurrentCommit() throws Exception { + final CommitterKey oldKey = manager.getCommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID); + final Runnable oldOnCommitted = mock(Runnable.class); + manager.deregister(PIPE_NAME, CREATION_TIME, REGION_ID); + + manager.commit(createEvent(oldOnCommitted), oldKey); + assertEquals( + Integer.valueOf(1), + getManagerMap("eventCommitterRestartTimesMap") + .get(new CommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID))); + + manager.register(PIPE_NAME, CREATION_TIME, REGION_ID, "iotdb-thrift-sink"); + final CommitterKey newKey = manager.getCommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID); + assertEquals(1, newKey.getRestartTimes()); + final Runnable newOnCommitted = mock(Runnable.class); + manager.commit(createEvent(oldOnCommitted), oldKey); + manager.commit(createEvent(newOnCommitted), newKey); + + verify(oldOnCommitted, never()).run(); + verify(newOnCommitted).run(); + assertEquals(1, getManagerMap("eventCommitterMap").size()); + assertEquals(1, getManagerMap("eventCommitterRestartTimesMap").size()); + } + + private static EnrichedEvent createEvent(final Runnable onCommitted) { + final EnrichedEvent event = + mock( + EnrichedEvent.class, + withSettings() + .useConstructor( + PIPE_NAME, + CREATION_TIME, + null, + null, + null, + null, + null, + null, + false, + Long.MIN_VALUE, + Long.MAX_VALUE)); + when(event.needToCommit()).thenReturn(true); + when(event.getCommitId()).thenReturn(1L); + when(event.getOnCommittedHooks()).thenReturn(Collections.singletonList(onCommitted)); + return event; + } + + private Map getManagerMap(final String fieldName) throws Exception { + final Field field = PipeEventCommitManager.class.getDeclaredField(fieldName); + field.setAccessible(true); + return (Map) field.get(manager); + } +} From df8d30d4215d9d662831801925a982c023c74c93 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:11:29 +0800 Subject: [PATCH 06/14] Pipe: Avoid stale metrics state after drop --- .../metric/overview/PipeDataNodeSinglePipeMetrics.java | 7 ++++--- .../iotdb/db/pipe/metric/PipeMetricsRestartTest.java | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java index 4f50eaa151a69..3afd849e8a8be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java @@ -526,9 +526,10 @@ public void markTsFileCollectInvocationCount( public Pair getRemainingEventAndTime( final String pipeName, final long creationTime) { final PipeDataNodeRemainingEventAndTimeOperator operator = - remainingEventAndTimeOperatorMap.computeIfAbsent( - generatePipeID(pipeName, creationTime), - k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)); + remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName, creationTime)); + if (Objects.isNull(operator)) { + return new Pair<>(0L, 0.0); + } return new Pair<>(operator.getRemainingNonHeartbeatEvents(), operator.getRemainingTime()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/PipeMetricsRestartTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/PipeMetricsRestartTest.java index 50e7b63d9dc7d..79277b651af0d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/PipeMetricsRestartTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/PipeMetricsRestartTest.java @@ -148,6 +148,7 @@ public void testSinglePipeMetricsIgnoreLateEventsAfterDeregister() { metrics.updateTsFileTransferTimer(PIPE, CREATION_TIME, 10); metrics.increaseHeartbeatEventCount(PIPE, CREATION_TIME); metrics.decreaseHeartbeatEventCount(PIPE, CREATION_TIME); + assertEquals(0L, (long) metrics.getRemainingEventAndTime(PIPE, CREATION_TIME).getLeft()); assertTrue(metrics.remainingEventAndTimeOperatorMap.isEmpty()); service.restartService(); From d7e2c2e00e9fc249a9036848ade31fa43f10be8f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:39:45 +0800 Subject: [PATCH 07/14] Pipe: Serialize late metric creation with drop --- .../PipeDataNodeSinglePipeMetrics.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java index 3afd849e8a8be..3a47244663edb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java @@ -240,16 +240,24 @@ private boolean isPipeAlive(final String pipeName, final long creationTime) { private PipeDataNodeRemainingEventAndTimeOperator getOrCreateOperatorIfPipeAlive( final String pipeName, final long creationTime) { final String pipeID = generatePipeID(pipeName, creationTime); - final PipeDataNodeRemainingEventAndTimeOperator operator = + PipeDataNodeRemainingEventAndTimeOperator operator = remainingEventAndTimeOperatorMap.get(pipeID); if (Objects.nonNull(operator)) { return operator; } - if (!isPipeAlive(pipeName, creationTime)) { - return null; + + // Serialize creation with deregistration so a late update cannot restore a removed operator. + synchronized (this) { + operator = remainingEventAndTimeOperatorMap.get(pipeID); + if (Objects.nonNull(operator)) { + return operator; + } + if (!isPipeAlive(pipeName, creationTime)) { + return null; + } + return remainingEventAndTimeOperatorMap.computeIfAbsent( + pipeID, key -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)); } - return remainingEventAndTimeOperatorMap.computeIfAbsent( - pipeID, key -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime)); } public synchronized void register(final IoTDBDataRegionSource source) { From 33e41a31cf344bdba5be549069a446071138258b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:20:12 +0800 Subject: [PATCH 08/14] Declare Logback test dependency in node-commons --- iotdb-core/node-commons/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index 3bb68a29562c3..c1b896cb54c80 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -177,6 +177,11 @@ reflections test + + ch.qos.logback + logback-classic + test + org.mockito mockito-core From 6f2cc670214385796d95b94d27db34384b78534e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:29:16 +0800 Subject: [PATCH 09/14] Pipe: prevent stale state after pipe drop --- .../agent/task/PipeDataNodeTaskAgent.java | 3 + .../PipeDataNodeSinglePipeMetrics.java | 2 +- .../PipeDataNodeSinglePipeMetricsTest.java | 210 ++++++++++++++++++ .../pipe/agent/task/PipeTaskAgent.java | 88 ++++---- .../task/progress/PipeEventCommitManager.java | 9 +- .../commons/pipe/sink/protocol/IoTDBSink.java | 10 +- .../pipe/agent/task/PipeTaskAgentTest.java | 146 ++++++++++++ .../progress/PipeEventCommitManagerTest.java | 11 + .../protocol/IoTDBSinkRateLimiterTest.java | 67 ++++++ 9 files changed, 492 insertions(+), 54 deletions(-) create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 17c4a1e689e0a..9cd802bda6cae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -50,6 +50,7 @@ import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; +import org.apache.iotdb.commons.pipe.sink.protocol.IoTDBSink; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -421,6 +422,7 @@ protected boolean dropPipe(final String pipeName, final long creationTime) { PipeTsFileToTabletsMetrics.getInstance().deregister(taskId); PipeDataNodeSinglePipeMetrics.getInstance().deregister(taskId); PipeEventCommitManager.getInstance().clear(pipeName, creationTime); + IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); PipeDataNodeResourceManager.tsfile().cleanPipeTsFileDir(pipeTsFileResourcePipeName); return true; @@ -464,6 +466,7 @@ protected boolean dropPipe(final String pipeName) { PipeTsFileToTabletsMetrics.getInstance().deregister(taskId); PipeDataNodeSinglePipeMetrics.getInstance().deregister(taskId); PipeEventCommitManager.getInstance().clear(pipeName, creationTime); + IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); PipeDataNodeResourceManager.tsfile().cleanPipeTsFileDir(pipeTsFileResourcePipeName); // When the pipe contains no pipe tasks, there is no corresponding prefetching queue for the // subscribed pipe, so the subscription needs to be manually marked as completed. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java index 3a47244663edb..02d18ba488cd1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java @@ -234,7 +234,7 @@ private static String generatePipeID(final String pipeName, final long creationT } private boolean isPipeAlive(final String pipeName, final long creationTime) { - return PipeDataNodeAgent.task().getPipeCreationTime(pipeName) == creationTime; + return PipeDataNodeAgent.task().isPipeExisted(pipeName, creationTime); } private PipeDataNodeRemainingEventAndTimeOperator getOrCreateOperatorIfPipeAlive( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java index 8246f9080d38c..2c751e7b6ecbd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java @@ -20,10 +20,17 @@ package org.apache.iotdb.db.pipe.metric.overview; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; +import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMetaKeeper; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent; import org.apache.iotdb.db.pipe.source.dataregion.IoTDBDataRegionSource; import org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionSource; @@ -37,10 +44,20 @@ import org.mockito.invocation.Invocation; import java.lang.reflect.Field; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockingDetails; @@ -154,6 +171,143 @@ public void testDeregisterClearsOperatorsWhenMetricsAreUnbound() throws Exceptio } } + @Test + public void testMetricCreationUsesExactPipeInstanceAcrossModels() throws Exception { + final PipeDataNodeSinglePipeMetrics metrics = PipeDataNodeSinglePipeMetrics.getInstance(); + final Field metricServiceField = + PipeDataNodeSinglePipeMetrics.class.getDeclaredField("metricService"); + metricServiceField.setAccessible(true); + reset(metrics, metricServiceField); + + final String pipeName = "shared_model_metric_" + System.nanoTime(); + final long treeCreationTime = System.nanoTime(); + final long tableCreationTime = treeCreationTime + 1; + final PipeMeta treePipe = + createPipeMeta(pipeName, treeCreationTime, SystemConstant.SQL_DIALECT_TREE_VALUE); + final PipeMeta tablePipe = + createPipeMeta(pipeName, tableCreationTime, SystemConstant.SQL_DIALECT_TABLE_VALUE); + final PipeMetaKeeper pipeMetaKeeper = getPipeMetaKeeper(); + + try { + pipeMetaKeeper.addPipeMeta(treePipe); + pipeMetaKeeper.addPipeMeta(tablePipe); + + assertEquals(treeCreationTime, PipeDataNodeAgent.task().getPipeCreationTime(pipeName)); + assertTrue(PipeDataNodeAgent.task().isPipeExisted(pipeName, tableCreationTime)); + + metrics.increaseTsFileEventCount(pipeName, tableCreationTime); + + assertTrue( + metrics.remainingEventAndTimeOperatorMap.containsKey(pipeName + "_" + tableCreationTime)); + } finally { + metrics.deregister(pipeName + "_" + tableCreationTime); + pipeMetaKeeper.removePipeMeta(treePipe.getStaticMeta()); + pipeMetaKeeper.removePipeMeta(tablePipe.getStaticMeta()); + reset(metrics, metricServiceField); + } + } + + @Test + public void testMetricCreationSerializesWithDeregister() throws Exception { + final PipeDataNodeSinglePipeMetrics metrics = PipeDataNodeSinglePipeMetrics.getInstance(); + final Field metricServiceField = + PipeDataNodeSinglePipeMetrics.class.getDeclaredField("metricService"); + metricServiceField.setAccessible(true); + reset(metrics, metricServiceField); + + final String pipeName = "metric_drop_race_" + System.nanoTime(); + final long creationTime = System.nanoTime(); + final String pipeID = pipeName + "_" + creationTime; + final PipeMeta pipeMeta = + createPipeMeta(pipeName, creationTime, SystemConstant.SQL_DIALECT_TREE_VALUE); + final PipeMetaKeeper pipeMetaKeeper = getPipeMetaKeeper(); + final Field pipeMetaMapField = PipeMetaKeeper.class.getDeclaredField("pipeNameToPipeMetaMap"); + pipeMetaMapField.setAccessible(true); + @SuppressWarnings("unchecked") + final Map originalPipeMetaMap = + (Map) pipeMetaMapField.get(pipeMetaKeeper); + final BlockingPipeMetaMap blockingPipeMetaMap = + new BlockingPipeMetaMap(originalPipeMetaMap, pipeName, creationTime); + pipeMetaMapField.set(pipeMetaKeeper, blockingPipeMetaMap); + pipeMetaKeeper.addPipeMeta(pipeMeta); + + final CountDownLatch metadataRemoved = new CountDownLatch(1); + final AtomicReference backgroundFailure = new AtomicReference<>(); + final Thread updateThread = + new Thread( + () -> { + try { + metrics.increaseTsFileEventCount(pipeName, creationTime); + } catch (final Throwable e) { + backgroundFailure.compareAndSet(null, e); + } + }); + final Thread deregisterThread = + new Thread( + () -> { + try { + pipeMetaKeeper.removePipeMeta(pipeMeta.getStaticMeta()); + metadataRemoved.countDown(); + metrics.deregister(pipeID); + } catch (final Throwable e) { + backgroundFailure.compareAndSet(null, e); + } + }); + + try { + try { + updateThread.start(); + assertTrue(blockingPipeMetaMap.lookupPaused.await(10, TimeUnit.SECONDS)); + deregisterThread.start(); + assertTrue(metadataRemoved.await(10, TimeUnit.SECONDS)); + assertEquals(Thread.State.BLOCKED, waitForBlockedOrTerminated(deregisterThread)); + } finally { + blockingPipeMetaMap.resumeLookup.countDown(); + updateThread.join(TimeUnit.SECONDS.toMillis(10)); + deregisterThread.join(TimeUnit.SECONDS.toMillis(10)); + } + + assertFalse(updateThread.isAlive()); + assertFalse(deregisterThread.isAlive()); + assertNull(backgroundFailure.get()); + assertFalse(metrics.remainingEventAndTimeOperatorMap.containsKey(pipeID)); + } finally { + pipeMetaKeeper.removePipeMeta(pipeMeta.getStaticMeta()); + pipeMetaMapField.set(pipeMetaKeeper, originalPipeMetaMap); + metrics.deregister(pipeID); + reset(metrics, metricServiceField); + } + } + + private static PipeMeta createPipeMeta( + final String pipeName, final long creationTime, final String sqlDialect) { + final Map sourceAttributes = new HashMap<>(); + sourceAttributes.put(SystemConstant.SQL_DIALECT_KEY, sqlDialect); + return new PipeMeta( + new PipeStaticMeta( + pipeName, creationTime, sourceAttributes, new HashMap<>(), new HashMap<>()), + new PipeRuntimeMeta(new ConcurrentHashMap<>())); + } + + private static PipeMetaKeeper getPipeMetaKeeper() throws Exception { + final Field pipeMetaKeeperField = PipeTaskAgent.class.getDeclaredField("pipeMetaKeeper"); + pipeMetaKeeperField.setAccessible(true); + return (PipeMetaKeeper) pipeMetaKeeperField.get(PipeDataNodeAgent.task()); + } + + private static Thread.State waitForBlockedOrTerminated(final Thread thread) + throws InterruptedException { + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + Thread.State state = thread.getState(); + while (state != Thread.State.BLOCKED + && state != Thread.State.TERMINATED + && System.nanoTime() < deadline) { + Thread.sleep(1); + state = thread.getState(); + } + return state; + } + private static boolean hasCompletionGaugeCreation( final AbstractMetricService metricService, final String pipeName, final long creationTime) { for (final Invocation invocation : mockingDetails(metricService).getInvocations()) { @@ -185,6 +339,62 @@ private static void reset( metricServiceField.set(metrics, null); } + private static class BlockingPipeMetaMap extends ConcurrentHashMap { + + private static final long serialVersionUID = 1L; + + private final String pipeName; + private final long creationTime; + private final AtomicBoolean pauseNextLookup = new AtomicBoolean(true); + private final CountDownLatch lookupPaused = new CountDownLatch(1); + private final CountDownLatch resumeLookup = new CountDownLatch(1); + + private BlockingPipeMetaMap( + final Map originalMap, final String pipeName, final long creationTime) { + putAll(originalMap); + this.pipeName = pipeName; + this.creationTime = creationTime; + } + + @Override + public PipeMeta get(final Object key) { + final PipeMeta pipeMeta = super.get(key); + pauseIfMatching(pipeMeta); + return pipeMeta; + } + + @Override + public Collection values() { + final Collection values = super.values(); + final ArrayList snapshot = new ArrayList<>(values); + if (snapshot.stream().anyMatch(this::matches)) { + pauseIfMatching(snapshot.stream().filter(this::matches).findFirst().orElse(null)); + return snapshot; + } + return values; + } + + private boolean matches(final PipeMeta pipeMeta) { + return pipeMeta != null + && pipeName.equals(pipeMeta.getStaticMeta().getPipeName()) + && creationTime == pipeMeta.getStaticMeta().getCreationTime(); + } + + private void pauseIfMatching(final PipeMeta pipeMeta) { + if (matches(pipeMeta) && pauseNextLookup.compareAndSet(true, false)) { + lookupPaused.countDown(); + try { + if (!resumeLookup.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("Timed out waiting to resume pipe metadata lookup"); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + } + } + private static class TestSource extends PipeRealtimeDataRegionSource { private TestSource( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index 9eb1bd0917d96..cb13440ad84f1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -1296,50 +1296,27 @@ private static String generatePipeNameWithCreationTime( return pipeName + "_" + creationTime; } - private AtomicLong getOrCreateFloatingMemoryUsageCounter( - final String pipeName, final long creationTime) { - pipeName2CreationTimeSetMap - .computeIfAbsent(pipeName, key -> ConcurrentHashMap.newKeySet()) - .add(creationTime); - return pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.computeIfAbsent( - generatePipeNameWithCreationTime(pipeName, creationTime), key -> new AtomicLong(0)); - } - - private void tryCleanupFloatingMemoryUsageCounter( + private AtomicLong cleanupFloatingMemoryUsageCounterIfUnused( final String pipeName, final long creationTime, final AtomicLong counter) { - if (counter.get() != 0) { - return; - } - - final PipeMeta currentPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); - if (Objects.nonNull(currentPipeMeta) - && currentPipeMeta.getStaticMeta().getCreationTime() == creationTime) { - return; + if (counter.get() != 0 || isPipeExisted(pipeName, creationTime)) { + return counter; } - final String pipeNameWithCreationTime = - generatePipeNameWithCreationTime(pipeName, creationTime); - pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.remove(pipeNameWithCreationTime, counter); - - if (!pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.containsKey( - pipeNameWithCreationTime)) { - pipeName2CreationTimeSetMap.computeIfPresent( - pipeName, - (key, creationTimes) -> { - creationTimes.remove(creationTime); - return creationTimes.isEmpty() ? null : creationTimes; - }); - } + pipeName2CreationTimeSetMap.computeIfPresent( + pipeName, + (key, creationTimes) -> { + creationTimes.remove(creationTime); + return creationTimes.isEmpty() ? null : creationTimes; + }); + return null; } private void cleanupFloatingMemoryUsageCounterIfNecessary( final String pipeName, final long creationTime) { - final AtomicLong counter = - pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.get( - generatePipeNameWithCreationTime(pipeName, creationTime)); - if (Objects.nonNull(counter)) { - tryCleanupFloatingMemoryUsageCounter(pipeName, creationTime, counter); - } + pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.computeIfPresent( + generatePipeNameWithCreationTime(pipeName, creationTime), + (key, counter) -> + cleanupFloatingMemoryUsageCounterIfUnused(pipeName, creationTime, counter)); } public long getPipeCreationTime(final String pipeName) { @@ -1396,22 +1373,35 @@ public long getFloatingMemoryUsageInByte(final String pipeName, final long creat public void addFloatingMemoryUsageInByte( final String pipeName, final long creationTime, final long sizeInByte) { - final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); - // To avoid stale pipe before alter - if (Objects.nonNull(pipeMeta) && pipeMeta.getStaticMeta().getCreationTime() == creationTime) { - getOrCreateFloatingMemoryUsageCounter(pipeName, creationTime).addAndGet(sizeInByte); - } + // Late events still own memory after drop and must be paired with their eventual release. + pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.compute( + generatePipeNameWithCreationTime(pipeName, creationTime), + (key, counter) -> { + if (pipeMetaKeeper.getPipeMeta(pipeName, creationTime) == null) { + return counter; + } + final AtomicLong updatedCounter = Objects.isNull(counter) ? new AtomicLong(0) : counter; + updatedCounter.addAndGet(sizeInByte); + pipeName2CreationTimeSetMap.compute( + pipeName, + (name, creationTimes) -> { + final Set updatedCreationTimes = + Objects.isNull(creationTimes) ? ConcurrentHashMap.newKeySet() : creationTimes; + updatedCreationTimes.add(creationTime); + return updatedCreationTimes; + }); + return updatedCounter; + }); } public void decreaseFloatingMemoryUsageInByte( final String pipeName, final long creationTime, final long sizeInByte) { - final AtomicLong counter = - pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.get( - generatePipeNameWithCreationTime(pipeName, creationTime)); - if (Objects.nonNull(counter)) { - counter.addAndGet(-sizeInByte); - tryCleanupFloatingMemoryUsageCounter(pipeName, creationTime, counter); - } + pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.computeIfPresent( + generatePipeNameWithCreationTime(pipeName, creationTime), + (key, counter) -> { + counter.addAndGet(-sizeInByte); + return cleanupFloatingMemoryUsageCounterIfUnused(pipeName, creationTime, counter); + }); } public void recordPipeResourceFailure( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java index 0d0ce8cc410bf..47a519184fde4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java @@ -94,8 +94,15 @@ public void enrichWithCommitterKeyAndCommitId( return; } + final String pipeName = event.getPipeName(); + final CommitterKey restartTimesKey = + generateCommitterRestartTimesKey(pipeName, creationTime, regionId); final CommitterKey committerKey = - generateCommitterKey(event.getPipeName(), creationTime, regionId); + taskAgent.getCommitterKey( + pipeName, + creationTime, + regionId, + eventCommitterRestartTimesMap.getOrDefault(restartTimesKey, 0)); final PipeEventCommitter committer = eventCommitterMap.get(committerKey); if (committer == null) { return; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java index f5b32008bd6e4..f865fb1734a48 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java @@ -689,13 +689,17 @@ public void rateLimitIfNeeded( GLOBAL_RATE_LIMITER.acquire(bytesLength); } + public static void clearPipeRateLimiter(final String pipeName, final long creationTime) { + PIPE_END_POINT_RATE_LIMITER_MAP.remove(new Pair<>(pipeName, creationTime)); + } + /** - * When a pipe is dropped, the connector maybe reused and will not be closed. We need to discard - * its batched or queued events in the output pipe connector. + * When a region task is dropped, the connector may be reused and will not be closed. Discard its + * batched or queued events in the output pipe connector. */ public synchronized void discardEventsOfPipe( final String pipeName, final long creationTime, final int regionId) { - PIPE_END_POINT_RATE_LIMITER_MAP.remove(new Pair<>(pipeName, creationTime)); + // Rate limiter state is pipe-wide and is cleared only after the whole pipe is dropped. } public PipeReceiverStatusHandler statusHandler() { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java index 557d60d48d5ec..7b8afe6b860d1 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java @@ -35,7 +35,14 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.Function; public class PipeTaskAgentTest { @@ -87,6 +94,62 @@ public void testZeroFloatingMemoryUsageCounterIsCleanedAfterDrop() throws Except Assert.assertTrue(getMapField(agent, "pipeName2CreationTimeSetMap").isEmpty()); } + @Test + public void testFloatingMemoryIncrementIsNotLostDuringDropCleanup() throws Exception { + final DummyPipeTaskAgent agent = new DummyPipeTaskAgent(); + final PipeMeta pipeMeta = generatePipeMeta("concurrent-pipe", 1L); + agent.createPipeForTest(pipeMeta); + agent.addFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L); + agent.decreaseFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L); + + final Field counterMapField = + PipeTaskAgent.class.getDeclaredField( + "pipeNameWithCreationTime2FloatingMemoryUsageInByteMap"); + counterMapField.setAccessible(true); + @SuppressWarnings("unchecked") + final Map originalCounterMap = + (Map) counterMapField.get(agent); + final BlockingFloatingMemoryMap blockingCounterMap = + new BlockingFloatingMemoryMap(originalCounterMap, "concurrent-pipe_1"); + counterMapField.set(agent, blockingCounterMap); + + final AtomicReference backgroundFailure = new AtomicReference<>(); + final Thread increaseThread = + new Thread( + () -> + runAndRecordFailure( + () -> agent.addFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L), + backgroundFailure)); + final Thread dropThread = + new Thread( + () -> + runAndRecordFailure( + () -> agent.dropPipeForTest("concurrent-pipe", 1L), backgroundFailure)); + + try { + blockingCounterMap.pauseNextUpdate(); + increaseThread.start(); + Assert.assertTrue(blockingCounterMap.updatePaused.await(10, TimeUnit.SECONDS)); + dropThread.start(); + Assert.assertTrue(waitForDrop(agent, "concurrent-pipe", 1L)); + } finally { + blockingCounterMap.resumeUpdate.countDown(); + increaseThread.join(TimeUnit.SECONDS.toMillis(10)); + dropThread.join(TimeUnit.SECONDS.toMillis(10)); + counterMapField.set(agent, originalCounterMap); + } + + Assert.assertFalse(increaseThread.isAlive()); + Assert.assertFalse(dropThread.isAlive()); + Assert.assertNull(backgroundFailure.get()); + Assert.assertEquals(100L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("concurrent-pipe")); + + agent.decreaseFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L); + Assert.assertEquals(0L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(0L, agent.getFloatingMemoryUsageInByte("concurrent-pipe")); + } + @SuppressWarnings("unchecked") private static Map getMapField(final PipeTaskAgent agent, final String fieldName) throws NoSuchFieldException, IllegalAccessException { @@ -102,6 +165,28 @@ private static PipeMeta generatePipeMeta(final String pipeName, final long creat new PipeRuntimeMeta(new ConcurrentHashMap<>())); } + private static void runAndRecordFailure( + final Runnable operation, final AtomicReference failure) { + try { + operation.run(); + } catch (final Throwable e) { + failure.compareAndSet(null, e); + } + } + + private static boolean waitForDrop( + final DummyPipeTaskAgent agent, final String pipeName, final long creationTime) + throws InterruptedException { + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (agent.getPipeMetaForTest(pipeName, creationTime) == null) { + return true; + } + Thread.sleep(1); + } + return false; + } + private static class DummyPipeTaskAgent extends PipeTaskAgent { private boolean createPipeForTest(final PipeMeta pipeMeta) throws IllegalPathException { @@ -112,6 +197,10 @@ private boolean dropPipeForTest(final String pipeName, final long creationTime) return dropPipe(pipeName, creationTime); } + private PipeMeta getPipeMetaForTest(final String pipeName, final long creationTime) { + return pipeMetaKeeper.getPipeMeta(pipeName, creationTime); + } + @Override protected boolean isShutdown() { return false; @@ -152,4 +241,61 @@ public void runPipeTasks( pipeTasks.forEach(runSingle); } } + + private static class BlockingFloatingMemoryMap extends ConcurrentHashMap { + + private static final long serialVersionUID = 1L; + + private final String pipeID; + private final AtomicBoolean shouldPause = new AtomicBoolean(); + private final CountDownLatch updatePaused = new CountDownLatch(1); + private final CountDownLatch resumeUpdate = new CountDownLatch(1); + + private BlockingFloatingMemoryMap( + final Map originalMap, final String pipeID) { + putAll(originalMap); + this.pipeID = pipeID; + } + + private void pauseNextUpdate() { + shouldPause.set(true); + } + + @Override + public AtomicLong compute( + final String key, + final BiFunction + remappingFunction) { + return super.compute( + key, + (currentKey, currentValue) -> { + final AtomicLong updatedValue = remappingFunction.apply(currentKey, currentValue); + pauseIfNecessary(currentKey); + return updatedValue; + }); + } + + @Override + public AtomicLong computeIfAbsent( + final String key, final Function mappingFunction) { + final AtomicLong value = super.computeIfAbsent(key, mappingFunction); + pauseIfNecessary(key); + return value; + } + + private void pauseIfNecessary(final String key) { + if (!pipeID.equals(key) || !shouldPause.compareAndSet(true, false)) { + return; + } + updatePaused.countDown(); + try { + if (!resumeUpdate.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("Timed out waiting to resume floating memory update"); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java index 1a639ea1116f2..f7054dba44add 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java @@ -127,6 +127,17 @@ public void testLateMultipleCommitsAfterClearDoNotRestoreState() throws Exceptio verify(onCommitted, never()).run(); } + @Test + public void testLateEventEnrichmentAfterClearDoesNotRestoreState() throws Exception { + final EnrichedEvent event = createEvent(mock(Runnable.class)); + manager.clear(PIPE_NAME, CREATION_TIME); + + manager.enrichWithCommitterKeyAndCommitId(event, CREATION_TIME, REGION_ID); + + assertTrue(getManagerMap("eventCommitterMap").isEmpty()); + assertTrue(getManagerMap("eventCommitterRestartTimesMap").isEmpty()); + } + @Test public void testRestartRejectsStaleCommitAndAcceptsCurrentCommit() throws Exception { final CommitterKey oldKey = manager.getCommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID); diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java new file mode 100644 index 0000000000000..0424a3181ad43 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.protocol; + +import org.apache.iotdb.commons.pipe.sink.limiter.PipeEndPointRateLimiter; + +import org.apache.tsfile.utils.Pair; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Map; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; + +public class IoTDBSinkRateLimiterTest { + + @Test + public void testRegionTaskDiscardKeepsPipeWideRateLimiter() throws Exception { + final String pipeName = "rate_limiter_lifecycle_test"; + final long creationTime = System.nanoTime(); + final Pair pipeKey = new Pair<>(pipeName, creationTime); + final Map, PipeEndPointRateLimiter> limiters = getRateLimiterMap(); + final PipeEndPointRateLimiter limiter = mock(PipeEndPointRateLimiter.class); + final IoTDBSink sink = mock(IoTDBSink.class, CALLS_REAL_METHODS); + limiters.put(pipeKey, limiter); + + try { + sink.discardEventsOfPipe(pipeName, creationTime, 1); + + assertSame(limiter, limiters.get(pipeKey)); + + IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); + + assertFalse(limiters.containsKey(pipeKey)); + } finally { + IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); + } + } + + @SuppressWarnings("unchecked") + private static Map, PipeEndPointRateLimiter> getRateLimiterMap() + throws ReflectiveOperationException { + final Field field = IoTDBSink.class.getDeclaredField("PIPE_END_POINT_RATE_LIMITER_MAP"); + field.setAccessible(true); + return (Map, PipeEndPointRateLimiter>) field.get(null); + } +} From ca5d222e2265078b9fcd0ffc48b23b4ea308b8c9 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:07:47 +0800 Subject: [PATCH 10/14] Pipe: balance late memory accounting and avoid passive listener recreation --- .../schemaregion/IoTDBSchemaRegionSource.java | 10 +++++ .../PipeSchemaRegionListenerManagerTest.java | 44 +++++++++++++++++++ .../pipe/agent/task/PipeTaskAgent.java | 3 -- .../pipe/source/IoTDBNonDataRegionSource.java | 12 ++++- .../pipe/agent/task/PipeTaskAgentTest.java | 41 +++++++++++++++++ 5 files changed, 105 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java index 8e33be13bdbe1..05b3d29f39828 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java @@ -329,6 +329,16 @@ protected AbstractPipeListeningQueue getListeningQueue() { return PipeDataNodeAgent.runtime().schemaListener(schemaRegionId); } + @Override + public long getUnTransferredEventCount() { + if (hasBeenClosed.get() || Objects.isNull(pipeTaskMeta)) { + return 0L; + } + // Metric callbacks can outlive region cleanup and must not recreate a listener. + return getUnTransferredEventCount( + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + } + @Override protected boolean isTypeListened(final PipeWritePlanEvent event) { final PlanNode planNode = ((PipeSchemaRegionWritePlanEvent) event).getPlanNode(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java index fd7106c74b5a2..7506c7ead9362 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java @@ -20,6 +20,10 @@ package org.apache.iotdb.db.pipe.agent.runtime; import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; +import org.apache.iotdb.db.pipe.source.schemaregion.IoTDBSchemaRegionSource; import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; import org.junit.Assert; @@ -29,6 +33,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.Field; public class PipeSchemaRegionListenerManagerTest { @@ -77,6 +82,38 @@ public void testAutoCleanupAfterLastReferenceReleased() { Assert.assertTrue(manager.regionIds().isEmpty()); } + @Test + public void testMetricsDoNotRecreateListenerAfterRegionCleanup() throws Exception { + final SchemaRegionId schemaRegionId = new SchemaRegionId(Integer.MAX_VALUE - 10); + final TestSchemaRegionSource source = new TestSchemaRegionSource(); + final Field schemaRegionIdField = + IoTDBSchemaRegionSource.class.getDeclaredField("schemaRegionId"); + schemaRegionIdField.setAccessible(true); + schemaRegionIdField.set(source, schemaRegionId); + + try { + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime().schemaListener(schemaRegionId); + listener.open(); + listener.close(); + PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId); + + Assert.assertEquals(0L, source.getUnTransferredEventCount()); + Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + + source.close(); + Assert.assertEquals(0L, source.getUnTransferredEventCount()); + Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + } finally { + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId); + if (listener != null) { + listener.close(); + } + PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId); + } + } + @Test public void testCreateListenerForSnapshotLoadOnlyWhenSnapshotExists() throws IOException { final PipeSchemaRegionListenerManager manager = new PipeSchemaRegionListenerManager(); @@ -101,4 +138,11 @@ public void testCreateListenerForSnapshotLoadOnlyWhenSnapshotExists() throws IOE restoredListener.close(); manager.clearSchemaRegionState(schemaRegionId); } + + private static class TestSchemaRegionSource extends IoTDBSchemaRegionSource { + + private TestSchemaRegionSource() { + pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 0); + } + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index cb13440ad84f1..206a98a548921 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -1377,9 +1377,6 @@ public void addFloatingMemoryUsageInByte( pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.compute( generatePipeNameWithCreationTime(pipeName, creationTime), (key, counter) -> { - if (pipeMetaKeeper.getPipeMeta(pipeName, creationTime) == null) { - return counter; - } final AtomicLong updatedCounter = Objects.isNull(counter) ? new AtomicLong(0) : counter; updatedCounter.addAndGet(sizeInByte); pipeName2CreationTimeSetMap.compute( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java index d4d55ba7dac7a..c827fe2b98543 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java @@ -302,12 +302,20 @@ public long getUnTransferredEventCount() { if (Objects.isNull(pipeTaskMeta)) { return 0L; } + return getUnTransferredEventCount(getListeningQueue()); + } + + /** Calculate metrics from a captured queue without looking it up again during cleanup. */ + protected long getUnTransferredEventCount(final AbstractPipeListeningQueue listeningQueue) { + if (Objects.isNull(pipeTaskMeta) || Objects.isNull(listeningQueue)) { + return 0L; + } final ProgressIndex progressIndex = pipeTaskMeta.getProgressIndex(); warnIfUnexpectedHybridProgressIndex(progressIndex); final MetaProgressIndex metaProgressIndex = extractMetaProgressIndex(progressIndex); return Objects.nonNull(metaProgressIndex) - ? getListeningQueue().getTailIndex() - metaProgressIndex.getIndex() - 1 - : getListeningQueue().getSize() + historicalEventsCount; + ? listeningQueue.getTailIndex() - metaProgressIndex.getIndex() - 1 + : listeningQueue.getSize() + historicalEventsCount; } private static MetaProgressIndex extractMetaProgressIndex(final ProgressIndex progressIndex) { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java index 7b8afe6b860d1..c7c1773dadf4f 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java @@ -94,6 +94,47 @@ public void testZeroFloatingMemoryUsageCounterIsCleanedAfterDrop() throws Except Assert.assertTrue(getMapField(agent, "pipeName2CreationTimeSetMap").isEmpty()); } + @Test + public void testLateFloatingMemoryAcquisitionDoesNotConsumeOutstandingUsage() throws Exception { + final DummyPipeTaskAgent agent = new DummyPipeTaskAgent(); + agent.createPipeForTest(generatePipeMeta("pipe", 1L)); + agent.addFloatingMemoryUsageInByte("pipe", 1L, 100L); + agent.dropPipeForTest("pipe", 1L); + + // An assigner can retain a matched source across drop while other events still own memory. + agent.addFloatingMemoryUsageInByte("pipe", 1L, 20L); + Assert.assertEquals(120L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(120L, agent.getFloatingMemoryUsageInByte("pipe")); + agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 20L); + Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe", 1L)); + agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 100L); + + Assert.assertEquals(0L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertTrue( + getMapField(agent, "pipeNameWithCreationTime2FloatingMemoryUsageInByteMap").isEmpty()); + Assert.assertTrue(getMapField(agent, "pipeName2CreationTimeSetMap").isEmpty()); + } + + @Test + public void testLateFloatingMemoryAcquisitionAfterCounterCleanup() throws Exception { + final DummyPipeTaskAgent agent = new DummyPipeTaskAgent(); + agent.createPipeForTest(generatePipeMeta("pipe", 1L)); + agent.addFloatingMemoryUsageInByte("pipe", 1L, 100L); + agent.dropPipeForTest("pipe", 1L); + agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 100L); + + // Late acquisitions must also be tracked after the previous events released the last byte. + agent.addFloatingMemoryUsageInByte("pipe", 1L, 20L); + Assert.assertEquals(20L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(20L, agent.getFloatingMemoryUsageInByte("pipe")); + Assert.assertEquals(20L, agent.getFloatingMemoryUsageInByte("pipe", 1L)); + agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 20L); + + Assert.assertTrue( + getMapField(agent, "pipeNameWithCreationTime2FloatingMemoryUsageInByteMap").isEmpty()); + Assert.assertTrue(getMapField(agent, "pipeName2CreationTimeSetMap").isEmpty()); + } + @Test public void testFloatingMemoryIncrementIsNotLostDuringDropCleanup() throws Exception { final DummyPipeTaskAgent agent = new DummyPipeTaskAgent(); From 06f675ca98f7daa36d666dae74abf4e31cf6f6f1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:49:06 +0800 Subject: [PATCH 11/14] Pipe: clear shared runtime state on pipe drop --- .../agent/task/PipeDataNodeTaskAgent.java | 6 -- .../pipe/agent/task/PipeTaskAgent.java | 8 +++ .../pipe/agent/task/PipeTaskAgentTest.java | 71 +++++++++++++++++++ 3 files changed, 79 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 9cd802bda6cae..1b54f1a0742cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -44,13 +44,11 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTemporaryMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTemporaryMetaInAgent; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeType; -import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; -import org.apache.iotdb.commons.pipe.sink.protocol.IoTDBSink; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -421,8 +419,6 @@ protected boolean dropPipe(final String pipeName, final long creationTime) { final String taskId = pipeName + "_" + creationTime; PipeTsFileToTabletsMetrics.getInstance().deregister(taskId); PipeDataNodeSinglePipeMetrics.getInstance().deregister(taskId); - PipeEventCommitManager.getInstance().clear(pipeName, creationTime); - IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); PipeDataNodeResourceManager.tsfile().cleanPipeTsFileDir(pipeTsFileResourcePipeName); return true; @@ -465,8 +461,6 @@ protected boolean dropPipe(final String pipeName) { final String taskId = pipeName + "_" + creationTime; PipeTsFileToTabletsMetrics.getInstance().deregister(taskId); PipeDataNodeSinglePipeMetrics.getInstance().deregister(taskId); - PipeEventCommitManager.getInstance().clear(pipeName, creationTime); - IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); PipeDataNodeResourceManager.tsfile().cleanPipeTsFileDir(pipeTsFileResourcePipeName); // When the pipe contains no pipe tasks, there is no corresponding prefetching queue for the // subscribed pipe, so the subscription needs to be manually marked as completed. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index 206a98a548921..71d463e359205 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -39,6 +39,7 @@ import org.apache.iotdb.commons.pipe.resource.PipeResourceFailureType; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.commons.pipe.sink.limiter.PipeEndPointRateLimiter; +import org.apache.iotdb.commons.pipe.sink.protocol.IoTDBSink; import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq; import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaRespExceptionMessage; @@ -661,6 +662,7 @@ protected boolean dropPipe(final String pipeName, final long creationTime) { // Remove pipe meta from pipe meta keeper pipeMetaKeeper.removePipeMeta(existedPipeMeta.getStaticMeta()); cleanupFloatingMemoryUsageCounterIfNecessary(pipeName, creationTime); + clearPipeRuntimeState(pipeName, creationTime); return true; } @@ -705,6 +707,7 @@ protected boolean dropPipe(final String pipeName) { pipeMetaKeeper.removePipeMeta(existedPipeMeta.getStaticMeta()); cleanupFloatingMemoryUsageCounterIfNecessary( pipeName, existedPipeMeta.getStaticMeta().getCreationTime()); + clearPipeRuntimeState(pipeName, existedPipeMeta.getStaticMeta().getCreationTime()); return true; } @@ -1319,6 +1322,11 @@ private void cleanupFloatingMemoryUsageCounterIfNecessary( cleanupFloatingMemoryUsageCounterIfUnused(pipeName, creationTime, counter)); } + private void clearPipeRuntimeState(final String pipeName, final long creationTime) { + PipeEventCommitManager.getInstance().clear(pipeName, creationTime); + IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); + } + public long getPipeCreationTime(final String pipeName) { final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); return pipeMeta == null ? 0 : pipeMeta.getStaticMeta().getCreationTime(); diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java index c7c1773dadf4f..1f340bd20dbd0 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java @@ -24,9 +24,14 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; +import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager; +import org.apache.iotdb.commons.pipe.sink.limiter.PipeEndPointRateLimiter; +import org.apache.iotdb.commons.pipe.sink.protocol.IoTDBSink; import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq; import org.apache.thrift.TException; +import org.apache.tsfile.utils.Pair; import org.junit.Assert; import org.junit.Test; @@ -191,6 +196,68 @@ public void testFloatingMemoryIncrementIsNotLostDuringDropCleanup() throws Excep Assert.assertEquals(0L, agent.getFloatingMemoryUsageInByte("concurrent-pipe")); } + @Test + public void testPipeDropClearsSharedRuntimeState() throws Exception { + assertPipeDropClearsSharedRuntimeState(false); + assertPipeDropClearsSharedRuntimeState(true); + } + + private static void assertPipeDropClearsSharedRuntimeState(final boolean dropByName) + throws Exception { + final String pipeName = "runtime_state_cleanup_" + System.nanoTime(); + final long creationTime = System.nanoTime(); + final int regionId = 17; + final DummyPipeTaskAgent agent = new DummyPipeTaskAgent(); + final PipeEventCommitManager commitManager = PipeEventCommitManager.getInstance(); + final Map, PipeEndPointRateLimiter> rateLimiters = getRateLimiterMap(); + final Pair pipeKey = new Pair<>(pipeName, creationTime); + + try { + commitManager.register(pipeName, creationTime, regionId, "test-sink"); + final CommitterKey committerKey = + commitManager.getCommitterKey(pipeName, creationTime, regionId); + final CommitterKey restartTimesKey = new CommitterKey(pipeName, creationTime, regionId, -1); + rateLimiters.put(pipeKey, new PipeEndPointRateLimiter(pipeName, creationTime, 1.0)); + agent.createPipeForTest(generatePipeMeta(pipeName, creationTime)); + + if (dropByName) { + Assert.assertTrue(agent.dropPipeByNameForTest(pipeName)); + } else { + Assert.assertTrue(agent.dropPipeForTest(pipeName, creationTime)); + } + + Assert.assertFalse( + getCommitManagerMap(commitManager, "eventCommitterMap").containsKey(committerKey)); + Assert.assertFalse( + getCommitManagerMap(commitManager, "eventCommitterRestartTimesMap") + .containsKey(restartTimesKey)); + Assert.assertFalse(rateLimiters.containsKey(pipeKey)); + } finally { + commitManager.clear(pipeName, creationTime); + IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); + final PipeMeta pipeMeta = agent.getPipeMetaForTest(pipeName, creationTime); + if (pipeMeta != null) { + agent.dropPipeForTest(pipeName, creationTime); + } + } + } + + @SuppressWarnings("unchecked") + private static Map, PipeEndPointRateLimiter> getRateLimiterMap() + throws ReflectiveOperationException { + final Field field = IoTDBSink.class.getDeclaredField("PIPE_END_POINT_RATE_LIMITER_MAP"); + field.setAccessible(true); + return (Map, PipeEndPointRateLimiter>) field.get(null); + } + + private static Map getCommitManagerMap( + final PipeEventCommitManager commitManager, final String fieldName) + throws ReflectiveOperationException { + final Field field = PipeEventCommitManager.class.getDeclaredField(fieldName); + field.setAccessible(true); + return (Map) field.get(commitManager); + } + @SuppressWarnings("unchecked") private static Map getMapField(final PipeTaskAgent agent, final String fieldName) throws NoSuchFieldException, IllegalAccessException { @@ -238,6 +305,10 @@ private boolean dropPipeForTest(final String pipeName, final long creationTime) return dropPipe(pipeName, creationTime); } + private boolean dropPipeByNameForTest(final String pipeName) { + return dropPipe(pipeName); + } + private PipeMeta getPipeMetaForTest(final String pipeName, final long creationTime) { return pipeMetaKeeper.getPipeMeta(pipeName, creationTime); } From bd13b0cd21b431224daa4d97b9b3eaa9032f2ece Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:40:42 +0800 Subject: [PATCH 12/14] Fix late Pipe cleanup state recreation --- .../PipeDataNodeSinglePipeMetrics.java | 2 +- .../PipeDataNodeSinglePipeMetricsTest.java | 35 ++++++++ .../pipe/agent/task/PipeTaskAgent.java | 9 +- .../task/progress/PipeEventCommitManager.java | 22 +++-- .../sink/limiter/PipeEndPointRateLimiter.java | 9 +- .../commons/pipe/sink/protocol/IoTDBSink.java | 14 ++-- .../pipe/agent/task/PipeTaskAgentTest.java | 44 ++++++---- .../progress/PipeEventCommitManagerTest.java | 83 +++++++++++++++++++ .../protocol/IoTDBSinkRateLimiterTest.java | 32 +++++++ 9 files changed, 217 insertions(+), 33 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java index 02d18ba488cd1..92d8499fa0511 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java @@ -234,7 +234,7 @@ private static String generatePipeID(final String pipeName, final long creationT } private boolean isPipeAlive(final String pipeName, final long creationTime) { - return PipeDataNodeAgent.task().isPipeExisted(pipeName, creationTime); + return PipeDataNodeAgent.task().isPipeAlive(pipeName, creationTime); } private PipeDataNodeRemainingEventAndTimeOperator getOrCreateOperatorIfPipeAlive( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java index 2c751e7b6ecbd..702d2e1dc209d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMetaKeeper; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; @@ -279,6 +280,40 @@ public void testMetricCreationSerializesWithDeregister() throws Exception { } } + @Test + public void testLateMetricUpdateAfterDropDeregisterDoesNotRecreateOperator() throws Exception { + final PipeDataNodeSinglePipeMetrics metrics = PipeDataNodeSinglePipeMetrics.getInstance(); + final Field metricServiceField = + PipeDataNodeSinglePipeMetrics.class.getDeclaredField("metricService"); + metricServiceField.setAccessible(true); + reset(metrics, metricServiceField); + + final String pipeName = "metric_late_update_after_drop_" + System.nanoTime(); + final long creationTime = System.nanoTime(); + final String pipeID = pipeName + "_" + creationTime; + final PipeMeta pipeMeta = + createPipeMeta(pipeName, creationTime, SystemConstant.SQL_DIALECT_TREE_VALUE); + final PipeMetaKeeper pipeMetaKeeper = getPipeMetaKeeper(); + + try { + pipeMetaKeeper.addPipeMeta(pipeMeta); + metrics.increaseTsFileEventCount(pipeName, creationTime); + assertTrue(metrics.remainingEventAndTimeOperatorMap.containsKey(pipeID)); + + // dropPipe marks the metadata dropped before task deregistration and removes it afterward. + pipeMeta.getRuntimeMeta().getStatus().set(PipeStatus.DROPPED); + metrics.deregister(pipeID); + + metrics.increaseTsFileEventCount(pipeName, creationTime); + + assertFalse(metrics.remainingEventAndTimeOperatorMap.containsKey(pipeID)); + } finally { + pipeMetaKeeper.removePipeMeta(pipeMeta.getStaticMeta()); + metrics.deregister(pipeID); + reset(metrics, metricServiceField); + } + } + private static PipeMeta createPipeMeta( final String pipeName, final long creationTime, final String sqlDialect) { final Map sourceAttributes = new HashMap<>(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index 71d463e359205..355708481ac75 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -1301,10 +1301,12 @@ private static String generatePipeNameWithCreationTime( private AtomicLong cleanupFloatingMemoryUsageCounterIfUnused( final String pipeName, final long creationTime, final AtomicLong counter) { - if (counter.get() != 0 || isPipeExisted(pipeName, creationTime)) { + if (counter.get() != 0) { return counter; } + // A missing counter also reports zero; a later acquisition recreates it without scanning all + // pipe metadata on the event-release path. pipeName2CreationTimeSetMap.computeIfPresent( pipeName, (key, creationTimes) -> { @@ -1336,6 +1338,11 @@ public boolean isPipeExisted(final String pipeName, final long creationTime) { return pipeMetaKeeper.getPipeMeta(pipeName, creationTime) != null; } + public boolean isPipeAlive(final String pipeName, final long creationTime) { + final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); + return pipeMeta != null && pipeMeta.getRuntimeMeta().getStatus().get() != PipeStatus.DROPPED; + } + public String getPipeNameWithCreationTime(final String pipeName, final long creationTime) { final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); return pipeMeta == null diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java index 47a519184fde4..a04f7519551d3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java @@ -194,13 +194,25 @@ public CommitterKey getCommitterKey( * older committer after the same Pipe/DataRegion task has been restarted. */ public boolean isCurrentCommitterKey(final CommitterKey committerKey) { - return committerKey != null - && eventCommitterMap.containsKey(committerKey) - && committerKey.equals( - generateCommitterKey( + if (committerKey == null) { + return false; + } + + final PipeEventCommitter committer = eventCommitterMap.get(committerKey); + if (committer == null) { + return false; + } + + final int restartTimes = + eventCommitterRestartTimesMap.getOrDefault( + generateCommitterRestartTimesKey(committerKey), 0); + return committerKey.equals( + taskAgent.getCommitterKey( committerKey.getPipeName(), committerKey.getCreationTime(), - committerKey.getRegionId())); + committerKey.getRegionId(), + restartTimes)) + && eventCommitterMap.get(committerKey) == committer; } private CommitterKey generateCommitterKey( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java index 2701e5bb23702..fbbe95a703450 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java @@ -55,6 +55,11 @@ public static void setTaskAgent(final PipeTaskAgent taskAgent) { PipeEndPointRateLimiter.taskAgent = taskAgent; } + public static boolean isPipeAlive(final String pipeName, final long creationTime) { + final PipeTaskAgent finalTaskAgent = taskAgent; + return Objects.isNull(finalTaskAgent) || finalTaskAgent.isPipeAlive(pipeName, creationTime); + } + public void acquire(final TEndPoint endPoint, long bytes) { if (endPoint == null) { return; @@ -82,9 +87,7 @@ private boolean tryAcquireWithPipeCheck(final RateLimiter rateLimiter, final int bytes, PipeConfig.getInstance().getRateLimiterHotReloadCheckIntervalMs(), TimeUnit.MILLISECONDS)) { - final PipeTaskAgent finalTaskAgent = taskAgent; - if (Objects.nonNull(finalTaskAgent) - && !finalTaskAgent.isPipeExisted(pipeName, creationTime)) { + if (!isPipeAlive(pipeName, creationTime)) { return false; } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java index f865fb1734a48..6bbf9cc5b3ba0 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java @@ -677,13 +677,17 @@ public void rateLimitIfNeeded( final TEndPoint endPoint, final long bytesLength) { if (pipeName != null && endPointRateLimitBytesPerSecond > 0) { - PIPE_END_POINT_RATE_LIMITER_MAP - .computeIfAbsent( + final PipeEndPointRateLimiter limiter = + PIPE_END_POINT_RATE_LIMITER_MAP.computeIfAbsent( new Pair<>(pipeName, creationTime), endpoint -> - new PipeEndPointRateLimiter( - pipeName, creationTime, endPointRateLimitBytesPerSecond)) - .acquire(endPoint, bytesLength); + PipeEndPointRateLimiter.isPipeAlive(pipeName, creationTime) + ? new PipeEndPointRateLimiter( + pipeName, creationTime, endPointRateLimitBytesPerSecond) + : null); + if (limiter != null) { + limiter.acquire(endPoint, bytesLength); + } } GLOBAL_RATE_LIMITER.acquire(bytesLength); diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java index 1f340bd20dbd0..1f54a0d6d15ba 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java @@ -92,6 +92,11 @@ public void testZeroFloatingMemoryUsageCounterIsCleanedAfterDrop() throws Except agent.createPipeForTest(generatePipeMeta("pipe", 1L)); agent.addFloatingMemoryUsageInByte("pipe", 1L, 100L); agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 100L); + + Assert.assertTrue( + getMapField(agent, "pipeNameWithCreationTime2FloatingMemoryUsageInByteMap").isEmpty()); + Assert.assertTrue(getMapField(agent, "pipeName2CreationTimeSetMap").isEmpty()); + agent.dropPipeForTest("pipe", 1L); Assert.assertTrue( @@ -173,27 +178,30 @@ public void testFloatingMemoryIncrementIsNotLostDuringDropCleanup() throws Excep () -> agent.dropPipeForTest("concurrent-pipe", 1L), backgroundFailure)); try { - blockingCounterMap.pauseNextUpdate(); - increaseThread.start(); - Assert.assertTrue(blockingCounterMap.updatePaused.await(10, TimeUnit.SECONDS)); - dropThread.start(); - Assert.assertTrue(waitForDrop(agent, "concurrent-pipe", 1L)); + try { + blockingCounterMap.pauseNextUpdate(); + increaseThread.start(); + Assert.assertTrue(blockingCounterMap.updatePaused.await(10, TimeUnit.SECONDS)); + dropThread.start(); + Assert.assertTrue(waitForDrop(agent, "concurrent-pipe", 1L)); + } finally { + blockingCounterMap.resumeUpdate.countDown(); + increaseThread.join(TimeUnit.SECONDS.toMillis(10)); + dropThread.join(TimeUnit.SECONDS.toMillis(10)); + } + + Assert.assertFalse(increaseThread.isAlive()); + Assert.assertFalse(dropThread.isAlive()); + Assert.assertNull(backgroundFailure.get()); + Assert.assertEquals(100L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("concurrent-pipe")); + + agent.decreaseFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L); + Assert.assertEquals(0L, agent.getAllFloatingMemoryUsageInByte()); + Assert.assertEquals(0L, agent.getFloatingMemoryUsageInByte("concurrent-pipe")); } finally { - blockingCounterMap.resumeUpdate.countDown(); - increaseThread.join(TimeUnit.SECONDS.toMillis(10)); - dropThread.join(TimeUnit.SECONDS.toMillis(10)); counterMapField.set(agent, originalCounterMap); } - - Assert.assertFalse(increaseThread.isAlive()); - Assert.assertFalse(dropThread.isAlive()); - Assert.assertNull(backgroundFailure.get()); - Assert.assertEquals(100L, agent.getAllFloatingMemoryUsageInByte()); - Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("concurrent-pipe")); - - agent.decreaseFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L); - Assert.assertEquals(0L, agent.getAllFloatingMemoryUsageInByte()); - Assert.assertEquals(0L, agent.getFloatingMemoryUsageInByte("concurrent-pipe")); } @Test diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java index f7054dba44add..2d81059d623e3 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java @@ -36,8 +36,14 @@ import java.util.Arrays; import java.util.Collections; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; @@ -138,6 +144,48 @@ public void testLateEventEnrichmentAfterClearDoesNotRestoreState() throws Except assertTrue(getManagerMap("eventCommitterRestartTimesMap").isEmpty()); } + @Test + public void testLateCommitterCheckAfterClearDoesNotRestoreRestartState() throws Exception { + final CommitterKey key = manager.getCommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID); + final Field eventCommitterMapField = + PipeEventCommitManager.class.getDeclaredField("eventCommitterMap"); + eventCommitterMapField.setAccessible(true); + @SuppressWarnings("unchecked") + final Map originalEventCommitterMap = + (Map) eventCommitterMapField.get(manager); + final BlockingCommitterMap blockingMap = new BlockingCommitterMap(originalEventCommitterMap); + eventCommitterMapField.set(manager, blockingMap); + + final AtomicReference result = new AtomicReference<>(); + final AtomicReference backgroundFailure = new AtomicReference<>(); + final Thread checkThread = + new Thread( + () -> { + try { + result.set(manager.isCurrentCommitterKey(key)); + } catch (final Throwable e) { + backgroundFailure.compareAndSet(null, e); + } + }); + + try { + blockingMap.pauseNextLookup(); + checkThread.start(); + assertTrue(blockingMap.lookupPaused.await(10, TimeUnit.SECONDS)); + + manager.clear(PIPE_NAME, CREATION_TIME); + } finally { + blockingMap.resumeLookup.countDown(); + checkThread.join(TimeUnit.SECONDS.toMillis(10)); + eventCommitterMapField.set(manager, originalEventCommitterMap); + } + + assertFalse(checkThread.isAlive()); + assertTrue(backgroundFailure.get() == null); + assertFalse(result.get()); + assertTrue(getManagerMap("eventCommitterRestartTimesMap").isEmpty()); + } + @Test public void testRestartRejectsStaleCommitAndAcceptsCurrentCommit() throws Exception { final CommitterKey oldKey = manager.getCommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID); @@ -191,4 +239,39 @@ private static EnrichedEvent createEvent(final Runnable onCommitted) { field.setAccessible(true); return (Map) field.get(manager); } + + private static class BlockingCommitterMap + extends ConcurrentHashMap { + + private static final long serialVersionUID = 1L; + + private final AtomicBoolean shouldPause = new AtomicBoolean(); + private final CountDownLatch lookupPaused = new CountDownLatch(1); + private final CountDownLatch resumeLookup = new CountDownLatch(1); + + private BlockingCommitterMap(final Map originalMap) { + putAll(originalMap); + } + + private void pauseNextLookup() { + shouldPause.set(true); + } + + @Override + public PipeEventCommitter get(final Object key) { + final PipeEventCommitter committer = super.get(key); + if (committer != null && shouldPause.compareAndSet(true, false)) { + lookupPaused.countDown(); + try { + if (!resumeLookup.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("Timed out waiting to resume committer lookup"); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + return committer; + } + } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java index 0424a3181ad43..a98a3a59b3ce4 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java @@ -19,6 +19,8 @@ package org.apache.iotdb.commons.pipe.sink.protocol; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent; import org.apache.iotdb.commons.pipe.sink.limiter.PipeEndPointRateLimiter; import org.apache.tsfile.utils.Pair; @@ -31,6 +33,7 @@ import static org.junit.Assert.assertSame; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class IoTDBSinkRateLimiterTest { @@ -57,6 +60,35 @@ public void testRegionTaskDiscardKeepsPipeWideRateLimiter() throws Exception { } } + @Test + public void testLateTransferDoesNotRecreateRateLimiterAfterDrop() throws Exception { + final String pipeName = "late_rate_limiter_lifecycle_test"; + final long creationTime = System.nanoTime(); + final Pair pipeKey = new Pair<>(pipeName, creationTime); + final Map, PipeEndPointRateLimiter> limiters = getRateLimiterMap(); + final IoTDBSink sink = mock(IoTDBSink.class, CALLS_REAL_METHODS); + final PipeTaskAgent taskAgent = mock(PipeTaskAgent.class); + when(taskAgent.isPipeAlive(pipeName, creationTime)).thenReturn(false); + + final Field taskAgentField = PipeEndPointRateLimiter.class.getDeclaredField("taskAgent"); + taskAgentField.setAccessible(true); + final PipeTaskAgent originalTaskAgent = (PipeTaskAgent) taskAgentField.get(null); + final Field rateLimitField = + IoTDBSink.class.getDeclaredField("endPointRateLimitBytesPerSecond"); + rateLimitField.setAccessible(true); + rateLimitField.setDouble(sink, 1.0); + + PipeEndPointRateLimiter.setTaskAgent(taskAgent); + try { + sink.rateLimitIfNeeded(pipeName, creationTime, new TEndPoint("localhost", 1), 0); + + assertFalse(limiters.containsKey(pipeKey)); + } finally { + IoTDBSink.clearPipeRateLimiter(pipeName, creationTime); + PipeEndPointRateLimiter.setTaskAgent(originalTaskAgent); + } + } + @SuppressWarnings("unchecked") private static Map, PipeEndPointRateLimiter> getRateLimiterMap() throws ReflectiveOperationException { From 2654d8b5776d842584e41fbdc29f9a3511a60972 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:23:41 +0800 Subject: [PATCH 13/14] Pipe: reset stale schema queues when loading snapshots --- .../PipeSchemaRegionListenerManager.java | 12 ++- .../PipeSchemaRegionListenerManagerTest.java | 77 +++++++++++++++++++ 2 files changed, 86 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java index 6d12e95979c61..61a904c339cc6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java @@ -54,10 +54,16 @@ public synchronized SchemaRegionListeningQueue listenerIfPresent( public synchronized SchemaRegionListeningQueue listenerForSnapshotLoad( final SchemaRegionId schemaRegionId, final File snapshotDir) { final PipeSchemaRegionListener listener = id2ListenerMap.get(schemaRegionId); - if (listener != null) { - return listener.listeningQueue; + if (!SchemaRegionListeningQueue.hasSnapshot(snapshotDir)) { + // The snapshot can replace a follower that missed the queue-close log entry. Absence of a + // queue snapshot represents a closed queue, so discard the follower's previous contents. + if (listener != null) { + listener.listeningQueue.close(); + cleanupListenerIfUnused(schemaRegionId, listener); + } + return null; } - return SchemaRegionListeningQueue.hasSnapshot(snapshotDir) ? listener(schemaRegionId) : null; + return listener == null ? listener(schemaRegionId) : listener.listeningQueue; } public synchronized int increaseAndGetReferenceCount(final SchemaRegionId schemaRegionId) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java index 7506c7ead9362..aec6094935f6c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java @@ -22,9 +22,14 @@ import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.consensus.statemachine.schemaregion.SchemaRegionStateMachine; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; +import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionListenerMetrics; import org.apache.iotdb.db.pipe.source.schemaregion.IoTDBSchemaRegionSource; import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.junit.Assert; import org.junit.Rule; @@ -34,6 +39,10 @@ import java.io.File; import java.io.IOException; import java.lang.reflect.Field; +import java.util.Map; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class PipeSchemaRegionListenerManagerTest { @@ -139,6 +148,74 @@ public void testCreateListenerForSnapshotLoadOnlyWhenSnapshotExists() throws IOE manager.clearSchemaRegionState(schemaRegionId); } + @Test + public void testSnapshotWithoutListenerClearsStaleFollowerQueue() throws Exception { + assertSnapshotWithoutListenerClearsStaleQueue(false); + } + + @Test + public void testSnapshotWithoutListenerRetainsReferencedClosedQueue() throws Exception { + assertSnapshotWithoutListenerClearsStaleQueue(true); + } + + private void assertSnapshotWithoutListenerClearsStaleQueue(final boolean hasLocalReference) + throws Exception { + final SchemaRegionId schemaRegionId = + new SchemaRegionId(Integer.MAX_VALUE - (hasLocalReference ? 12 : 11)); + final File snapshotDir = temporaryFolder.newFolder(); + final ISchemaRegion schemaRegion = mock(ISchemaRegion.class); + when(schemaRegion.getSchemaRegionId()).thenReturn(schemaRegionId); + when(schemaRegion.createSnapshot(snapshotDir)).thenReturn(true); + when(schemaRegion.loadSnapshot(snapshotDir)).thenReturn(true); + final SchemaRegionStateMachine stateMachine = new SchemaRegionStateMachine(schemaRegion); + final Field queueMapField = + PipeSchemaRegionListenerMetrics.class.getDeclaredField("listeningQueueMap"); + queueMapField.setAccessible(true); + final Map metricQueues = + (Map) queueMapField.get(PipeSchemaRegionListenerMetrics.getInstance()); + + try { + // The leader has applied queue-close and cleaned its listener before taking this snapshot. + Assert.assertTrue(stateMachine.takeSnapshot(snapshotDir)); + Assert.assertFalse(SchemaRegionListeningQueue.hasSnapshot(snapshotDir)); + Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + + // A lagging follower can still have the queue from before the close log entry. + final SchemaRegionListeningQueue staleQueue = + PipeDataNodeAgent.runtime().schemaListener(schemaRegionId); + if (hasLocalReference) { + PipeDataNodeAgent.runtime().increaseAndGetSchemaListenerReferenceCount(schemaRegionId); + } + staleQueue.open(); + final PlanNode node = mock(PlanNode.class); + when(node.getType()).thenReturn(PlanNodeType.CREATE_TIME_SERIES); + staleQueue.tryListenToNode(node); + Assert.assertEquals(1L, staleQueue.getSize()); + + Assert.assertTrue(stateMachine.loadSnapshot(snapshotDir)); + + Assert.assertFalse(staleQueue.isOpened()); + Assert.assertEquals(0L, staleQueue.getSize()); + if (hasLocalReference) { + Assert.assertSame( + staleQueue, PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + Assert.assertTrue(metricQueues.containsKey(schemaRegionId.getId())); + Assert.assertEquals( + 0, + PipeDataNodeAgent.runtime().decreaseAndGetSchemaListenerReferenceCount(schemaRegionId)); + } + Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + Assert.assertFalse(metricQueues.containsKey(schemaRegionId.getId())); + } finally { + final SchemaRegionListeningQueue listener = + PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId); + if (listener != null) { + listener.close(); + } + PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId); + } + } + private static class TestSchemaRegionSource extends IoTDBSchemaRegionSource { private TestSchemaRegionSource() { From ab0d52e88322245f8ac030cfc4d04d37c1a1c6b1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:28:37 +0800 Subject: [PATCH 14/14] Pipe: add schema snapshot recovery IT --- .../PipeSchemaRegionSnapshotRecoveryIT.java | 284 ++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/single/PipeSchemaRegionSnapshotRecoveryIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/PipeSchemaRegionSnapshotRecoveryIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/PipeSchemaRegionSnapshotRecoveryIT.java new file mode 100644 index 0000000000000..c327b2d806d11 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/PipeSchemaRegionSnapshotRecoveryIT.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.pipe.it.single; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.consensus.ConsensusGroupId; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.consensus.IConsensus; +import org.apache.iotdb.consensus.common.Peer; +import org.apache.iotdb.consensus.config.ConsensusConfig; +import org.apache.iotdb.consensus.config.RatisConfig; +import org.apache.iotdb.db.consensus.statemachine.schemaregion.SchemaRegionStateMachine; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; +import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeOperateSchemaQueueNode; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.apache.ratis.util.ExitUtils; +import org.apache.ratis.util.TimeDuration; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.awaitility.Awaitility; +import org.junit.After; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Proxy; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +@Category({LocalStandaloneIT.class}) +public class PipeSchemaRegionSnapshotRecoveryIT { + + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @After + public void clearRatisExitState() { + ExitUtils.clear(); + } + + @Test + public void testInstallingSnapshotWithoutListenerClosesStaleQueue() throws Exception { + ExitUtils.disableSystemExit(); + + final SchemaRegionId schemaRegionId = new SchemaRegionId(Integer.MAX_VALUE - 17666); + final int leaderPort = freePort(); + int followerPort = freePort(); + while (followerPort == leaderPort) { + followerPort = freePort(); + } + final Peer leaderPeer = new Peer(schemaRegionId, 0, new TEndPoint("127.0.0.1", leaderPort)); + final Peer followerPeer = new Peer(schemaRegionId, 1, new TEndPoint("127.0.0.1", followerPort)); + final CountDownLatch leaderSnapshotCreated = new CountDownLatch(1); + final CountDownLatch followerSnapshotLoaded = new CountDownLatch(1); + final AtomicReference leaderSnapshotDir = new AtomicReference<>(); + final AtomicBoolean followerSnapshotLoadSucceeded = new AtomicBoolean(); + final AtomicReference staleQueueRef = new AtomicReference<>(); + final ISchemaRegion schemaRegion = + newSchemaRegion(schemaRegionId, leaderSnapshotDir, leaderSnapshotCreated); + + final RatisConfig ratisConfig = + RatisConfig.newBuilder() + .setRpc( + RatisConfig.Rpc.newBuilder() + .setFirstElectionTimeoutMin(TimeDuration.valueOf(1, TimeUnit.SECONDS)) + .setFirstElectionTimeoutMax(TimeDuration.valueOf(4, TimeUnit.SECONDS)) + .setTimeoutMin(TimeDuration.valueOf(1, TimeUnit.SECONDS)) + .setTimeoutMax(TimeDuration.valueOf(4, TimeUnit.SECONDS)) + .build()) + .setLog(RatisConfig.Log.newBuilder().setPurgeUptoSnapshotIndex(true).build()) + .build(); + final IConsensus leader = + newConsensus( + leaderPeer, + ratisConfig, + schemaRegion, + new File(temporaryFolder.getRoot(), "leader"), + null, + null); + final IConsensus follower = + newConsensus( + followerPeer, + ratisConfig, + schemaRegion, + new File(temporaryFolder.getRoot(), "follower"), + followerSnapshotLoaded, + followerSnapshotLoadSucceeded); + boolean leaderStarted = false; + boolean followerStarted = false; + + try { + leader.start(); + leaderStarted = true; + follower.start(); + followerStarted = true; + + leader.createLocalPeer(schemaRegionId, Collections.singletonList(leaderPeer)); + Awaitility.await() + .atMost(30, TimeUnit.SECONDS) + .until(() -> leader.isLeaderReady(schemaRegionId)); + + for (int i = 0; i < 10; i++) { + Assert.assertEquals( + 200, + leader + .write( + schemaRegionId, + new PipeOperateSchemaQueueNode(new PlanNodeId("close" + i), false)) + .getCode()); + } + leader.triggerSnapshot(schemaRegionId, true); + Assert.assertTrue(leaderSnapshotCreated.await(30, TimeUnit.SECONDS)); + Assert.assertFalse(SchemaRegionListeningQueue.hasSnapshot(leaderSnapshotDir.get())); + + final SchemaRegionListeningQueue staleQueue = + PipeDataNodeAgent.runtime().schemaListener(schemaRegionId); + staleQueueRef.set(staleQueue); + PipeDataNodeAgent.runtime().increaseAndGetSchemaListenerReferenceCount(schemaRegionId); + staleQueue.open(); + staleQueue.tryListenToNode( + new CreateTimeSeriesNode( + new PlanNodeId("stale"), + new MeasurementPath("root.db.d1.s1"), + TSDataType.INT32, + TSEncoding.PLAIN, + CompressionType.GZIP, + null, + null, + null, + "alias")); + Assert.assertEquals(1, staleQueue.getSize()); + + follower.createLocalPeer(schemaRegionId, Collections.emptyList()); + leader.addRemotePeer(schemaRegionId, followerPeer); + + Assert.assertTrue(followerSnapshotLoaded.await(30, TimeUnit.SECONDS)); + Assert.assertTrue(followerSnapshotLoadSucceeded.get()); + Assert.assertFalse(staleQueue.isOpened()); + Assert.assertEquals(0, staleQueue.getSize()); + Assert.assertSame( + staleQueue, PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + Assert.assertEquals( + 0, + PipeDataNodeAgent.runtime().decreaseAndGetSchemaListenerReferenceCount(schemaRegionId)); + Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId)); + } finally { + if (followerStarted) { + follower.stop(); + } + if (leaderStarted) { + leader.stop(); + } + if (staleQueueRef.get() != null) { + staleQueueRef.get().close(); + } + PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId); + } + } + + private ISchemaRegion newSchemaRegion( + final SchemaRegionId schemaRegionId, + final AtomicReference snapshotDir, + final CountDownLatch snapshotCreated) { + return (ISchemaRegion) + Proxy.newProxyInstance( + ISchemaRegion.class.getClassLoader(), + new Class[] {ISchemaRegion.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "getSchemaRegionId": + return schemaRegionId; + case "createSnapshot": + final File dir = (File) args[0]; + Files.createDirectories(dir.toPath()); + Files.writeString(new File(dir, "schema.snapshot").toPath(), "schema"); + snapshotDir.set(dir); + snapshotCreated.countDown(); + return true; + case "loadSnapshot": + return new File((File) args[0], "schema.snapshot").isFile(); + default: + throw new UnsupportedOperationException(method.toString()); + } + }); + } + + private IConsensus newConsensus( + final Peer peer, + final RatisConfig ratisConfig, + final ISchemaRegion schemaRegion, + final File storageDir, + final CountDownLatch snapshotLoaded, + final AtomicBoolean snapshotLoadSucceeded) { + return ConsensusFactory.getConsensusImpl( + ConsensusFactory.RATIS_CONSENSUS, + ConsensusConfig.newBuilder() + .setThisNodeId(peer.getNodeId()) + .setThisNode(peer.getEndpoint()) + .setRatisConfig(ratisConfig) + .setStorageDir(storageDir.getAbsolutePath()) + .setConsensusGroupType(TConsensusGroupType.SchemaRegion) + .build(), + ignored -> + new QuietSchemaRegionStateMachine( + schemaRegion, snapshotLoaded, snapshotLoadSucceeded)) + .orElseThrow(IllegalStateException::new); + } + + private int freePort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } + + private static class QuietSchemaRegionStateMachine extends SchemaRegionStateMachine { + + private final CountDownLatch snapshotLoaded; + private final AtomicBoolean snapshotLoadSucceeded; + + private QuietSchemaRegionStateMachine( + final ISchemaRegion schemaRegion, + final CountDownLatch snapshotLoaded, + final AtomicBoolean snapshotLoadSucceeded) { + super(schemaRegion); + this.snapshotLoaded = snapshotLoaded; + this.snapshotLoadSucceeded = snapshotLoadSucceeded; + } + + @Override + public boolean loadSnapshot(final File snapshotDir) { + final boolean succeeded = super.loadSnapshot(snapshotDir); + if (snapshotLoaded != null) { + snapshotLoadSucceeded.set(succeeded); + snapshotLoaded.countDown(); + } + return succeeded; + } + + @Override + public void notifyLeaderChanged(final ConsensusGroupId groupId, final int newLeaderId) {} + + @Override + public void notifyLeaderReady() {} + + @Override + public void notifyNotLeader() {} + + @Override + public void stop() {} + } +}