From 67488c69c59aefccf83bbca6968dd7a69abebf34 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Wed, 11 Nov 2026 18:25:01 +0800
Subject: [PATCH 01/10] Retry replicated LOAD commands after consensus
reconnects
---
.../IoTConsensusDataRegionStateMachine.java | 14 +++
.../storageengine/load/LoadTsFileManager.java | 36 ++++----
.../DataRegionStateMachineTest.java | 35 ++++++++
.../load/LoadTsFileManagerTest.java | 86 +++++++++++++++++++
4 files changed, 155 insertions(+), 16 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java
index f3fecdf69f38b..896241265da91 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java
@@ -31,6 +31,7 @@
import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest;
import org.apache.iotdb.consensus.common.request.IoTConsensusRequest;
import org.apache.iotdb.db.i18n.DataNodeMiscMessages;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -50,6 +51,19 @@ public IoTConsensusDataRegionStateMachine(DataRegion region) {
super(region);
}
+ @Override
+ protected TSStatus write(final PlanNode planNode) {
+ final TSStatus status = super.write(planNode);
+ if (planNode instanceof LoadTsFileConsensusNode
+ && ((LoadTsFileConsensusNode) planNode).isGeneratedByRemoteConsensusLeader()
+ && status.getCode() == TSStatusCode.LOAD_FILE_ERROR.getStatusCode()) {
+ // A reconnect can deliver PREPARE or COMMIT before an earlier piece finishes retrying.
+ // Keep the batch pending so replication cannot discard the command and its staged bytes.
+ status.setCode(TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode());
+ }
+ return status;
+ }
+
@Override
public TSStatus write(IConsensusRequest request) {
try {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
index 7e373d774668c..3cb4319d4f232 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
@@ -99,7 +99,7 @@ public class LoadTsFileManager {
private final LoadTaskRetention retention;
/**
- * The tasks this region finished committing, most recent last, to answer a repeated COMMIT.
+ * The tasks this region finished, most recent last, to answer repeated consensus commands.
*
*
The coordinator cannot tell a lost answer from a lost command - an RPC that timed out may
* have applied - so it has to be able to re-send the COMMIT that ended a task. A task this region
@@ -110,14 +110,15 @@ public class LoadTsFileManager {
* forget it: a repeated COMMIT of a task that committed before the restart is then reported as
* unknown again, which is what it is.
*/
- private static final int COMMITTED_TASK_HISTORY = 256;
+ private static final int FINISHED_TASK_HISTORY = 256;
- private final Map committedLoadIds =
+ private final Map finishedLoadIds =
Collections.synchronizedMap(
- new LinkedHashMap(16, 0.75f, true) {
+ new LinkedHashMap(16, 0.75f, true) {
@Override
- protected boolean removeEldestEntry(final Map.Entry eldest) {
- return size() > COMMITTED_TASK_HISTORY;
+ protected boolean removeEldestEntry(
+ final Map.Entry eldest) {
+ return size() > FINISHED_TASK_HISTORY;
}
});
@@ -384,10 +385,11 @@ public List getActiveTaskDirs() {
return result;
}
- /** Whether this region already committed the task, so that a repeated COMMIT is a no-op. */
- private boolean isAlreadyCommitted(final String loadId) {
- if (committedLoadIds.containsKey(loadId)) {
- return true;
+ /** The terminal command this region already applied to the task, or null if it is unknown. */
+ private LoadTsFileConsensusOp getTerminalOp(final String loadId) {
+ final LoadTsFileConsensusOp finishedOp = finishedLoadIds.get(loadId);
+ if (finishedOp != null) {
+ return finishedOp;
}
for (final String baseDir : LoadStagingDirs.baseDirs()) {
final File taskDir =
@@ -404,8 +406,8 @@ private boolean isAlreadyCommitted(final String loadId) {
for (final File file : files) {
try {
final LoadTsFileProgress.TerminalRecord terminal = LoadTsFileProgress.readTerminal(file);
- if (terminal != null && terminal.op() == LoadTsFileConsensusOp.COMMIT) {
- return true;
+ if (terminal != null) {
+ return terminal.op();
}
} catch (final IOException e) {
LOGGER.warn(
@@ -415,7 +417,7 @@ private boolean isAlreadyCommitted(final String loadId) {
}
}
}
- return false;
+ return null;
}
public boolean prepare(
@@ -423,7 +425,8 @@ public boolean prepare(
final Map timePartitionProgressIndexMap)
throws IOException, LoadFileException {
if (!uuid2WriterManager.containsKey(node.getLoadId())) {
- return false;
+ // A lost PREPARE acknowledgement can be retried after the follower applied COMMIT or ABORT.
+ return node.isGeneratedByRemoteConsensusLeader() && getTerminalOp(node.getLoadId()) != null;
}
if (!uuid2WriterManager.get(node.getLoadId()).hasStagedData()) {
throw new LoadFileException(
@@ -454,7 +457,7 @@ public boolean loadAll(
final Map timePartitionProgressIndexMap)
throws IOException, LoadFileException {
if (!uuid2WriterManager.containsKey(node.getLoadId())) {
- return isAlreadyCommitted(node.getLoadId());
+ return getTerminalOp(node.getLoadId()) == LoadTsFileConsensusOp.COMMIT;
}
LOGGER.info(
StorageEngineMessages
@@ -470,7 +473,7 @@ public boolean loadAll(
.get(node.getLoadId())
.loadAll(node.isGeneratedByPipe(), !mustRetainStagedFiles);
finishConsensusTask(node, mustRetainStagedFiles, LoadTsFileConsensusOp.COMMIT);
- committedLoadIds.put(node.getLoadId(), Boolean.TRUE);
+ finishedLoadIds.put(node.getLoadId(), LoadTsFileConsensusOp.COMMIT);
logLoadNodeToWAL(node);
dataRegion.insertSeparatorToWAL(node);
return true;
@@ -482,6 +485,7 @@ public boolean deleteAll(final LoadTsFileConsensusNode node) throws IOException
}
finishConsensusTask(
node, retention.mustRetain(node.getSearchIndex()), LoadTsFileConsensusOp.ABORT);
+ finishedLoadIds.put(node.getLoadId(), LoadTsFileConsensusOp.ABORT);
logLoadNodeToWAL(node);
dataRegion.insertSeparatorToWAL(node);
return true;
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java
index b77765301ecc8..493125f75d334 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java
@@ -25,14 +25,19 @@
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.IPlanVisitor;
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.commons.utils.RetryUtils;
+import org.apache.iotdb.db.exception.load.LoadFileException;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
+import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.enums.TSDataType;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.Mockito;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -43,6 +48,36 @@
public class DataRegionStateMachineTest {
+ @Test
+ public void testOnlyReplicatedLoadFailuresAreRetried() throws Exception {
+ final DataRegion dataRegion = Mockito.mock(DataRegion.class);
+ final LoadFileException failure = new LoadFileException("injected incomplete LOAD");
+ Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap()))
+ .thenThrow(failure);
+ final IoTConsensusDataRegionStateMachine stateMachine =
+ new IoTConsensusDataRegionStateMachine(dataRegion);
+ final LoadTsFileConsensusNode prepare =
+ LoadTsFileConsensusNode.prepare(
+ new PlanNodeId("prepare"), "load", 1, 1L, false, Collections.emptyMap());
+ final LoadTsFileConsensusNode commit =
+ LoadTsFileConsensusNode.commit(
+ new PlanNodeId("commit"), "load", false, Collections.emptyMap());
+ for (final LoadTsFileConsensusNode node : new LoadTsFileConsensusNode[] {prepare, commit}) {
+ final TSStatus localStatus = stateMachine.write(node);
+ Assert.assertEquals(TSStatusCode.LOAD_FILE_ERROR.getStatusCode(), localStatus.getCode());
+ Assert.assertFalse(RetryUtils.needRetryForWrite(localStatus.getCode()));
+
+ node.markAsGeneratedByRemoteConsensusLeader();
+ final TSStatus remoteStatus = stateMachine.write(node);
+ Assert.assertTrue(RetryUtils.needRetryForWrite(remoteStatus.getCode()));
+ Assert.assertEquals(localStatus.getMessage(), remoteStatus.getMessage());
+ }
+ Mockito.verify(dataRegion, Mockito.times(2))
+ .writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap());
+ Mockito.verify(dataRegion, Mockito.times(2))
+ .writeLoadTsFileCommit(Mockito.any(), Mockito.anyMap());
+ }
+
@Test
public void testMergeInsertRowNodes() throws IllegalPathException {
List list = new ArrayList<>();
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
index 9c0060c5fa9f2..933070908bec2 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
@@ -23,9 +23,12 @@
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
import org.apache.iotdb.commons.consensus.index.impl.RecoverProgressIndex;
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.commons.request.IConsensusRequest;
+import org.apache.iotdb.commons.utils.RetryUtils;
import org.apache.iotdb.consensus.iot.log.ConsensusReqReader;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.consensus.statemachine.dataregion.IoTConsensusDataRegionStateMachine;
import org.apache.iotdb.db.exception.load.LoadFileException;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
@@ -45,6 +48,7 @@
import org.apache.iotdb.db.storageengine.load.splitter.ChunkPayloadUnavailableException;
import org.apache.iotdb.db.storageengine.load.splitter.NonAlignedChunkData;
import org.apache.iotdb.db.storageengine.load.splitter.TsFileData;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.header.ChunkHeader;
@@ -501,6 +505,88 @@ public void testPrepareFailsWhileAPieceOfTheTaskIsStillMissing() throws Exceptio
assertTrue(manager.prepare(prepareNode(loadId), Collections.emptyMap()));
}
+ @Test
+ public void testReplicatedLoadRetriesCommandsThatOvertakePieces() throws Exception {
+ final String loadId = "replicated-out-of-order-load";
+ final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
+ final IoTConsensusDataRegionStateMachine stateMachine =
+ new IoTConsensusDataRegionStateMachine(dataRegion);
+ Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap()))
+ .thenAnswer(
+ invocation -> manager.prepare(invocation.getArgument(0), invocation.getArgument(1)));
+ Mockito.when(dataRegion.writeLoadTsFileCommit(Mockito.any(), Mockito.anyMap()))
+ .thenAnswer(
+ invocation -> manager.loadAll(invocation.getArgument(0), invocation.getArgument(1)));
+ emulateStagedFileImport();
+
+ final LoadTsFileConsensusNode prepare = prepareNode(loadId);
+ final LoadTsFileConsensusNode commit = commitNode(loadId);
+ prepare.markAsGeneratedByRemoteConsensusLeader();
+ commit.markAsGeneratedByRemoteConsensusLeader();
+ try {
+ // Neither command may be acknowledged while the piece is still retrying after reconnect.
+ assertTrue(
+ RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) prepare).getCode()));
+ assertTrue(
+ RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) commit).getCode()));
+
+ final NonAlignedChunkData[] chunks = twoChunksOfTheSamePartition();
+ final List refs =
+ manager.writePiece(loadId, toTsFileDataList(Collections.singletonList(chunks[1])));
+ final File staged = new File(tempDir, refs.get(0).getRelativePath());
+ // A partially received piece set must remain retryable as well.
+ assertTrue(
+ RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) prepare).getCode()));
+ manager.writePiece(loadId, toTsFileDataList(Collections.singletonList(chunks[0])));
+ assertTrue(
+ RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) commit).getCode()));
+
+ assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ stateMachine.write((IConsensusRequest) prepare).getCode());
+ assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ stateMachine.write((IConsensusRequest) commit).getCode());
+ final File imported = new File(new File(tempDir, "data"), staged.getName());
+ assertAllSeriesReadable(
+ imported,
+ java.util.Arrays.asList(
+ new SeriesExpectation(
+ new StringArrayDeviceID("root", "load_manager_test", "d0"), "s0", 10, 0, 9),
+ new SeriesExpectation(
+ new StringArrayDeviceID("root", "load_manager_test", "d1"), "s1", 10, 100, 109)));
+
+ // Losing an acknowledgement must not leave PREPARE retrying forever after COMMIT succeeds.
+ assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ stateMachine.write((IConsensusRequest) prepare).getCode());
+ assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ stateMachine.write((IConsensusRequest) commit).getCode());
+ } finally {
+ manager.stop();
+ }
+ }
+
+ @Test
+ public void testReplicatedPrepareAfterAbortIsANoOp() throws Exception {
+ final String loadId = "replicated-aborted-load";
+ final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
+ final LoadTsFileConsensusNode prepare = prepareNode(loadId);
+ try {
+ manager.writePiece(loadId, Collections.singletonList(laidOutChunk()));
+ assertTrue(manager.prepare(prepare, Collections.emptyMap()));
+ assertTrue(manager.deleteAll(abortNode(loadId)));
+
+ assertFalse(manager.prepare(prepare, Collections.emptyMap()));
+ prepare.markAsGeneratedByRemoteConsensusLeader();
+ assertTrue(manager.prepare(prepare, Collections.emptyMap()));
+ assertFalse(manager.loadAll(commitNode(loadId), Collections.emptyMap()));
+ } finally {
+ manager.stop();
+ }
+ }
+
/**
* A progress file that ends in the middle of an entry is what a snapshot of the staging directory
* holds when it was taken while a piece was being appended. The fragment describes nothing, while
From 001b1e23b1724fca36aab5ca102d021bd5260db7 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Wed, 11 Nov 2026 18:56:49 +0800
Subject: [PATCH 02/10] Retry only incomplete replicated LOAD commands
---
.../dataregion/DataExecutionVisitor.java | 14 +++++++
.../dataregion/DataRegionStateMachine.java | 5 +++
.../IoTConsensusDataRegionStateMachine.java | 14 -------
.../load/LoadFileNotReadyException.java | 32 ++++++++++++++
.../storageengine/load/LoadTsFileManager.java | 26 ++++++++++--
.../load/TsFileWriterManager.java | 5 ++-
.../DataRegionStateMachineTest.java | 42 +++++++++++++++++--
7 files changed, 115 insertions(+), 23 deletions(-)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileNotReadyException.java
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java
index 4f8eca8f27551..461ee1daabf3a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java
@@ -32,6 +32,7 @@
import org.apache.iotdb.db.exception.BatchProcessException;
import org.apache.iotdb.db.exception.WriteProcessException;
import org.apache.iotdb.db.exception.WriteProcessRejectException;
+import org.apache.iotdb.db.exception.load.LoadFileNotReadyException;
import org.apache.iotdb.db.exception.query.OutOfTTLException;
import org.apache.iotdb.db.exception.runtime.TableLostRuntimeException;
import org.apache.iotdb.db.i18n.DataNodeMiscMessages;
@@ -90,6 +91,8 @@ public TSStatus visitLoadTsFileConsensus(
final boolean prepared =
dataRegion.writeLoadTsFilePrepare(node, deserializeProgressIndexes(node));
yield prepared ? StatusUtils.OK : RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR);
+ } catch (final LoadFileNotReadyException e) {
+ yield getLoadFileNotReadyStatus(node, e);
} catch (final Exception e) {
LOGGER.error(DataNodeMiscMessages.ERROR_EXECUTING_PLAN_NODE, node, e);
yield RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, e.getMessage());
@@ -100,6 +103,8 @@ public TSStatus visitLoadTsFileConsensus(
final boolean committed =
dataRegion.writeLoadTsFileCommit(node, deserializeProgressIndexes(node));
yield committed ? StatusUtils.OK : RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR);
+ } catch (final LoadFileNotReadyException e) {
+ yield getLoadFileNotReadyStatus(node, e);
} catch (final Exception e) {
LOGGER.error(DataNodeMiscMessages.ERROR_EXECUTING_PLAN_NODE, node, e);
yield RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, e.getMessage());
@@ -117,6 +122,15 @@ public TSStatus visitLoadTsFileConsensus(
};
}
+ private TSStatus getLoadFileNotReadyStatus(
+ final LoadTsFileConsensusNode node, final LoadFileNotReadyException exception) {
+ if (node.isGeneratedByRemoteConsensusLeader()) {
+ return RpcUtils.getStatus(TSStatusCode.WRITE_PROCESS_REJECT, exception.getMessage());
+ }
+ LOGGER.error(DataNodeMiscMessages.ERROR_EXECUTING_PLAN_NODE, node, exception);
+ return RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exception.getMessage());
+ }
+
/** Restores the pipe progress a PREPARE or COMMIT carries, per time partition. */
private Map deserializeProgressIndexes(
final LoadTsFileConsensusNode node) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java
index 3d2fb3d2006b3..f3c833f4b9b92 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java
@@ -36,6 +36,7 @@
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceManager;
import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.SearchNode;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
@@ -343,6 +344,10 @@ public static boolean needRetryForSpecificCases(int statusCode, PlanNode planNod
// consensus layer.
if (statusCode == TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode()
&& !planNode.isGeneratedByPipe()) {
+ if (planNode instanceof LoadTsFileConsensusNode
+ && ((LoadTsFileConsensusNode) planNode).isGeneratedByRemoteConsensusLeader()) {
+ return false;
+ }
return true;
}
return statusCode == TSStatusCode.METADATA_LEASE_FENCED_RETRY_REQUIRED.getStatusCode()
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java
index 896241265da91..f3fecdf69f38b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java
@@ -31,7 +31,6 @@
import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest;
import org.apache.iotdb.consensus.common.request.IoTConsensusRequest;
import org.apache.iotdb.db.i18n.DataNodeMiscMessages;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -51,19 +50,6 @@ public IoTConsensusDataRegionStateMachine(DataRegion region) {
super(region);
}
- @Override
- protected TSStatus write(final PlanNode planNode) {
- final TSStatus status = super.write(planNode);
- if (planNode instanceof LoadTsFileConsensusNode
- && ((LoadTsFileConsensusNode) planNode).isGeneratedByRemoteConsensusLeader()
- && status.getCode() == TSStatusCode.LOAD_FILE_ERROR.getStatusCode()) {
- // A reconnect can deliver PREPARE or COMMIT before an earlier piece finishes retrying.
- // Keep the batch pending so replication cannot discard the command and its staged bytes.
- status.setCode(TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode());
- }
- return status;
- }
-
@Override
public TSStatus write(IConsensusRequest request) {
try {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileNotReadyException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileNotReadyException.java
new file mode 100644
index 0000000000000..213b039598a9c
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileNotReadyException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.exception.load;
+
+/** A LOAD command arrived before the staged data needed to apply it. */
+public class LoadFileNotReadyException extends LoadFileException {
+
+ public LoadFileNotReadyException() {
+ super("");
+ }
+
+ public LoadFileNotReadyException(final String message) {
+ super(message);
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
index 3cb4319d4f232..23aa43c1f283d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
@@ -25,6 +25,7 @@
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.load.LoadFileException;
+import org.apache.iotdb.db.exception.load.LoadFileNotReadyException;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusOp;
@@ -426,13 +427,23 @@ public boolean prepare(
throws IOException, LoadFileException {
if (!uuid2WriterManager.containsKey(node.getLoadId())) {
// A lost PREPARE acknowledgement can be retried after the follower applied COMMIT or ABORT.
- return node.isGeneratedByRemoteConsensusLeader() && getTerminalOp(node.getLoadId()) != null;
+ if (node.isGeneratedByRemoteConsensusLeader()) {
+ if (getTerminalOp(node.getLoadId()) != null) {
+ return true;
+ }
+ throw new LoadFileNotReadyException();
+ }
+ return false;
}
if (!uuid2WriterManager.get(node.getLoadId()).hasStagedData()) {
- throw new LoadFileException(
+ final String message =
String.format(
StorageEngineMessages.MESSAGE_LOAD_CONSENSUS_PREPARE_WITHOUT_STAGED_DATA_FE8ADC37,
- node.getLoadId()));
+ node.getLoadId());
+ if (node.isGeneratedByRemoteConsensusLeader()) {
+ throw new LoadFileNotReadyException(message);
+ }
+ throw new LoadFileException(message);
}
LOGGER.info(
StorageEngineMessages
@@ -457,7 +468,14 @@ public boolean loadAll(
final Map timePartitionProgressIndexMap)
throws IOException, LoadFileException {
if (!uuid2WriterManager.containsKey(node.getLoadId())) {
- return getTerminalOp(node.getLoadId()) == LoadTsFileConsensusOp.COMMIT;
+ final LoadTsFileConsensusOp terminalOp = getTerminalOp(node.getLoadId());
+ if (terminalOp == LoadTsFileConsensusOp.COMMIT) {
+ return true;
+ }
+ if (terminalOp == null && node.isGeneratedByRemoteConsensusLeader()) {
+ throw new LoadFileNotReadyException();
+ }
+ return false;
}
LOGGER.info(
StorageEngineMessages
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java
index ca30a78b6df23..55e4539372098 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java
@@ -30,6 +30,7 @@
import org.apache.iotdb.commons.utils.RetryUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.load.LoadFileException;
+import org.apache.iotdb.db.exception.load.LoadFileNotReadyException;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode;
@@ -497,7 +498,7 @@ private void validateStagedFiles(
|| progress == null
|| !progress.isReady(fileLength)
|| (expectedDataEnd >= 0L && fileLength < expectedDataEnd)) {
- throw new LoadFileException(
+ throw new LoadFileNotReadyException(
String.format(
StorageEngineMessages.EXCEPTION_LOAD_CONSENSUS_STAGED_FILE_NOT_CONTINUOUS_F9408C19,
stagedFile.getAbsolutePath(),
@@ -570,7 +571,7 @@ void loadAll(final boolean isGeneratedByPipe, final boolean deleteStagedSource)
final TsFilePrecalculatedChunkWriter writer = timePartition2Writer.get(entry.getKey());
if (writer != null && !writer.isSealed()) {
- throw new LoadFileException(
+ throw new LoadFileNotReadyException(
String.format(
StorageEngineMessages.EXCEPTION_LOAD_CONSENSUS_STAGED_FILE_INCOMPLETE_1CDE954B,
tsFileResource.getTsFilePath(),
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java
index 493125f75d334..04302d520cc82 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java
@@ -27,6 +27,7 @@
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.commons.utils.RetryUtils;
import org.apache.iotdb.db.exception.load.LoadFileException;
+import org.apache.iotdb.db.exception.load.LoadFileNotReadyException;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
@@ -49,9 +50,9 @@
public class DataRegionStateMachineTest {
@Test
- public void testOnlyReplicatedLoadFailuresAreRetried() throws Exception {
+ public void testGenericReplicatedLoadFailuresAreNotRetried() throws Exception {
final DataRegion dataRegion = Mockito.mock(DataRegion.class);
- final LoadFileException failure = new LoadFileException("injected incomplete LOAD");
+ final LoadFileException failure = new LoadFileException("injected permanent LOAD failure");
Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap()))
.thenThrow(failure);
final IoTConsensusDataRegionStateMachine stateMachine =
@@ -69,7 +70,8 @@ public void testOnlyReplicatedLoadFailuresAreRetried() throws Exception {
node.markAsGeneratedByRemoteConsensusLeader();
final TSStatus remoteStatus = stateMachine.write(node);
- Assert.assertTrue(RetryUtils.needRetryForWrite(remoteStatus.getCode()));
+ Assert.assertEquals(TSStatusCode.LOAD_FILE_ERROR.getStatusCode(), remoteStatus.getCode());
+ Assert.assertFalse(RetryUtils.needRetryForWrite(remoteStatus.getCode()));
Assert.assertEquals(localStatus.getMessage(), remoteStatus.getMessage());
}
Mockito.verify(dataRegion, Mockito.times(2))
@@ -78,6 +80,40 @@ public void testOnlyReplicatedLoadFailuresAreRetried() throws Exception {
.writeLoadTsFileCommit(Mockito.any(), Mockito.anyMap());
}
+ @Test
+ public void testRemoteLoadNotReadyIsRetriedByConsensus() throws Exception {
+ final DataRegion dataRegion = Mockito.mock(DataRegion.class);
+ Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap()))
+ .thenThrow(new LoadFileNotReadyException());
+ final CountingIoTConsensusDataRegionStateMachine stateMachine =
+ new CountingIoTConsensusDataRegionStateMachine(dataRegion);
+ final LoadTsFileConsensusNode prepare =
+ LoadTsFileConsensusNode.prepare(
+ new PlanNodeId("prepare"), "load", 1, 1L, false, Collections.emptyMap());
+ prepare.markAsGeneratedByRemoteConsensusLeader();
+
+ final TSStatus status = stateMachine.write(prepare);
+
+ Assert.assertEquals(TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode(), status.getCode());
+ Assert.assertTrue(RetryUtils.needRetryForWrite(status.getCode()));
+ Assert.assertEquals(0, stateMachine.localRetries);
+ }
+
+ private static class CountingIoTConsensusDataRegionStateMachine
+ extends IoTConsensusDataRegionStateMachine {
+
+ private int localRetries;
+
+ private CountingIoTConsensusDataRegionStateMachine(final DataRegion region) {
+ super(region);
+ }
+
+ @Override
+ protected void waitBeforeNextWriteRetry() {
+ localRetries++;
+ }
+ }
+
@Test
public void testMergeInsertRowNodes() throws IllegalPathException {
List list = new ArrayList<>();
From a0fe99b6319a4b5858cdd4dc14eaf5ad95cd7cb6 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Thu, 12 Nov 2026 10:30:52 +0800
Subject: [PATCH 03/10] Preserve LOAD terminal decisions across consensus
retries
---
.../storageengine/dataregion/DataRegion.java | 1 +
.../storageengine/load/LoadTsFileCleaner.java | 4 +-
.../storageengine/load/LoadTsFileManager.java | 135 ++++++++++++++----
.../load/LoadTsFileSnapshot.java | 7 +-
.../load/LoadTsFileManagerTest.java | 101 +++++++++++++
5 files changed, 218 insertions(+), 30 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 94f44022ceffe..e8e7ea8ef66fe 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -5664,6 +5664,7 @@ public void markDeleted() {
try {
deleted = true;
stopLoadTsFileManager();
+ LoadTsFileManager.clearTerminalHistory(databaseName, dataRegionIdString);
shutdownUpgradeModFileThreadPool();
releaseDirectBufferMemory();
MetricService.getInstance().removeMetricSet(metrics);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java
index 93a537c1e4aca..8df83a79ada0e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java
@@ -180,7 +180,9 @@ private void scanRegions() {
}
for (final File taskDir : taskDirs) {
- if (!taskDir.isDirectory() || loadId2RetainedTask.containsKey(taskDir.getName())) {
+ if (!taskDir.isDirectory()
+ || LoadTsFileManager.TERMINAL_HISTORY_DIR.equals(taskDir.getName())
+ || loadId2RetainedTask.containsKey(taskDir.getName())) {
continue;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
index 23aa43c1f283d..05b6f8831a7b7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
@@ -48,7 +48,9 @@
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -57,6 +59,7 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
@@ -79,6 +82,8 @@ public class LoadTsFileManager {
private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
+ static final String TERMINAL_HISTORY_DIR = ".finished";
+
public static final Cache MEASUREMENT_ID_CACHE =
Caffeine.newBuilder()
.maximumWeight(CONFIG.getLoadMeasurementIdCacheSizeInBytes())
@@ -102,14 +107,10 @@ public class LoadTsFileManager {
/**
* The tasks this region finished, most recent last, to answer repeated consensus commands.
*
- * The coordinator cannot tell a lost answer from a lost command - an RPC that timed out may
- * have applied - so it has to be able to re-send the COMMIT that ended a task. A task this region
- * already imported is then a no-op and not a failure, otherwise a retry that follows a lost
- * answer would report a load that committed everywhere as failed. A task whose staged directory
- * is still there is recognized from the terminal record of its progress files as well; the record
- * below covers the tasks that were released right away, and it is bounded because a restart may
- * forget it: a repeated COMMIT of a task that committed before the restart is then reported as
- * unknown again, which is what it is.
+ *
This is only a bounded cache. Terminal records also live outside the staged task directories
+ * for the lifetime of the region, and are included in its LOAD snapshot. A replica must still
+ * recognize a command whose successful response was lost after staging cleanup or a restart;
+ * treating that command as missing data would otherwise retry it forever.
*/
private static final int FINISHED_TASK_HISTORY = 256;
@@ -240,7 +241,7 @@ private void recover() {
continue;
}
for (final File uuidDir : uuidDirs) {
- if (!uuidDir.isDirectory()) {
+ if (!uuidDir.isDirectory() || TERMINAL_HISTORY_DIR.equals(uuidDir.getName())) {
continue;
}
final String uuid = uuidDir.getName();
@@ -343,6 +344,9 @@ public List writePiece(final LoadTsFileConsens
if (stopped) {
throw newStoppedIOException();
}
+ if (node.isGeneratedByRemoteConsensusLeader() && getTerminalOp(node.getLoadId()) != null) {
+ return Collections.emptyList();
+ }
final Lock readLock = writersLock.readLock();
readLock.lock();
try {
@@ -386,13 +390,78 @@ public List getActiveTaskDirs() {
return result;
}
+ /** Terminal metadata is snapshotted separately from active tasks and their staged bytes. */
+ List getTerminalHistoryDirs() {
+ final List result = new ArrayList<>();
+ for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) {
+ final File historyDir =
+ new File(getDataRegionLoadDir(new File(baseDir)), TERMINAL_HISTORY_DIR);
+ if (historyDir.isDirectory()) {
+ result.add(historyDir);
+ }
+ }
+ return result;
+ }
+
+ /** Region deletion ends consensus retries; ordinary shutdown must keep these records. */
+ public static void clearTerminalHistory(final String databaseName, final String dataRegionId) {
+ for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) {
+ LoadStagingDirs.deleteTaskDir(
+ new File(
+ LoadStagingDirs.regionLoadDir(new File(baseDir), databaseName, dataRegionId),
+ TERMINAL_HISTORY_DIR));
+ }
+ }
+
+ private void rememberTerminal(final LoadTsFileConsensusNode node, final LoadTsFileConsensusOp op)
+ throws IOException {
+ rememberTerminal(node.getLoadId(), op, node.getSearchIndex());
+ }
+
+ private void rememberTerminal(
+ final String loadId, final LoadTsFileConsensusOp op, final long searchIndex)
+ throws IOException {
+ final File historyDir =
+ new File(
+ getDataRegionLoadDir(new File(LoadStagingDirs.configuredBaseDirs()[0])),
+ TERMINAL_HISTORY_DIR);
+ final LoadTsFileProgress terminal =
+ new LoadTsFileProgress(new File(historyDir, loadId + ".copying." + UUID.randomUUID()));
+ final File temporaryRecord = terminal.getProgressFile();
+ final File record = LoadTsFileProgress.progressFileFor(new File(historyDir, loadId));
+ try {
+ terminal.recordTerminal(op, searchIndex);
+ try {
+ Files.move(
+ temporaryRecord.toPath(),
+ record.toPath(),
+ StandardCopyOption.ATOMIC_MOVE,
+ StandardCopyOption.REPLACE_EXISTING);
+ } catch (final AtomicMoveNotSupportedException e) {
+ Files.move(temporaryRecord.toPath(), record.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+ } finally {
+ Files.deleteIfExists(temporaryRecord.toPath());
+ }
+ finishedLoadIds.put(loadId, op);
+ }
+
/** The terminal command this region already applied to the task, or null if it is unknown. */
- private LoadTsFileConsensusOp getTerminalOp(final String loadId) {
+ private LoadTsFileConsensusOp getTerminalOp(final String loadId) throws IOException {
final LoadTsFileConsensusOp finishedOp = finishedLoadIds.get(loadId);
if (finishedOp != null) {
return finishedOp;
}
- for (final String baseDir : LoadStagingDirs.baseDirs()) {
+ for (final File historyDir : getTerminalHistoryDirs()) {
+ final LoadTsFileProgress.TerminalRecord terminal =
+ LoadTsFileProgress.readTerminal(
+ LoadTsFileProgress.progressFileFor(new File(historyDir, loadId)));
+ if (terminal != null) {
+ finishedLoadIds.put(loadId, terminal.op());
+ return terminal.op();
+ }
+ }
+ for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) {
final File taskDir =
new File(
LoadStagingDirs.regionLoadDir(
@@ -405,16 +474,19 @@ private LoadTsFileConsensusOp getTerminalOp(final String loadId) {
continue;
}
for (final File file : files) {
+ final LoadTsFileProgress.TerminalRecord terminal;
try {
- final LoadTsFileProgress.TerminalRecord terminal = LoadTsFileProgress.readTerminal(file);
- if (terminal != null) {
- return terminal.op();
- }
+ terminal = LoadTsFileProgress.readTerminal(file);
} catch (final IOException e) {
LOGGER.warn(
StorageEngineMessages.LOG_LOAD_CONSENSUS_RECOVER_TASK_META_FAILED_C39E04BB,
file.getPath(),
e);
+ continue;
+ }
+ if (terminal != null) {
+ rememberTerminal(loadId, terminal.op(), terminal.searchIndex());
+ return terminal.op();
}
}
}
@@ -425,12 +497,13 @@ public boolean prepare(
final LoadTsFileConsensusNode node,
final Map timePartitionProgressIndexMap)
throws IOException, LoadFileException {
+ // A terminal decision also fences late pieces and retries from an older consensus batch.
+ if (node.isGeneratedByRemoteConsensusLeader() && getTerminalOp(node.getLoadId()) != null) {
+ return true;
+ }
if (!uuid2WriterManager.containsKey(node.getLoadId())) {
// A lost PREPARE acknowledgement can be retried after the follower applied COMMIT or ABORT.
if (node.isGeneratedByRemoteConsensusLeader()) {
- if (getTerminalOp(node.getLoadId()) != null) {
- return true;
- }
throw new LoadFileNotReadyException();
}
return false;
@@ -467,12 +540,12 @@ public boolean loadAll(
final LoadTsFileConsensusNode node,
final Map timePartitionProgressIndexMap)
throws IOException, LoadFileException {
+ final LoadTsFileConsensusOp terminalOp = getTerminalOp(node.getLoadId());
+ if (terminalOp != null) {
+ return terminalOp == LoadTsFileConsensusOp.COMMIT;
+ }
if (!uuid2WriterManager.containsKey(node.getLoadId())) {
- final LoadTsFileConsensusOp terminalOp = getTerminalOp(node.getLoadId());
- if (terminalOp == LoadTsFileConsensusOp.COMMIT) {
- return true;
- }
- if (terminalOp == null && node.isGeneratedByRemoteConsensusLeader()) {
+ if (node.isGeneratedByRemoteConsensusLeader()) {
throw new LoadFileNotReadyException();
}
return false;
@@ -489,9 +562,13 @@ public boolean loadAll(
final boolean mustRetainStagedFiles = retention.mustRetain(node.getSearchIndex());
uuid2WriterManager
.get(node.getLoadId())
- .loadAll(node.isGeneratedByPipe(), !mustRetainStagedFiles);
+ .loadAll(
+ node.isGeneratedByPipe(),
+ !mustRetainStagedFiles && !node.isGeneratedByRemoteConsensusLeader());
+ // Keep a follower's staged copy until the terminal answer is durable, even without a local WAL
+ // watermark to retain it. A restart before this record can then resume the prepared task.
+ rememberTerminal(node, LoadTsFileConsensusOp.COMMIT);
finishConsensusTask(node, mustRetainStagedFiles, LoadTsFileConsensusOp.COMMIT);
- finishedLoadIds.put(node.getLoadId(), LoadTsFileConsensusOp.COMMIT);
logLoadNodeToWAL(node);
dataRegion.insertSeparatorToWAL(node);
return true;
@@ -499,11 +576,17 @@ public boolean loadAll(
public boolean deleteAll(final LoadTsFileConsensusNode node) throws IOException {
if (!uuid2WriterManager.containsKey(node.getLoadId())) {
+ if (getTerminalOp(node.getLoadId()) == null) {
+ // ABORT may overtake every PIECE. Its success must still settle pending PREPARE retries.
+ rememberTerminal(node, LoadTsFileConsensusOp.ABORT);
+ logLoadNodeToWAL(node);
+ dataRegion.insertSeparatorToWAL(node);
+ }
return false;
}
+ rememberTerminal(node, LoadTsFileConsensusOp.ABORT);
finishConsensusTask(
node, retention.mustRetain(node.getSearchIndex()), LoadTsFileConsensusOp.ABORT);
- finishedLoadIds.put(node.getLoadId(), LoadTsFileConsensusOp.ABORT);
logLoadNodeToWAL(node);
dataRegion.insertSeparatorToWAL(node);
return true;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java
index 64e3e44a524fc..08ded6de6805f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java
@@ -107,7 +107,8 @@ public static boolean snapshot(final DataRegion dataRegion, final File snapshotD
return true;
}
- final List taskDirs = manager.getActiveTaskDirs();
+ final List taskDirs = new ArrayList<>(manager.getActiveTaskDirs());
+ taskDirs.addAll(manager.getTerminalHistoryDirs());
if (taskDirs.isEmpty()) {
return true;
}
@@ -146,14 +147,14 @@ public static boolean snapshot(final DataRegion dataRegion, final File snapshotD
private static void copyTask(final File taskDir, final File targetDir) throws IOException {
// 1. Snapshot raw staged and modification files first
for (final File file : listFiles(taskDir)) {
- if (file.isFile() && !isProgressLog(file)) {
+ if (file.isFile() && !file.getName().contains(TEMP_FILE_SUFFIX) && !isProgressLog(file)) {
publishAtomicCopy(file, new File(targetDir, file.getName()));
}
}
// 2. Snapshot progress logs afterwards so they describe an existing snapshot boundary
for (final File file : listFiles(taskDir)) {
- if (file.isFile() && isProgressLog(file)) {
+ if (file.isFile() && !file.getName().contains(TEMP_FILE_SUFFIX) && isProgressLog(file)) {
publishAtomicCopy(file, new File(targetDir, file.getName()));
}
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
index 933070908bec2..6448d5f2c83d8 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
@@ -25,6 +25,7 @@
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.commons.request.IConsensusRequest;
import org.apache.iotdb.commons.utils.RetryUtils;
+import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.consensus.iot.log.ConsensusReqReader;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -587,6 +588,106 @@ public void testReplicatedPrepareAfterAbortIsANoOp() throws Exception {
}
}
+ @Test
+ public void testReplicatedCommitRetrySurvivesRestartAndSnapshot() throws Exception {
+ final String loadId = "replicated-committed-retry";
+ LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
+ emulateStagedFileImport();
+ final LoadTsFileConsensusNode prepare = prepareNode(loadId);
+ final LoadTsFileConsensusNode commit = commitNode(loadId);
+ prepare.markAsGeneratedByRemoteConsensusLeader();
+ commit.markAsGeneratedByRemoteConsensusLeader();
+ try {
+ manager.writePiece(loadId, Collections.singletonList(laidOutChunk()));
+ assertTrue(manager.prepare(prepare, Collections.emptyMap()));
+ assertTrue(manager.loadAll(commit, Collections.emptyMap()));
+ assertTrue(manager.getActiveTaskDirs().isEmpty());
+
+ // The COMMIT response was lost, and the follower restarts before it is retried.
+ manager.stop();
+ manager = new LoadTsFileManager(dataRegion);
+ assertTrue(manager.loadAll(commit, Collections.emptyMap()));
+ assertTrue(manager.prepare(prepare, Collections.emptyMap()));
+
+ final String originalProtocol = config.getDataRegionConsensusProtocolClass();
+ try {
+ config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager));
+ final File snapshot = new File(tempDir, "finished-load-snapshot");
+ assertTrue(LoadTsFileSnapshot.snapshot(dataRegion, snapshot));
+ manager.stop();
+ LoadTsFileSnapshot.clear(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString());
+ LoadTsFileSnapshot.restore(
+ dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), snapshot);
+ manager = new LoadTsFileManager(dataRegion);
+ assertTrue(manager.prepare(prepare, Collections.emptyMap()));
+ assertTrue(manager.loadAll(commit, Collections.emptyMap()));
+ } finally {
+ config.setDataRegionConsensusProtocolClass(originalProtocol);
+ }
+ Mockito.verify(dataRegion, Mockito.times(1))
+ .loadNewTsFile(
+ Mockito.any(),
+ Mockito.anyBoolean(),
+ Mockito.anyBoolean(),
+ Mockito.anyBoolean(),
+ Mockito.any());
+ } finally {
+ manager.stop();
+ }
+ }
+
+ @Test
+ public void testReplicatedAbortBeforePiecesStopsRetries() throws Exception {
+ final String loadId = "replicated-abort-before-pieces";
+ LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
+ final LoadTsFileConsensusNode prepare = prepareNode(loadId);
+ final LoadTsFileConsensusNode commit = commitNode(loadId);
+ final LoadTsFileConsensusNode abort = abortNode(loadId);
+ final LoadTsFileConsensusNode piece = stagedPiece(loadId, laidOutChunk());
+ prepare.markAsGeneratedByRemoteConsensusLeader();
+ commit.markAsGeneratedByRemoteConsensusLeader();
+ abort.markAsGeneratedByRemoteConsensusLeader();
+ piece.markAsGeneratedByRemoteConsensusLeader();
+ try {
+ Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap()))
+ .thenAnswer(
+ invocation ->
+ dataRegion
+ .getLoadTsFileManagerIfPresent()
+ .get()
+ .prepare(invocation.getArgument(0), invocation.getArgument(1)));
+ Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager));
+ final IoTConsensusDataRegionStateMachine stateMachine =
+ new IoTConsensusDataRegionStateMachine(dataRegion);
+ assertTrue(
+ RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) prepare).getCode()));
+
+ // ABORT can overtake the same missing PIECE as PREPARE did.
+ assertFalse(manager.deleteAll(abort));
+ assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ stateMachine.write((IConsensusRequest) prepare).getCode());
+ manager.writePiece(piece);
+ assertTrue(manager.getActiveTaskDirs().isEmpty());
+ assertTrue(manager.prepare(prepare, Collections.emptyMap()));
+ assertFalse(manager.loadAll(commit, Collections.emptyMap()));
+
+ manager.stop();
+ manager = new LoadTsFileManager(dataRegion);
+ assertTrue(manager.prepare(prepare, Collections.emptyMap()));
+ assertFalse(manager.loadAll(commit, Collections.emptyMap()));
+ manager.writePiece(piece);
+ assertTrue(manager.getActiveTaskDirs().isEmpty());
+ manager.stop();
+ LoadTsFileManager.clearTerminalHistory(
+ dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString());
+ assertTrue(manager.getTerminalHistoryDirs().isEmpty());
+ } finally {
+ manager.stop();
+ }
+ }
+
/**
* A progress file that ends in the middle of an entry is what a snapshot of the staging directory
* holds when it was taken while a piece was being appended. The fragment describes nothing, while
From 0b872f14f661b3971419481bb6ccfcbb8417ea9d Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Thu, 12 Nov 2026 11:00:15 +0800
Subject: [PATCH 04/10] Preserve LOAD terminal decisions during snapshot and
recovery
---
.../dataregion/snapshot/SnapshotTaker.java | 4 +-
.../storageengine/load/LoadTsFileManager.java | 107 ++++++++++------
.../load/LoadTsFileSnapshot.java | 68 ++++++++--
.../load/LoadTsFileManagerTest.java | 119 ++++++++++++++++--
4 files changed, 240 insertions(+), 58 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java
index d32615fec33b6..e0db011fc470a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java
@@ -94,6 +94,7 @@ public boolean takeFullSnapshot(
boolean success;
snapshotLogger.logSnapshotId(finalSnapshotId);
+ final List loadTerminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion);
try {
readLockTheFile();
if (flushBeforeSnapshot) {
@@ -107,7 +108,8 @@ public boolean takeFullSnapshot(
success = createSnapshot(seqFiles, tempSnapshotId);
success = success && createSnapshot(unseqFiles, tempSnapshotId);
success = success && snapshotCompressionRatio(snapshotDirPath);
- success = success && LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir);
+ success =
+ success && LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir, loadTerminalHistory);
} finally {
readUnlockTheFile();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
index 05b6f8831a7b7..4bc1cbbd1ccfa 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
@@ -245,10 +245,18 @@ private void recover() {
continue;
}
final String uuid = uuidDir.getName();
- if (retention.recoverTerminalTask(uuid, uuidDir)) {
- continue;
- }
try {
+ final LoadTsFileProgress.TerminalRecord terminal = readTerminalHistory(uuid);
+ if (terminal != null) {
+ // The process may have stopped after publishing the decision but before closing or
+ // marking staging. That directory belongs to a finished task, never an active writer.
+ finishedLoadIds.put(uuid, terminal.op());
+ retention.retain(uuid, uuidDir, terminal.op(), terminal.searchIndex());
+ continue;
+ }
+ if (retention.recoverTerminalTask(uuid, uuidDir)) {
+ continue;
+ }
final TsFileWriterManager recovered =
new TsFileWriterManager(dataRegion, uuidDir).recoverFromDisk();
if (recovered.hasPendingTsFiles()) {
@@ -392,10 +400,16 @@ public List getActiveTaskDirs() {
/** Terminal metadata is snapshotted separately from active tasks and their staged bytes. */
List getTerminalHistoryDirs() {
+ return getTerminalHistoryDirs(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString());
+ }
+
+ static List getTerminalHistoryDirs(final String databaseName, final String dataRegionId) {
final List result = new ArrayList<>();
for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) {
final File historyDir =
- new File(getDataRegionLoadDir(new File(baseDir)), TERMINAL_HISTORY_DIR);
+ new File(
+ LoadStagingDirs.regionLoadDir(new File(baseDir), databaseName, dataRegionId),
+ TERMINAL_HISTORY_DIR);
if (historyDir.isDirectory()) {
result.add(historyDir);
}
@@ -421,46 +435,66 @@ private void rememberTerminal(final LoadTsFileConsensusNode node, final LoadTsFi
private void rememberTerminal(
final String loadId, final LoadTsFileConsensusOp op, final long searchIndex)
throws IOException {
- final File historyDir =
- new File(
- getDataRegionLoadDir(new File(LoadStagingDirs.configuredBaseDirs()[0])),
- TERMINAL_HISTORY_DIR);
- final LoadTsFileProgress terminal =
- new LoadTsFileProgress(new File(historyDir, loadId + ".copying." + UUID.randomUUID()));
- final File temporaryRecord = terminal.getProgressFile();
- final File record = LoadTsFileProgress.progressFileFor(new File(historyDir, loadId));
+ // Region deletion clears history after stop(), so no terminal publication may outlive it.
+ final Lock readLock = writersLock.readLock();
+ readLock.lock();
try {
- terminal.recordTerminal(op, searchIndex);
+ if (stopped) {
+ throw newStoppedIOException();
+ }
+ final File historyDir =
+ new File(
+ getDataRegionLoadDir(new File(LoadStagingDirs.configuredBaseDirs()[0])),
+ TERMINAL_HISTORY_DIR);
+ final LoadTsFileProgress terminal =
+ new LoadTsFileProgress(new File(historyDir, loadId + ".copying." + UUID.randomUUID()));
+ final File temporaryRecord = terminal.getProgressFile();
+ final File record = LoadTsFileProgress.progressFileFor(new File(historyDir, loadId));
try {
- Files.move(
- temporaryRecord.toPath(),
- record.toPath(),
- StandardCopyOption.ATOMIC_MOVE,
- StandardCopyOption.REPLACE_EXISTING);
- } catch (final AtomicMoveNotSupportedException e) {
- Files.move(temporaryRecord.toPath(), record.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ terminal.recordTerminal(op, searchIndex);
+ try {
+ Files.move(
+ temporaryRecord.toPath(),
+ record.toPath(),
+ StandardCopyOption.ATOMIC_MOVE,
+ StandardCopyOption.REPLACE_EXISTING);
+ } catch (final AtomicMoveNotSupportedException e) {
+ Files.move(
+ temporaryRecord.toPath(), record.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+ } finally {
+ Files.deleteIfExists(temporaryRecord.toPath());
}
+ finishedLoadIds.put(loadId, op);
} finally {
- Files.deleteIfExists(temporaryRecord.toPath());
+ readLock.unlock();
}
- finishedLoadIds.put(loadId, op);
}
- /** The terminal command this region already applied to the task, or null if it is unknown. */
- private LoadTsFileConsensusOp getTerminalOp(final String loadId) throws IOException {
- final LoadTsFileConsensusOp finishedOp = finishedLoadIds.get(loadId);
- if (finishedOp != null) {
- return finishedOp;
- }
+ private LoadTsFileProgress.TerminalRecord readTerminalHistory(final String loadId)
+ throws IOException {
for (final File historyDir : getTerminalHistoryDirs()) {
final LoadTsFileProgress.TerminalRecord terminal =
LoadTsFileProgress.readTerminal(
LoadTsFileProgress.progressFileFor(new File(historyDir, loadId)));
if (terminal != null) {
- finishedLoadIds.put(loadId, terminal.op());
- return terminal.op();
+ return terminal;
}
}
+ return null;
+ }
+
+ /** The terminal command this region already applied to the task, or null if it is unknown. */
+ private LoadTsFileConsensusOp getTerminalOp(final String loadId) throws IOException {
+ final LoadTsFileConsensusOp finishedOp = finishedLoadIds.get(loadId);
+ if (finishedOp != null) {
+ return finishedOp;
+ }
+ final LoadTsFileProgress.TerminalRecord history = readTerminalHistory(loadId);
+ if (history != null) {
+ finishedLoadIds.put(loadId, history.op());
+ return history.op();
+ }
for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) {
final File taskDir =
new File(
@@ -575,13 +609,14 @@ public boolean loadAll(
}
public boolean deleteAll(final LoadTsFileConsensusNode node) throws IOException {
+ if (getTerminalOp(node.getLoadId()) != null) {
+ return false;
+ }
if (!uuid2WriterManager.containsKey(node.getLoadId())) {
- if (getTerminalOp(node.getLoadId()) == null) {
- // ABORT may overtake every PIECE. Its success must still settle pending PREPARE retries.
- rememberTerminal(node, LoadTsFileConsensusOp.ABORT);
- logLoadNodeToWAL(node);
- dataRegion.insertSeparatorToWAL(node);
- }
+ // ABORT may overtake every PIECE. Its success must still settle pending PREPARE retries.
+ rememberTerminal(node, LoadTsFileConsensusOp.ABORT);
+ logLoadNodeToWAL(node);
+ dataRegion.insertSeparatorToWAL(node);
return false;
}
rememberTerminal(node, LoadTsFileConsensusOp.ABORT);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java
index 08ded6de6805f..c0a23843c0306 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java
@@ -89,37 +89,73 @@ private LoadTsFileSnapshot() {}
// Snapshot Entrypoint
// -------------------------------------------------------------------------
- /** Captures an atomic snapshot of active staged tasks under the DataRegion. */
+ /** Captures the LOAD state when no data-file snapshot boundary needs to be supplied. */
public static boolean snapshot(final DataRegion dataRegion, final File snapshotDir) {
+ return snapshot(dataRegion, snapshotDir, null);
+ }
+
+ /**
+ * Selects terminal records before the data-file snapshot captures its resource lists. A COMMIT
+ * published afterwards may describe data absent from those lists, so replay must not be fenced by
+ * that decision. Records are immutable and can be copied after the data files.
+ */
+ public static List captureTerminalHistory(final DataRegion dataRegion) throws IOException {
+ Objects.requireNonNull(
+ dataRegion, StorageEngineMessages.EXCEPTION_DATAREGION_CANNOT_BE_NULL_0B936879);
+ final List result = new ArrayList<>();
+ if (isSupportedProtocol()) {
+ // A restarted region need not have initialized its lazy LOAD manager yet.
+ for (final File historyDir :
+ LoadTsFileManager.getTerminalHistoryDirs(
+ dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString())) {
+ for (final File file : listFiles(historyDir)) {
+ if (file.isFile() && isProgressLog(file) && !file.getName().contains(TEMP_FILE_SUFFIX)) {
+ result.add(file);
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ /** Captures active staging and only the terminal decisions covered by the data-file snapshot. */
+ public static boolean snapshot(
+ final DataRegion dataRegion,
+ final File snapshotDir,
+ final List capturedTerminalHistory) {
Objects.requireNonNull(
dataRegion, StorageEngineMessages.EXCEPTION_DATAREGION_CANNOT_BE_NULL_0B936879);
Objects.requireNonNull(
snapshotDir, StorageEngineMessages.EXCEPTION_SNAPSHOTDIR_CANNOT_BE_NULL_283211CB);
- if (!SUPPORTED_PROTOCOLS.contains(
- ConsensusFactory.normalizeConsensusProtocolClass(
- IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass()))) {
+ if (!isSupportedProtocol()) {
return true;
}
final LoadTsFileManager manager = dataRegion.getLoadTsFileManagerIfPresent().orElse(null);
- if (manager == null) {
- return true;
- }
-
- final List taskDirs = new ArrayList<>(manager.getActiveTaskDirs());
- taskDirs.addAll(manager.getTerminalHistoryDirs());
- if (taskDirs.isEmpty()) {
- return true;
- }
+ final List taskDirs =
+ manager == null ? Collections.emptyList() : manager.getActiveTaskDirs();
final File loadSnapshotDir = new File(snapshotDir, SNAPSHOT_SUBDIR_NAME);
try {
+ final List terminalHistory =
+ capturedTerminalHistory == null
+ ? captureTerminalHistory(dataRegion)
+ : capturedTerminalHistory;
+ if (taskDirs.isEmpty() && terminalHistory.isEmpty()) {
+ return true;
+ }
writeRootsManifest(loadSnapshotDir, taskDirs);
for (final File taskDir : taskDirs) {
final File targetDir = new File(loadSnapshotDir, taskDir.getName());
copyTask(taskDir, targetDir);
}
+ // Terminal records have no payload references and may share any restored staging root.
+ final File historyTargetDir =
+ new File(loadSnapshotDir, LoadTsFileManager.TERMINAL_HISTORY_DIR);
+ for (final File terminal : terminalHistory) {
+ publishAtomicCopy(terminal, new File(historyTargetDir, terminal.getName()));
+ }
LOGGER.info(
String.format(
@@ -137,6 +173,12 @@ public static boolean snapshot(final DataRegion dataRegion, final File snapshotD
}
}
+ private static boolean isSupportedProtocol() {
+ return SUPPORTED_PROTOCOLS.contains(
+ ConsensusFactory.normalizeConsensusProtocolClass(
+ IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass()));
+ }
+
// -------------------------------------------------------------------------
// Task Copy Pipeline & Live Catch-Up
// -------------------------------------------------------------------------
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
index 6448d5f2c83d8..5ba11443062ac 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
@@ -38,6 +38,8 @@
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusOp;
import org.apache.iotdb.db.queryengine.plan.scheduler.load.ChunkOffsetCalculator;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
+import org.apache.iotdb.db.storageengine.dataregion.snapshot.SnapshotTaker;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.db.storageengine.dataregion.wal.node.IWALNode;
@@ -49,6 +51,7 @@
import org.apache.iotdb.db.storageengine.load.splitter.ChunkPayloadUnavailableException;
import org.apache.iotdb.db.storageengine.load.splitter.NonAlignedChunkData;
import org.apache.iotdb.db.storageengine.load.splitter.TsFileData;
+import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.enums.TSDataType;
@@ -123,6 +126,8 @@ public class LoadTsFileManagerTest {
public void setUp() throws Exception {
tempDir = Files.createTempDirectory("load-tsfile-manager-test").toFile();
config = IoTDBDescriptor.getInstance().getConfig();
+ // Startup resolves LOAD roots once. Do that before installing this test's isolated roots.
+ TierManager.getInstance();
originalLoadBaseDirs = config.getLoadTsFileDirs();
config.setLoadTsFileDirs(new String[] {tempDir.getAbsolutePath()});
@@ -251,12 +256,6 @@ public void testOutOfOrderPiecesSurviveTheLossOfTheInMemoryWriters() throws Exce
// progress records stays on disk.
dropInMemoryWriters(firstRun);
- // The first lookup of a staging directory makes the tier manager reformulate the configured
- // LOAD directories from the data directories, which in a test is not the directory the pieces
- // were just staged in. A DataNode does that once at startup, before any load, so pinning the
- // directory here is what keeps the restarted manager looking at the very same staged file.
- config.setLoadTsFileDirs(new String[] {tempDir.getAbsolutePath()});
-
// A new manager resumes the very same staged directory from those records.
final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion);
@@ -612,10 +611,11 @@ public void testReplicatedCommitRetrySurvivesRestartAndSnapshot() throws Excepti
final String originalProtocol = config.getDataRegionConsensusProtocolClass();
try {
config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
- Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager));
+ // After another restart no LOAD has initialized this region's lazy manager yet.
+ manager.stop();
+ Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.empty());
final File snapshot = new File(tempDir, "finished-load-snapshot");
assertTrue(LoadTsFileSnapshot.snapshot(dataRegion, snapshot));
- manager.stop();
LoadTsFileSnapshot.clear(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString());
LoadTsFileSnapshot.restore(
dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), snapshot);
@@ -637,6 +637,99 @@ public void testReplicatedCommitRetrySurvivesRestartAndSnapshot() throws Excepti
}
}
+ @Test
+ public void testSnapshotDoesNotFenceALoadCommittedAfterDataFileCapture() throws Exception {
+ final String loadId = "commit-during-snapshot";
+ final String originalProtocol = config.getDataRegionConsensusProtocolClass();
+ final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
+ try {
+ config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager));
+ emulateStagedFileImport();
+ manager.writePiece(loadId, Collections.singletonList(laidOutChunk()));
+ assertTrue(manager.prepare(prepareNode(loadId), Collections.emptyMap()));
+
+ final TsFileManager tsFileManager = Mockito.mock(TsFileManager.class);
+ Mockito.when(dataRegion.getTsFileManager()).thenReturn(tsFileManager);
+ Mockito.when(tsFileManager.getTsFileList(Mockito.anyBoolean()))
+ .thenAnswer(invocation -> new ArrayList<>());
+ Mockito.doAnswer(
+ invocation -> {
+ // SnapshotTaker already captured its data-file lists before flushing.
+ assertTrue(manager.loadAll(commitNode(loadId), Collections.emptyMap()));
+ return null;
+ })
+ .when(dataRegion)
+ .syncCloseAllWorkingTsFileProcessors();
+
+ final File snapshot = new File(tempDir, "concurrent-commit-snapshot");
+ assertTrue(new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true));
+ final File history =
+ new File(
+ new File(snapshot, LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME),
+ LoadTsFileManager.TERMINAL_HISTORY_DIR);
+ assertFalse(LoadTsFileProgress.progressFileFor(new File(history, loadId)).exists());
+
+ // The data-file snapshot predates COMMIT. Its replay must still stage the missing data.
+ manager.stop();
+ LoadTsFileSnapshot.clear(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString());
+ LoadTsFileSnapshot.restore(
+ dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), snapshot);
+ final LoadTsFileManager restored = new LoadTsFileManager(dataRegion);
+ try {
+ final LoadTsFileConsensusNode piece = stagedPiece(loadId, laidOutChunk());
+ piece.markAsGeneratedByRemoteConsensusLeader();
+ restored.writePiece(piece);
+ assertEquals(1, restored.getActiveTaskDirs().size());
+ final LoadTsFileConsensusNode prepare = prepareNode(loadId);
+ prepare.markAsGeneratedByRemoteConsensusLeader();
+ assertTrue(restored.prepare(prepare, Collections.emptyMap()));
+ } finally {
+ restored.stop();
+ }
+ } finally {
+ manager.stop();
+ config.setDataRegionConsensusProtocolClass(originalProtocol);
+ }
+ }
+
+ @Test
+ public void testAbortCannotReplaceACommittedDecisionBeforeStagingCleanup() throws Exception {
+ final String loadId = "commit-before-staging-cleanup";
+ final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
+ try {
+ emulateStagedFileImport();
+ manager.writePiece(loadId, Collections.singletonList(laidOutChunk()));
+ assertTrue(manager.prepare(prepareNode(loadId), Collections.emptyMap()));
+ final TsFileWriterManager writer = writerOf(manager, loadId);
+ writer.loadAll(false, false);
+ final File historyDir =
+ new File(writer.getTaskDir().getParentFile(), LoadTsFileManager.TERMINAL_HISTORY_DIR);
+ new LoadTsFileProgress(new File(historyDir, loadId))
+ .recordTerminal(LoadTsFileConsensusOp.COMMIT, 10L);
+ // A crash after the durable decision leaves staging without its per-file terminal tails.
+ writer.close(true);
+
+ final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion);
+ try {
+ assertFalse(recovered.deleteAll(abortNode(loadId)));
+ assertTrue(recovered.loadAll(commitNode(loadId), Collections.emptyMap()));
+ assertTrue(recovered.getActiveTaskDirs().isEmpty());
+ Mockito.verify(dataRegion, Mockito.times(1))
+ .loadNewTsFile(
+ Mockito.any(),
+ Mockito.anyBoolean(),
+ Mockito.anyBoolean(),
+ Mockito.anyBoolean(),
+ Mockito.any());
+ } finally {
+ recovered.stop();
+ }
+ } finally {
+ manager.stop();
+ }
+ }
+
@Test
public void testReplicatedAbortBeforePiecesStopsRetries() throws Exception {
final String loadId = "replicated-abort-before-pieces";
@@ -1299,6 +1392,16 @@ public void testStopIsTerminalForWrites() throws Exception {
} catch (final IOException e) {
assertTrue(e.getMessage().contains("has been stopped"));
}
+ LoadTsFileManager.clearTerminalHistory(
+ dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString());
+ try {
+ // A command that already obtained the manager must not recreate a deleted region's history.
+ manager.deleteAll(abortNode("abort-after-region-deletion"));
+ fail("A stopped LOAD manager has to refuse terminal publication");
+ } catch (final IOException e) {
+ assertTrue(e.getMessage().contains("has been stopped"));
+ }
+ assertTrue(manager.getTerminalHistoryDirs().isEmpty());
}
/** A stop waits for the piece in flight, and still closes the writer that piece created. */
From e2a11bdc021a6f013e17a4fb0f2227d0f8b2737d Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Thu, 12 Nov 2026 11:35:39 +0800
Subject: [PATCH 05/10] Keep LOAD snapshot state consistent across commits
---
.../dataregion/snapshot/SnapshotTaker.java | 28 ++++-
.../storageengine/load/LoadTsFileManager.java | 103 +++++++++++------
.../load/LoadTsFileManagerTest.java | 105 ++++++++++++++++++
3 files changed, 196 insertions(+), 40 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java
index e0db011fc470a..50f9aebbfaab3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java
@@ -29,6 +29,7 @@
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.load.LoadTsFileManager;
import org.apache.iotdb.db.storageengine.load.LoadTsFileSnapshot;
import org.slf4j.Logger;
@@ -94,15 +95,30 @@ public boolean takeFullSnapshot(
boolean success;
snapshotLogger.logSnapshotId(finalSnapshotId);
- final List loadTerminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion);
+ final List loadTerminalHistory;
try {
- readLockTheFile();
- if (flushBeforeSnapshot) {
+ synchronized (dataRegion) {
+ // Prevent lazy manager creation while selecting the LOAD and data snapshot boundary.
+ final LoadTsFileManager loadManager =
+ dataRegion.getLoadTsFileManagerIfPresent().orElse(null);
+ if (loadManager != null) {
+ loadManager.lockSnapshotBoundary();
+ }
try {
- dataRegion.writeLock("snapshotTaker");
- dataRegion.syncCloseAllWorkingTsFileProcessors();
+ loadTerminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion);
+ readLockTheFile();
+ if (flushBeforeSnapshot) {
+ dataRegion.writeLock("snapshotTaker");
+ try {
+ dataRegion.syncCloseAllWorkingTsFileProcessors();
+ } finally {
+ dataRegion.writeUnlock();
+ }
+ }
} finally {
- dataRegion.writeUnlock();
+ if (loadManager != null) {
+ loadManager.unlockSnapshotBoundary();
+ }
}
}
success = createSnapshot(seqFiles, tempSnapshotId);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
index 4bc1cbbd1ccfa..7d67ef8b78a5e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
@@ -129,6 +129,9 @@ protected boolean removeEldestEntry(
/** Pieces read, stop() and closing a writer write. Fair, so a stop is not starved. */
private final ReentrantReadWriteLock writersLock = new ReentrantReadWriteLock(true);
+ /** Fences COMMIT/ABORT publication against the data and LOAD-state snapshot boundary. */
+ private final ReentrantReadWriteLock snapshotBoundaryLock = new ReentrantReadWriteLock(true);
+
/** Set under the write side, read as the fast path of a piece. */
private volatile boolean stopped = false;
@@ -255,6 +258,13 @@ private void recover() {
continue;
}
if (retention.recoverTerminalTask(uuid, uuidDir)) {
+ // Older versions kept the decision only in each staged file's progress tail.
+ final LoadTsFileProgress.TerminalRecord retainedTerminal =
+ LoadTsFileProgress.readTerminalOfTaskDir(uuidDir);
+ if (retainedTerminal != null) {
+ rememberTerminal(uuid, retainedTerminal.op(), retainedTerminal.searchIndex());
+ finishedLoadIds.put(uuid, retainedTerminal.op());
+ }
continue;
}
final TsFileWriterManager recovered =
@@ -398,6 +408,15 @@ public List getActiveTaskDirs() {
return result;
}
+ /** Holds the write side while a region snapshot selects its data and LOAD state. */
+ public void lockSnapshotBoundary() {
+ snapshotBoundaryLock.writeLock().lock();
+ }
+
+ public void unlockSnapshotBoundary() {
+ snapshotBoundaryLock.writeLock().unlock();
+ }
+
/** Terminal metadata is snapshotted separately from active tasks and their staged bytes. */
List getTerminalHistoryDirs() {
return getTerminalHistoryDirs(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString());
@@ -574,54 +593,70 @@ public boolean loadAll(
final LoadTsFileConsensusNode node,
final Map timePartitionProgressIndexMap)
throws IOException, LoadFileException {
- final LoadTsFileConsensusOp terminalOp = getTerminalOp(node.getLoadId());
- if (terminalOp != null) {
- return terminalOp == LoadTsFileConsensusOp.COMMIT;
- }
- if (!uuid2WriterManager.containsKey(node.getLoadId())) {
- if (node.isGeneratedByRemoteConsensusLeader()) {
- throw new LoadFileNotReadyException();
+ final Lock snapshotReadLock = snapshotBoundaryLock.readLock();
+ snapshotReadLock.lock();
+ final boolean mustRetainStagedFiles;
+ try {
+ final LoadTsFileConsensusOp terminalOp = getTerminalOp(node.getLoadId());
+ if (terminalOp != null) {
+ return terminalOp == LoadTsFileConsensusOp.COMMIT;
}
- return false;
+ if (!uuid2WriterManager.containsKey(node.getLoadId())) {
+ if (node.isGeneratedByRemoteConsensusLeader()) {
+ throw new LoadFileNotReadyException();
+ }
+ return false;
+ }
+ LOGGER.info(
+ StorageEngineMessages
+ .LOG_COMMITTING_LOAD_TSFILE_ARG_LOADING_PREPARED_RESOURCES_INTO_DATAREGION_EA1D6335,
+ node.getLoadId());
+ // Whether the staged bytes are still needed has to be decided before they are imported: the
+ // WAL entries of the pieces of this task only reference those bytes, and they are expanded on
+ // this node for a replica that has not applied them yet. Importing them by moving the staged
+ // file away would leave those entries unexpandable, so a copy is imported while the watermark
+ // of the WAL has not got past this command yet.
+ mustRetainStagedFiles = retention.mustRetain(node.getSearchIndex());
+ uuid2WriterManager
+ .get(node.getLoadId())
+ .loadAll(
+ node.isGeneratedByPipe(),
+ !mustRetainStagedFiles && !node.isGeneratedByRemoteConsensusLeader());
+ // Keep a follower's staged copy until the terminal answer is durable, even without a local
+ // WAL watermark to retain it. A restart before this record can then resume the prepared task.
+ rememberTerminal(node, LoadTsFileConsensusOp.COMMIT);
+ finishConsensusTask(node, mustRetainStagedFiles, LoadTsFileConsensusOp.COMMIT);
+ } finally {
+ snapshotReadLock.unlock();
}
- LOGGER.info(
- StorageEngineMessages
- .LOG_COMMITTING_LOAD_TSFILE_ARG_LOADING_PREPARED_RESOURCES_INTO_DATAREGION_EA1D6335,
- node.getLoadId());
- // Whether the staged bytes are still needed has to be decided before they are imported: the WAL
- // entries of the pieces of this task only reference those bytes, and they are expanded on this
- // node for a replica that has not applied them yet. Importing them by moving the staged file
- // away would leave those entries unexpandable, so a copy is imported while the watermark of the
- // WAL has not got past this command yet.
- final boolean mustRetainStagedFiles = retention.mustRetain(node.getSearchIndex());
- uuid2WriterManager
- .get(node.getLoadId())
- .loadAll(
- node.isGeneratedByPipe(),
- !mustRetainStagedFiles && !node.isGeneratedByRemoteConsensusLeader());
- // Keep a follower's staged copy until the terminal answer is durable, even without a local WAL
- // watermark to retain it. A restart before this record can then resume the prepared task.
- rememberTerminal(node, LoadTsFileConsensusOp.COMMIT);
- finishConsensusTask(node, mustRetainStagedFiles, LoadTsFileConsensusOp.COMMIT);
logLoadNodeToWAL(node);
dataRegion.insertSeparatorToWAL(node);
return true;
}
public boolean deleteAll(final LoadTsFileConsensusNode node) throws IOException {
- if (getTerminalOp(node.getLoadId()) != null) {
- return false;
- }
- if (!uuid2WriterManager.containsKey(node.getLoadId())) {
+ final Lock snapshotReadLock = snapshotBoundaryLock.readLock();
+ snapshotReadLock.lock();
+ final boolean hasWriter;
+ try {
+ if (getTerminalOp(node.getLoadId()) != null) {
+ return false;
+ }
+ hasWriter = uuid2WriterManager.containsKey(node.getLoadId());
// ABORT may overtake every PIECE. Its success must still settle pending PREPARE retries.
rememberTerminal(node, LoadTsFileConsensusOp.ABORT);
+ if (hasWriter) {
+ finishConsensusTask(
+ node, retention.mustRetain(node.getSearchIndex()), LoadTsFileConsensusOp.ABORT);
+ }
+ } finally {
+ snapshotReadLock.unlock();
+ }
+ if (!hasWriter) {
logLoadNodeToWAL(node);
dataRegion.insertSeparatorToWAL(node);
return false;
}
- rememberTerminal(node, LoadTsFileConsensusOp.ABORT);
- finishConsensusTask(
- node, retention.mustRetain(node.getSearchIndex()), LoadTsFileConsensusOp.ABORT);
logLoadNodeToWAL(node);
dataRegion.insertSeparatorToWAL(node);
return true;
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
index 5ba11443062ac..9b84d0dffce3e 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
@@ -89,6 +89,7 @@
import java.util.Optional;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
@@ -730,6 +731,110 @@ public void testAbortCannotReplaceACommittedDecisionBeforeStagingCleanup() throw
}
}
+ @Test
+ public void testRecoveryMigratesLegacyTerminalRecordIntoSnapshotHistory() throws Exception {
+ final String originalProtocol = config.getDataRegionConsensusProtocolClass();
+ config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ final String loadId = "legacy-terminal-load";
+ try {
+ final LoadTsFileManager beforeUpgrade = new LoadTsFileManager(dataRegion);
+ try {
+ beforeUpgrade.writePiece(loadId, Collections.singletonList(laidOutChunk()));
+ assertTrue(beforeUpgrade.prepare(prepareNode(loadId), Collections.emptyMap()));
+ final TsFileWriterManager writer = writerOf(beforeUpgrade, loadId);
+ writer.recordTerminal(LoadTsFileConsensusOp.COMMIT, 10L, Collections.emptyMap());
+ writer.close(true);
+
+ final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion);
+ try {
+ final List terminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion);
+ assertEquals(1, terminalHistory.size());
+ assertEquals(
+ loadId + LoadTsFileProgress.PROGRESS_SUFFIX, terminalHistory.get(0).getName());
+ } finally {
+ recovered.stop();
+ }
+ } finally {
+ beforeUpgrade.stop();
+ }
+ } finally {
+ config.setDataRegionConsensusProtocolClass(originalProtocol);
+ }
+ }
+
+ @Test
+ public void testCommitCannotSplitSnapshotDataAndTerminalBoundary() throws Exception {
+ final String originalProtocol = config.getDataRegionConsensusProtocolClass();
+ config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ final String loadId = "commit-at-snapshot-boundary";
+ try {
+ final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
+ final AtomicBoolean commitStarted = new AtomicBoolean();
+ final CountDownLatch commitFinished = new CountDownLatch(1);
+ final AtomicReference commitFailure = new AtomicReference<>();
+ final AtomicReference commitThread = new AtomicReference<>();
+ try {
+ final LoadTsFileConsensusNode commit = commitNode(loadId);
+ commit.markAsGeneratedByRemoteConsensusLeader();
+ Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager));
+ emulateStagedFileImport();
+ manager.writePiece(loadId, Collections.singletonList(laidOutChunk()));
+ assertTrue(manager.prepare(prepareNode(loadId), Collections.emptyMap()));
+
+ final TsFileManager tsFileManager = Mockito.mock(TsFileManager.class);
+ Mockito.when(tsFileManager.getTsFileList(Mockito.anyBoolean()))
+ .thenAnswer(invocation -> new ArrayList<>());
+ Mockito.when(dataRegion.getTsFileManager())
+ .thenAnswer(
+ invocation -> {
+ if (commitStarted.compareAndSet(false, true)) {
+ final Thread thread =
+ new Thread(
+ () -> {
+ try {
+ assertTrue(manager.loadAll(commit, Collections.emptyMap()));
+ } catch (final Throwable t) {
+ commitFailure.set(t);
+ } finally {
+ commitFinished.countDown();
+ }
+ });
+ commitThread.set(thread);
+ thread.start();
+ assertFalse(
+ "COMMIT must wait for snapshot boundary capture",
+ commitFinished.await(1, TimeUnit.SECONDS));
+ }
+ return tsFileManager;
+ });
+
+ final File snapshot = new File(tempDir, "commit-boundary-snapshot");
+ assertTrue(
+ new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true));
+ assertTrue(
+ "COMMIT must continue after snapshot boundary capture",
+ commitFinished.await(10, TimeUnit.SECONDS));
+ commitThread.get().join(10_000L);
+ assertFalse(commitThread.get().isAlive());
+ assertNull(commitFailure.get());
+
+ final File history =
+ new File(
+ new File(snapshot, LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME),
+ LoadTsFileManager.TERMINAL_HISTORY_DIR);
+ assertFalse(LoadTsFileProgress.progressFileFor(new File(history, loadId)).exists());
+ } finally {
+ final Thread thread = commitThread.get();
+ if (thread != null) {
+ thread.join(10_000L);
+ }
+ manager.stop();
+ }
+ } finally {
+ config.setDataRegionConsensusProtocolClass(originalProtocol);
+ }
+ }
+
@Test
public void testReplicatedAbortBeforePiecesStopsRetries() throws Exception {
final String loadId = "replicated-abort-before-pieces";
From 9a4701faf79e7a4f2949a5a1c76b998d845f35b4 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Thu, 12 Nov 2026 12:00:26 +0800
Subject: [PATCH 06/10] Preserve LOAD staging and legacy decisions through
snapshots
---
.../dataregion/snapshot/SnapshotTaker.java | 7 ++-
.../storageengine/load/LoadTsFileManager.java | 15 +++--
.../load/LoadTsFileManagerTest.java | 61 ++++++++++++++-----
3 files changed, 58 insertions(+), 25 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java
index 50f9aebbfaab3..b2397c0088bb8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java
@@ -115,17 +115,18 @@ public boolean takeFullSnapshot(
dataRegion.writeUnlock();
}
}
+ // A COMMIT after data-file selection must not remove staging before it is copied.
+ // Its pieces may predate the new replica's sync channel and never be replayed there.
+ success = LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir, loadTerminalHistory);
} finally {
if (loadManager != null) {
loadManager.unlockSnapshotBoundary();
}
}
}
- success = createSnapshot(seqFiles, tempSnapshotId);
+ success = success && createSnapshot(seqFiles, tempSnapshotId);
success = success && createSnapshot(unseqFiles, tempSnapshotId);
success = success && snapshotCompressionRatio(snapshotDirPath);
- success =
- success && LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir, loadTerminalHistory);
} finally {
readUnlockTheFile();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
index 7d67ef8b78a5e..43a1ee954e284 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
@@ -129,7 +129,7 @@ protected boolean removeEldestEntry(
/** Pieces read, stop() and closing a writer write. Fair, so a stop is not starved. */
private final ReentrantReadWriteLock writersLock = new ReentrantReadWriteLock(true);
- /** Fences COMMIT/ABORT publication against the data and LOAD-state snapshot boundary. */
+ /** Fences COMMIT/ABORT against selection and copying of a data and LOAD-state snapshot. */
private final ReentrantReadWriteLock snapshotBoundaryLock = new ReentrantReadWriteLock(true);
/** Set under the write side, read as the fast path of a piece. */
@@ -257,14 +257,13 @@ private void recover() {
retention.retain(uuid, uuidDir, terminal.op(), terminal.searchIndex());
continue;
}
- if (retention.recoverTerminalTask(uuid, uuidDir)) {
+ final LoadTsFileProgress.TerminalRecord retainedTerminal =
+ LoadTsFileProgress.readTerminalOfTaskDir(uuidDir);
+ if (retainedTerminal != null) {
// Older versions kept the decision only in each staged file's progress tail.
- final LoadTsFileProgress.TerminalRecord retainedTerminal =
- LoadTsFileProgress.readTerminalOfTaskDir(uuidDir);
- if (retainedTerminal != null) {
- rememberTerminal(uuid, retainedTerminal.op(), retainedTerminal.searchIndex());
- finishedLoadIds.put(uuid, retainedTerminal.op());
- }
+ // Migrate it before registering a listener that may release the only staged copy.
+ rememberTerminal(uuid, retainedTerminal.op(), retainedTerminal.searchIndex());
+ retention.retain(uuid, uuidDir, retainedTerminal.op(), retainedTerminal.searchIndex());
continue;
}
final TsFileWriterManager recovered =
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
index 9b84d0dffce3e..f4af3318fe629 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
@@ -95,6 +95,7 @@
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.LongConsumer;
+import java.util.function.Predicate;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@@ -639,7 +640,7 @@ public void testReplicatedCommitRetrySurvivesRestartAndSnapshot() throws Excepti
}
@Test
- public void testSnapshotDoesNotFenceALoadCommittedAfterDataFileCapture() throws Exception {
+ public void testSnapshotRetainsStagingForLoadCommittedAfterDataFileCapture() throws Exception {
final String loadId = "commit-during-snapshot";
final String originalProtocol = config.getDataRegionConsensusProtocolClass();
final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
@@ -652,39 +653,55 @@ public void testSnapshotDoesNotFenceALoadCommittedAfterDataFileCapture() throws
final TsFileManager tsFileManager = Mockito.mock(TsFileManager.class);
Mockito.when(dataRegion.getTsFileManager()).thenReturn(tsFileManager);
+ final AtomicBoolean committed = new AtomicBoolean();
Mockito.when(tsFileManager.getTsFileList(Mockito.anyBoolean()))
- .thenAnswer(invocation -> new ArrayList<>());
- Mockito.doAnswer(
- invocation -> {
- // SnapshotTaker already captured its data-file lists before flushing.
- assertTrue(manager.loadAll(commitNode(loadId), Collections.emptyMap()));
- return null;
- })
- .when(dataRegion)
- .syncCloseAllWorkingTsFileProcessors();
+ .thenAnswer(
+ invocation ->
+ new ArrayList() {
+ @Override
+ public boolean removeIf(final Predicate super TsFileResource> filter) {
+ // Data-file copying starts after the snapshot boundary lock is released.
+ if (committed.compareAndSet(false, true)) {
+ try {
+ assertTrue(manager.loadAll(commitNode(loadId), Collections.emptyMap()));
+ } catch (final Exception e) {
+ throw new AssertionError(e);
+ }
+ }
+ return super.removeIf(filter);
+ }
+ });
final File snapshot = new File(tempDir, "concurrent-commit-snapshot");
assertTrue(new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true));
+ assertTrue(committed.get());
final File history =
new File(
new File(snapshot, LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME),
LoadTsFileManager.TERMINAL_HISTORY_DIR);
assertFalse(LoadTsFileProgress.progressFileFor(new File(history, loadId)).exists());
- // The data-file snapshot predates COMMIT. Its replay must still stage the missing data.
+ // PIECE may predate the new replica's sync channel, so it cannot be replayed after restore.
+ // The snapshot itself must retain the staging that its later COMMIT still needs.
manager.stop();
LoadTsFileSnapshot.clear(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString());
LoadTsFileSnapshot.restore(
dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), snapshot);
final LoadTsFileManager restored = new LoadTsFileManager(dataRegion);
try {
- final LoadTsFileConsensusNode piece = stagedPiece(loadId, laidOutChunk());
- piece.markAsGeneratedByRemoteConsensusLeader();
- restored.writePiece(piece);
assertEquals(1, restored.getActiveTaskDirs().size());
final LoadTsFileConsensusNode prepare = prepareNode(loadId);
prepare.markAsGeneratedByRemoteConsensusLeader();
assertTrue(restored.prepare(prepare, Collections.emptyMap()));
+ final File[] stagedFiles =
+ restored.getActiveTaskDirs().get(0).listFiles((dir, name) -> name.endsWith(".tsfile"));
+ assertNotNull(stagedFiles);
+ assertEquals(1, stagedFiles.length);
+ assertAllSeriesReadable(
+ stagedFiles[0],
+ Collections.singletonList(
+ new SeriesExpectation(
+ new StringArrayDeviceID("root", "load_manager_test", "d0"), "s0", 10, 0, 9)));
} finally {
restored.stop();
}
@@ -737,6 +754,7 @@ public void testRecoveryMigratesLegacyTerminalRecordIntoSnapshotHistory() throws
config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
final String loadId = "legacy-terminal-load";
try {
+ useWatermarkWALNode(5L);
final LoadTsFileManager beforeUpgrade = new LoadTsFileManager(dataRegion);
try {
beforeUpgrade.writePiece(loadId, Collections.singletonList(laidOutChunk()));
@@ -745,12 +763,27 @@ public void testRecoveryMigratesLegacyTerminalRecordIntoSnapshotHistory() throws
writer.recordTerminal(LoadTsFileConsensusOp.COMMIT, 10L, Collections.emptyMap());
writer.close(true);
+ final IWALNode walNode = dataRegion.getWALNode().get();
+ Mockito.doAnswer(
+ invocation -> {
+ // A watermark can advance as soon as recovery registers its release listener.
+ final LongConsumer listener = invocation.getArgument(0);
+ walWatermark.set(10L);
+ listener.accept(10L);
+ return null;
+ })
+ .when(walNode)
+ .setSafeDeletedSearchIndexListener(Mockito.any());
final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion);
try {
+ assertFalse(writer.getTaskDir().exists());
final List terminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion);
assertEquals(1, terminalHistory.size());
assertEquals(
loadId + LoadTsFileProgress.PROGRESS_SUFFIX, terminalHistory.get(0).getName());
+ final LoadTsFileConsensusNode commit = commitNode(loadId);
+ commit.markAsGeneratedByRemoteConsensusLeader();
+ assertTrue(recovered.loadAll(commit, Collections.emptyMap()));
} finally {
recovered.stop();
}
From c634776f157151c63022d249d854a6adb2dbd718 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Thu, 12 Nov 2026 13:19:40 +0800
Subject: [PATCH 07/10] Protect LOAD retry metadata during cleanup and deletion
---
.../storageengine/dataregion/DataRegion.java | 39 ++++----
.../storageengine/load/LoadTsFileCleaner.java | 93 +++++++++++--------
.../storageengine/load/LoadTsFileManager.java | 20 ++++
.../dataregion/DataRegionTest.java | 16 ++++
.../load/LoadTsFileCleanerTest.java | 87 +++++++++++++++++
5 files changed, 199 insertions(+), 56 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index e8e7ea8ef66fe..270927aeb1655 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -570,28 +570,29 @@ public String getDatabaseName() {
}
public LoadTsFileManager getLoadTsFileManager() {
- LoadTsFileManager manager = loadTsFileManager;
- if (manager == null) {
- synchronized (this) {
- manager = loadTsFileManager;
- if (manager == null) {
- manager = new LoadTsFileManager(this);
- loadTsFileManager = manager;
+ insertLock.readLock().lock();
+ try {
+ if (deleted) {
+ throw new IllegalStateException(
+ String.format(
+ StorageEngineMessages
+ .STORAGE_EXCEPTION_REGION_ARG_IS_DELETED_SO_IT_TAKES_NO_LOAD_WORK_6E05108F,
+ databaseName + "-" + dataRegionIdString));
+ }
+ LoadTsFileManager manager = loadTsFileManager;
+ if (manager == null) {
+ synchronized (this) {
+ manager = loadTsFileManager;
+ if (manager == null) {
+ manager = new LoadTsFileManager(this);
+ loadTsFileManager = manager;
+ }
}
}
+ return manager;
+ } finally {
+ insertLock.readLock().unlock();
}
- if (deleted) {
- // The region went away: it takes no LOAD work, and the manager is stopped for good. A manager
- // created in this window stops itself here, the one stopLoadTsFileManager saw was stopped by
- // it.
- manager.stop();
- throw new IllegalStateException(
- String.format(
- StorageEngineMessages
- .STORAGE_EXCEPTION_REGION_ARG_IS_DELETED_SO_IT_TAKES_NO_LOAD_WORK_6E05108F,
- databaseName + "-" + dataRegionIdString));
- }
- return manager;
}
public Optional getLoadTsFileManagerIfPresent() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java
index 8df83a79ada0e..7fbfebe6a7d6d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java
@@ -173,47 +173,66 @@ private void scanRegions() {
new File(baseDir),
dataRegion.getDatabaseName(),
dataRegion.getDataRegionIdString());
+ scanRegion(dataRegion, regionDir);
+ }
+ }
+ }
+
+ private void scanRegion(final DataRegion dataRegion, final File regionDir) {
+ File[] taskDirs = regionDir.listFiles();
+ if (taskDirs == null) {
+ return;
+ }
+
+ for (final File taskDir : taskDirs) {
+ if (!taskDir.isDirectory()
+ || LoadTsFileManager.TERMINAL_HISTORY_DIR.equals(taskDir.getName())
+ || loadId2RetainedTask.containsKey(taskDir.getName())) {
+ continue;
+ }
- final File[] taskDirs = regionDir.listFiles();
- if (taskDirs == null) {
+ // A task the region still holds is in progress: the command path and the terminal path
+ // own it, and reading its records here would open every progress file of the directory on
+ // every scan to learn nothing. The idle timeout is still applied to it.
+ final long recordedCommandTimeMillis =
+ recordedCommandTimeMillis(dataRegion, taskDir.getName());
+ if (recordedCommandTimeMillis >= 0L) {
+ reclaimIfIdle(dataRegion, taskDir.getName(), taskDir, recordedCommandTimeMillis);
+ continue;
+ }
+
+ final LoadTsFileProgress.TerminalRecord terminal =
+ LoadTsFileProgress.readTerminalOfTaskDir(taskDir);
+ // A task that never reached COMMIT or ABORT belongs to a coordinator that died before it
+ // sent one: nothing else would ever revisit its staged data.
+ if (terminal == null) {
+ reclaimIfIdle(
+ dataRegion, taskDir.getName(), taskDir, lastCommandTimeMillis(dataRegion, taskDir));
+ continue;
+ }
+
+ // Do not release the only copy of a legacy terminal decision if its migration failed. This
+ // check also covers a manager that was initialized before this scan.
+ if (!LoadTsFileManager.hasTerminalHistory(
+ dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), taskDir.getName())) {
+ try {
+ dataRegion.getLoadTsFileManager();
+ } catch (final IllegalStateException e) {
+ return;
+ }
+ if (!LoadTsFileManager.hasTerminalHistory(
+ dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), taskDir.getName())
+ || loadId2RetainedTask.containsKey(taskDir.getName())) {
continue;
}
+ }
- for (final File taskDir : taskDirs) {
- if (!taskDir.isDirectory()
- || LoadTsFileManager.TERMINAL_HISTORY_DIR.equals(taskDir.getName())
- || loadId2RetainedTask.containsKey(taskDir.getName())) {
- continue;
- }
-
- // A task the region still holds is in progress: the command path and the terminal path
- // own it, and reading its records here would open every progress file of the directory on
- // every scan to learn nothing. The idle timeout is still applied to it.
- final long recordedCommandTimeMillis =
- recordedCommandTimeMillis(dataRegion, taskDir.getName());
- if (recordedCommandTimeMillis >= 0L) {
- reclaimIfIdle(dataRegion, taskDir.getName(), taskDir, recordedCommandTimeMillis);
- continue;
- }
-
- final LoadTsFileProgress.TerminalRecord terminal =
- LoadTsFileProgress.readTerminalOfTaskDir(taskDir);
- // A task that never reached COMMIT or ABORT belongs to a coordinator that died before it
- // sent one: nothing else would ever revisit its staged data.
- if (terminal == null) {
- reclaimIfIdle(
- dataRegion, taskDir.getName(), taskDir, lastCommandTimeMillis(dataRegion, taskDir));
- continue;
- }
-
- final RetainedTask unmanagedTask =
- new RetainedTask(
- taskDir.getName(), dataRegion, taskDir, terminal.searchIndex(), terminal.op());
-
- if (canDelete(unmanagedTask)) {
- delete(unmanagedTask);
- }
- }
+ final RetainedTask unmanagedTask =
+ new RetainedTask(
+ taskDir.getName(), dataRegion, taskDir, terminal.searchIndex(), terminal.op());
+
+ if (canDelete(unmanagedTask)) {
+ delete(unmanagedTask);
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
index 43a1ee954e284..c6d71f2663945 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
@@ -435,6 +435,26 @@ static List getTerminalHistoryDirs(final String databaseName, final String
return result;
}
+ static boolean hasTerminalHistory(
+ final String databaseName, final String dataRegionId, final String loadId) {
+ for (final File historyDir : getTerminalHistoryDirs(databaseName, dataRegionId)) {
+ try {
+ if (LoadTsFileProgress.readTerminal(
+ LoadTsFileProgress.progressFileFor(new File(historyDir, loadId)))
+ != null) {
+ return true;
+ }
+ } catch (final IOException e) {
+ LOGGER.warn(
+ StorageEngineMessages.LOG_LOAD_CONSENSUS_RECOVER_TASK_META_FAILED_C39E04BB,
+ historyDir.getPath(),
+ e);
+ return false;
+ }
+ }
+ return false;
+ }
+
/** Region deletion ends consensus retries; ordinary shutdown must keep these records. */
public static void clearTerminalHistory(final String databaseName, final String dataRegionId) {
for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) {
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
index e131ae6e468d7..6eeb57c279025 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
@@ -2523,6 +2523,22 @@ public void testDeletedRegionStopsItsLoadManagerAndRefusesANewOne() throws Excep
}
}
+ @Test
+ public void testDeletedRegionDoesNotInitializeItsLoadManager() throws Exception {
+ Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty());
+
+ StorageEngine.getInstance().deleteDataRegion(new DataRegionId(0));
+
+ Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty());
+ try {
+ dataRegion.getLoadTsFileManager();
+ Assert.fail("A deleted region must not initialize a LOAD manager");
+ } catch (final IllegalStateException e) {
+ Assert.assertTrue(e.getMessage().contains("is deleted"));
+ }
+ Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty());
+ }
+
public static class DummyDataRegion extends DataRegion {
public DummyDataRegion(String systemInfoDir, String storageGroupName)
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleanerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleanerTest.java
index 01482f2dfe045..3d925c68d56ee 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleanerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleanerTest.java
@@ -45,12 +45,14 @@
import org.mockito.Mockito;
import java.io.File;
+import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
/** How the staged directory of a LOAD that received no command is reclaimed. */
@@ -266,6 +268,91 @@ public void testAFinishedCommittedTaskIsReleasedOnceItsStagedFilesAreComplete()
}
}
+ @Test
+ public void testLegacyTerminalIsMigratedBeforeCleanerReclaimsTask() throws Exception {
+ final String loadId = "legacy-terminal-task";
+ final File regionDir = LoadStagingDirs.regionLoadDir(tempDir, DATABASE, "0");
+ final File taskDir = new File(regionDir, loadId);
+ Assert.assertTrue(taskDir.mkdirs());
+ stageOneChunk(taskDir, 5L);
+ new LoadTsFileProgress(new File(taskDir, "root.sg-0-0.tsfile"))
+ .recordTerminal(LoadTsFileConsensusOp.ABORT, 7L);
+ walWatermark.set(10L);
+
+ final AtomicReference manager = new AtomicReference<>();
+ Mockito.when(dataRegion.getLoadTsFileManagerIfPresent())
+ .thenAnswer(invocation -> Optional.ofNullable(manager.get()));
+ Mockito.when(dataRegion.getLoadTsFileManager())
+ .thenAnswer(
+ invocation -> {
+ final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion);
+ manager.set(recovered);
+ return recovered;
+ });
+
+ try {
+ final Method scanRegion =
+ LoadTsFileCleaner.class.getDeclaredMethod("scanRegion", DataRegion.class, File.class);
+ scanRegion.setAccessible(true);
+ scanRegion.invoke(LoadTsFileCleaner.getInstance(), dataRegion, regionDir);
+
+ Assert.assertFalse("the replicated ABORT may release its old staging", taskDir.exists());
+ final File historyRecord =
+ LoadTsFileProgress.progressFileFor(
+ new File(new File(regionDir, LoadTsFileManager.TERMINAL_HISTORY_DIR), loadId));
+ final LoadTsFileProgress.TerminalRecord terminal =
+ LoadTsFileProgress.readTerminal(historyRecord);
+ Assert.assertNotNull(terminal);
+ Assert.assertEquals(LoadTsFileConsensusOp.ABORT, terminal.op());
+ } finally {
+ if (manager.get() != null) {
+ manager.get().stop();
+ }
+ }
+ }
+
+ @Test
+ public void testLegacyTerminalIsNotReclaimedWhenMigrationCannotPersist() throws Exception {
+ final String loadId = "unmigratable-legacy-terminal-task";
+ final File regionDir = LoadStagingDirs.regionLoadDir(tempDir, DATABASE, "0");
+ final File taskDir = new File(regionDir, loadId);
+ Assert.assertTrue(taskDir.mkdirs());
+ stageOneChunk(taskDir, 5L);
+ new LoadTsFileProgress(new File(taskDir, "root.sg-0-0.tsfile"))
+ .recordTerminal(LoadTsFileConsensusOp.ABORT, 7L);
+ final File historyDir = new File(regionDir, LoadTsFileManager.TERMINAL_HISTORY_DIR);
+ Files.write(historyDir.toPath(), new byte[0]);
+ walWatermark.set(10L);
+
+ final AtomicReference manager = new AtomicReference<>();
+ Mockito.when(dataRegion.getLoadTsFileManagerIfPresent())
+ .thenAnswer(invocation -> Optional.ofNullable(manager.get()));
+ Mockito.when(dataRegion.getLoadTsFileManager())
+ .thenAnswer(
+ invocation -> {
+ final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion);
+ manager.set(recovered);
+ return recovered;
+ });
+
+ try {
+ final Method scanRegion =
+ LoadTsFileCleaner.class.getDeclaredMethod("scanRegion", DataRegion.class, File.class);
+ scanRegion.setAccessible(true);
+ scanRegion.invoke(LoadTsFileCleaner.getInstance(), dataRegion, regionDir);
+
+ Assert.assertTrue(
+ "the staged legacy decision is the only retry record when migration fails",
+ taskDir.exists());
+ Assert.assertTrue(
+ "the test must keep the history destination unavailable", historyDir.isFile());
+ } finally {
+ if (manager.get() != null) {
+ manager.get().stop();
+ }
+ }
+ }
+
@Test
public void testALoadCommandReclaimsAnIdleTaskOfTheSameRegion() throws Exception {
config.setLoadCleanupTaskIdleTimeoutSeconds(1L);
From a7d199b65162c9d62bad1c4e64883e33472fc58f Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Thu, 12 Nov 2026 14:17:05 +0800
Subject: [PATCH 08/10] Avoid snapshot deadlock during LOAD manager
initialization
---
.../storageengine/dataregion/DataRegion.java | 34 ++++++++-------
.../dataregion/DataRegionTest.java | 42 +++++++++++++++++++
2 files changed, 60 insertions(+), 16 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 270927aeb1655..84e1a2b847647 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -570,29 +570,31 @@ public String getDatabaseName() {
}
public LoadTsFileManager getLoadTsFileManager() {
- insertLock.readLock().lock();
- try {
- if (deleted) {
- throw new IllegalStateException(
- String.format(
- StorageEngineMessages
- .STORAGE_EXCEPTION_REGION_ARG_IS_DELETED_SO_IT_TAKES_NO_LOAD_WORK_6E05108F,
- databaseName + "-" + dataRegionIdString));
- }
- LoadTsFileManager manager = loadTsFileManager;
- if (manager == null) {
- synchronized (this) {
+ LoadTsFileManager manager = loadTsFileManager;
+ if (manager == null) {
+ synchronized (this) {
+ // SnapshotTaker takes this monitor before insertLock. Keep that order while still draining
+ // initialization before markDeleted stops the manager and clears its terminal history.
+ insertLock.readLock().lock();
+ try {
manager = loadTsFileManager;
- if (manager == null) {
+ if (manager == null && !deleted) {
manager = new LoadTsFileManager(this);
loadTsFileManager = manager;
}
+ } finally {
+ insertLock.readLock().unlock();
}
}
- return manager;
- } finally {
- insertLock.readLock().unlock();
}
+ if (deleted) {
+ throw new IllegalStateException(
+ String.format(
+ StorageEngineMessages
+ .STORAGE_EXCEPTION_REGION_ARG_IS_DELETED_SO_IT_TAKES_NO_LOAD_WORK_6E05108F,
+ databaseName + "-" + dataRegionIdString));
+ }
+ return manager;
}
public Optional getLoadTsFileManagerIfPresent() {
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
index 6eeb57c279025..7e7c2f2543c9a 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
@@ -147,9 +147,12 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.FutureTask;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
import static org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils.genInsertRowNode;
import static org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils.genInsertTabletNode;
@@ -2539,6 +2542,45 @@ public void testDeletedRegionDoesNotInitializeItsLoadManager() throws Exception
Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty());
}
+ @Test
+ public void testSnapshotDoesNotDeadlockWithLazyLoadManagerInitialization() throws Exception {
+ Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty());
+ final Field insertLockField = DataRegion.class.getDeclaredField("insertLock");
+ insertLockField.setAccessible(true);
+ final Lock insertWriteLock = ((ReadWriteLock) insertLockField.get(dataRegion)).writeLock();
+ final FutureTask initialization =
+ new FutureTask<>(dataRegion::getLoadTsFileManager);
+ final Thread initializingThread = new Thread(initialization, "load-manager-initialization");
+ try {
+ synchronized (dataRegion) {
+ // SnapshotTaker holds this monitor before it flushes under the region's insert write lock.
+ initializingThread.start();
+ await()
+ .atMost(10, TimeUnit.SECONDS)
+ .until(() -> initializingThread.getState() == Thread.State.BLOCKED);
+ // Bound the acquisition so the regression fails instead of leaving deadlocked threads.
+ assertTrue(
+ "Snapshot must acquire insertLock while lazy initialization waits for its monitor",
+ insertWriteLock.tryLock(1, TimeUnit.SECONDS));
+ try {
+ final File snapshot =
+ new File(TestConstant.OUTPUT_DATA_DIR, "snapshot-during-load-manager-initialization");
+ assertTrue(
+ new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true));
+ Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty());
+ } finally {
+ insertWriteLock.unlock();
+ }
+ }
+ Assert.assertSame(
+ initialization.get(10, TimeUnit.SECONDS), dataRegion.getLoadTsFileManager());
+ } finally {
+ initializingThread.join(10_000L);
+ Assert.assertFalse(initializingThread.isAlive());
+ dataRegion.stopLoadTsFileManager();
+ }
+ }
+
public static class DummyDataRegion extends DataRegion {
public DummyDataRegion(String systemInfoDir, String storageGroupName)
From aa6b6acd01da446363f2e337c0163aaeeec0f4d6 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Thu, 12 Nov 2026 15:28:02 +0800
Subject: [PATCH 09/10] Preserve legacy LOAD terminals across snapshots
Capture preserved COMMIT and ABORT progress when terminal-history migration fails, including before lazy manager creation. Add snapshot and restore coverage, and exercise permanent COMMIT exceptions in the state-machine test.
---
.../dataregion/snapshot/SnapshotTaker.java | 3 +-
.../load/LoadTsFileSnapshot.java | 48 +++++++++--
.../DataRegionStateMachineTest.java | 3 +
.../load/LoadTsFileManagerTest.java | 81 ++++++++++++++++++-
4 files changed, 124 insertions(+), 11 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java
index b2397c0088bb8..7a580450e9069 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java
@@ -41,6 +41,7 @@
import java.nio.file.Files;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
@@ -95,7 +96,7 @@ public boolean takeFullSnapshot(
boolean success;
snapshotLogger.logSnapshotId(finalSnapshotId);
- final List loadTerminalHistory;
+ final Map loadTerminalHistory;
try {
synchronized (dataRegion) {
// Prevent lazy manager creation while selecting the LOAD and data snapshot boundary.
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java
index c0a23843c0306..f75b312c93bcb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java
@@ -99,10 +99,11 @@ public static boolean snapshot(final DataRegion dataRegion, final File snapshotD
* published afterwards may describe data absent from those lists, so replay must not be fenced by
* that decision. Records are immutable and can be copied after the data files.
*/
- public static List captureTerminalHistory(final DataRegion dataRegion) throws IOException {
+ public static Map captureTerminalHistory(final DataRegion dataRegion)
+ throws IOException {
Objects.requireNonNull(
dataRegion, StorageEngineMessages.EXCEPTION_DATAREGION_CANNOT_BE_NULL_0B936879);
- final List result = new ArrayList<>();
+ final Map result = new HashMap<>();
if (isSupportedProtocol()) {
// A restarted region need not have initialized its lazy LOAD manager yet.
for (final File historyDir :
@@ -110,7 +111,40 @@ public static List captureTerminalHistory(final DataRegion dataRegion) thr
dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString())) {
for (final File file : listFiles(historyDir)) {
if (file.isFile() && isProgressLog(file) && !file.getName().contains(TEMP_FILE_SUFFIX)) {
- result.add(file);
+ result.put(file.getName(), file);
+ }
+ }
+ }
+ final Set activeLoadIds = new HashSet<>();
+ dataRegion
+ .getLoadTsFileManagerIfPresent()
+ .ifPresent(
+ manager ->
+ manager.getActiveTaskDirs().forEach(dir -> activeLoadIds.add(dir.getName())));
+ for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) {
+ final File regionDir =
+ LoadStagingDirs.regionLoadDir(
+ new File(baseDir),
+ dataRegion.getDatabaseName(),
+ dataRegion.getDataRegionIdString());
+ for (final File taskDir : listFiles(regionDir)) {
+ final String recordName = taskDir.getName() + PROGRESS_SUFFIX;
+ if (!taskDir.isDirectory()
+ || LoadTsFileManager.TERMINAL_HISTORY_DIR.equals(taskDir.getName())
+ || activeLoadIds.contains(taskDir.getName())
+ || result.containsKey(recordName)) {
+ continue;
+ }
+ // A failed migration deliberately leaves the legacy decision in staging. Snapshot that
+ // record under its task identity, even when no lazy manager has recovered the directory.
+ for (final File file : listFiles(taskDir)) {
+ if (file.isFile()
+ && isProgressLog(file)
+ && !file.getName().contains(TEMP_FILE_SUFFIX)
+ && LoadTsFileProgress.readTerminal(file) != null) {
+ result.put(recordName, file);
+ break;
+ }
}
}
}
@@ -122,7 +156,7 @@ public static List captureTerminalHistory(final DataRegion dataRegion) thr
public static boolean snapshot(
final DataRegion dataRegion,
final File snapshotDir,
- final List capturedTerminalHistory) {
+ final Map capturedTerminalHistory) {
Objects.requireNonNull(
dataRegion, StorageEngineMessages.EXCEPTION_DATAREGION_CANNOT_BE_NULL_0B936879);
Objects.requireNonNull(
@@ -138,7 +172,7 @@ public static boolean snapshot(
final File loadSnapshotDir = new File(snapshotDir, SNAPSHOT_SUBDIR_NAME);
try {
- final List terminalHistory =
+ final Map terminalHistory =
capturedTerminalHistory == null
? captureTerminalHistory(dataRegion)
: capturedTerminalHistory;
@@ -153,8 +187,8 @@ public static boolean snapshot(
// Terminal records have no payload references and may share any restored staging root.
final File historyTargetDir =
new File(loadSnapshotDir, LoadTsFileManager.TERMINAL_HISTORY_DIR);
- for (final File terminal : terminalHistory) {
- publishAtomicCopy(terminal, new File(historyTargetDir, terminal.getName()));
+ for (final Map.Entry terminal : terminalHistory.entrySet()) {
+ publishAtomicCopy(terminal.getValue(), new File(historyTargetDir, terminal.getKey()));
}
LOGGER.info(
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java
index 04302d520cc82..c0cc8416a7ca4 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java
@@ -55,6 +55,8 @@ public void testGenericReplicatedLoadFailuresAreNotRetried() throws Exception {
final LoadFileException failure = new LoadFileException("injected permanent LOAD failure");
Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap()))
.thenThrow(failure);
+ Mockito.when(dataRegion.writeLoadTsFileCommit(Mockito.any(), Mockito.anyMap()))
+ .thenThrow(failure);
final IoTConsensusDataRegionStateMachine stateMachine =
new IoTConsensusDataRegionStateMachine(dataRegion);
final LoadTsFileConsensusNode prepare =
@@ -67,6 +69,7 @@ public void testGenericReplicatedLoadFailuresAreNotRetried() throws Exception {
final TSStatus localStatus = stateMachine.write(node);
Assert.assertEquals(TSStatusCode.LOAD_FILE_ERROR.getStatusCode(), localStatus.getCode());
Assert.assertFalse(RetryUtils.needRetryForWrite(localStatus.getCode()));
+ Assert.assertEquals(failure.getMessage(), localStatus.getMessage());
node.markAsGeneratedByRemoteConsensusLeader();
final TSStatus remoteStatus = stateMachine.write(node);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
index f4af3318fe629..f87d51cfb3329 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
@@ -777,10 +777,10 @@ public void testRecoveryMigratesLegacyTerminalRecordIntoSnapshotHistory() throws
final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion);
try {
assertFalse(writer.getTaskDir().exists());
- final List terminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion);
+ final Map terminalHistory =
+ LoadTsFileSnapshot.captureTerminalHistory(dataRegion);
assertEquals(1, terminalHistory.size());
- assertEquals(
- loadId + LoadTsFileProgress.PROGRESS_SUFFIX, terminalHistory.get(0).getName());
+ assertTrue(terminalHistory.containsKey(loadId + LoadTsFileProgress.PROGRESS_SUFFIX));
final LoadTsFileConsensusNode commit = commitNode(loadId);
commit.markAsGeneratedByRemoteConsensusLeader();
assertTrue(recovered.loadAll(commit, Collections.emptyMap()));
@@ -795,6 +795,81 @@ public void testRecoveryMigratesLegacyTerminalRecordIntoSnapshotHistory() throws
}
}
+ @Test
+ public void testSnapshotPreservesLegacyTerminalsWhenMigrationCannotPersist() throws Exception {
+ final String originalProtocol = config.getDataRegionConsensusProtocolClass();
+ config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ LoadStagingDirs.folderManager();
+ final File regionDir =
+ LoadStagingDirs.regionLoadDir(
+ tempDir, dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString());
+ assertTrue(regionDir.mkdirs());
+ final File historyDir = new File(regionDir, LoadTsFileManager.TERMINAL_HISTORY_DIR);
+ Files.write(historyDir.toPath(), new byte[0]);
+ final LoadTsFileConsensusOp[] outcomes = {
+ LoadTsFileConsensusOp.COMMIT, LoadTsFileConsensusOp.ABORT
+ };
+ for (final LoadTsFileConsensusOp outcome : outcomes) {
+ final File taskDir = new File(regionDir, "legacy-" + outcome);
+ new LoadTsFileProgress(new File(taskDir, "legacy.tsfile")).recordTerminal(outcome, 10L);
+ }
+
+ final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion);
+ try {
+ assertTrue(recovered.getActiveTaskDirs().isEmpty());
+ assertTrue(historyDir.isFile());
+ final TsFileManager tsFileManager = Mockito.mock(TsFileManager.class);
+ Mockito.when(dataRegion.getTsFileManager()).thenReturn(tsFileManager);
+ Mockito.when(tsFileManager.getTsFileList(Mockito.anyBoolean()))
+ .thenAnswer(invocation -> new ArrayList<>());
+ File snapshot = null;
+ for (final boolean hasManager : new boolean[] {false, true}) {
+ Mockito.when(dataRegion.getLoadTsFileManagerIfPresent())
+ .thenReturn(hasManager ? Optional.of(recovered) : Optional.empty());
+ snapshot = new File(tempDir, "legacy-terminal-snapshot-" + hasManager);
+ assertTrue(
+ new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true));
+ for (final LoadTsFileConsensusOp outcome : outcomes) {
+ final File snapshotHistory =
+ new File(
+ new File(snapshot, LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME),
+ LoadTsFileManager.TERMINAL_HISTORY_DIR);
+ final LoadTsFileProgress.TerminalRecord terminal =
+ LoadTsFileProgress.readTerminal(
+ new File(
+ snapshotHistory, "legacy-" + outcome + LoadTsFileProgress.PROGRESS_SUFFIX));
+ assertNotNull(terminal);
+ assertEquals(outcome, terminal.op());
+ assertEquals(10L, terminal.searchIndex());
+ }
+ }
+
+ recovered.stop();
+ LoadTsFileSnapshot.clear(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString());
+ LoadTsFileSnapshot.restore(
+ dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), snapshot);
+ final LoadTsFileManager restored = new LoadTsFileManager(dataRegion);
+ try {
+ for (final LoadTsFileConsensusOp outcome : outcomes) {
+ final String loadId = "legacy-" + outcome;
+ final LoadTsFileConsensusNode prepare = prepareNode(loadId);
+ prepare.markAsGeneratedByRemoteConsensusLeader();
+ assertTrue(restored.prepare(prepare, Collections.emptyMap()));
+ final LoadTsFileConsensusNode commit = commitNode(loadId);
+ commit.markAsGeneratedByRemoteConsensusLeader();
+ assertEquals(
+ outcome == LoadTsFileConsensusOp.COMMIT,
+ restored.loadAll(commit, Collections.emptyMap()));
+ }
+ } finally {
+ restored.stop();
+ }
+ } finally {
+ recovered.stop();
+ config.setDataRegionConsensusProtocolClass(originalProtocol);
+ }
+ }
+
@Test
public void testCommitCannotSplitSnapshotDataAndTerminalBoundary() throws Exception {
final String originalProtocol = config.getDataRegionConsensusProtocolClass();
From a730680121d64c526ea2c41efc8961f9b6eced6b Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Thu, 12 Nov 2026 16:12:08 +0800
Subject: [PATCH 10/10] Honor legacy LOAD terminal when migration fails
---
.../storageengine/load/LoadTsFileManager.java | 10 ++++-
.../load/LoadTsFileManagerTest.java | 38 +++++++++++++++++++
2 files changed, 47 insertions(+), 1 deletion(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
index c6d71f2663945..47e9b79c7787e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
@@ -557,7 +557,15 @@ private LoadTsFileConsensusOp getTerminalOp(final String loadId) throws IOExcept
continue;
}
if (terminal != null) {
- rememberTerminal(loadId, terminal.op(), terminal.searchIndex());
+ try {
+ rememberTerminal(loadId, terminal.op(), terminal.searchIndex());
+ } catch (final IOException e) {
+ // The legacy terminal is still authoritative when its migration cannot be published.
+ LOGGER.warn(
+ StorageEngineMessages.LOG_LOAD_CONSENSUS_RECOVER_TASK_META_FAILED_C39E04BB,
+ file.getPath(),
+ e);
+ }
return terminal.op();
}
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
index f87d51cfb3329..98a5facbc11c8 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java
@@ -795,6 +795,44 @@ public void testRecoveryMigratesLegacyTerminalRecordIntoSnapshotHistory() throws
}
}
+ @Test
+ public void testReplicatedRetriesUseLegacyTerminalWhenMigrationFails() throws Exception {
+ final String originalProtocol = config.getDataRegionConsensusProtocolClass();
+ config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ final String loadId = "unmigratable-legacy-abort";
+ final File regionDir =
+ LoadStagingDirs.regionLoadDir(tempDir, dataRegion.getDatabaseName(), "0");
+ final File taskDir = new File(regionDir, loadId);
+ final File historyDir = new File(regionDir, LoadTsFileManager.TERMINAL_HISTORY_DIR);
+ try {
+ assertTrue(taskDir.mkdirs());
+ new LoadTsFileProgress(new File(taskDir, "legacy.tsfile"))
+ .recordTerminal(LoadTsFileConsensusOp.ABORT, 10L);
+ Files.write(historyDir.toPath(), new byte[0]);
+
+ final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
+ try {
+ final LoadTsFileConsensusNode piece = stagedPiece(loadId, laidOutChunk());
+ final LoadTsFileConsensusNode prepare = prepareNode(loadId);
+ final LoadTsFileConsensusNode commit = commitNode(loadId);
+ piece.markAsGeneratedByRemoteConsensusLeader();
+ prepare.markAsGeneratedByRemoteConsensusLeader();
+ commit.markAsGeneratedByRemoteConsensusLeader();
+
+ assertTrue(manager.writePiece(piece).isEmpty());
+ assertTrue(manager.prepare(prepare, Collections.emptyMap()));
+ assertFalse(manager.loadAll(commit, Collections.emptyMap()));
+ assertFalse(manager.deleteAll(abortNode(loadId)));
+ assertTrue(taskDir.isDirectory());
+ assertTrue(historyDir.isFile());
+ } finally {
+ manager.stop();
+ }
+ } finally {
+ config.setDataRegionConsensusProtocolClass(originalProtocol);
+ }
+ }
+
@Test
public void testSnapshotPreservesLegacyTerminalsWhenMigrationCannotPersist() throws Exception {
final String originalProtocol = config.getDataRegionConsensusProtocolClass();