From 67488c69c59aefccf83bbca6968dd7a69abebf34 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:25:01 +0800 Subject: [PATCH 01/10] Retry replicated LOAD commands after consensus reconnects --- .../IoTConsensusDataRegionStateMachine.java | 14 +++ .../storageengine/load/LoadTsFileManager.java | 36 ++++---- .../DataRegionStateMachineTest.java | 35 ++++++++ .../load/LoadTsFileManagerTest.java | 86 +++++++++++++++++++ 4 files changed, 155 insertions(+), 16 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java index f3fecdf69f38b..896241265da91 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java @@ -31,6 +31,7 @@ import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest; import org.apache.iotdb.consensus.common.request.IoTConsensusRequest; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; import org.apache.iotdb.rpc.TSStatusCode; @@ -50,6 +51,19 @@ public IoTConsensusDataRegionStateMachine(DataRegion region) { super(region); } + @Override + protected TSStatus write(final PlanNode planNode) { + final TSStatus status = super.write(planNode); + if (planNode instanceof LoadTsFileConsensusNode + && ((LoadTsFileConsensusNode) planNode).isGeneratedByRemoteConsensusLeader() + && status.getCode() == TSStatusCode.LOAD_FILE_ERROR.getStatusCode()) { + // A reconnect can deliver PREPARE or COMMIT before an earlier piece finishes retrying. + // Keep the batch pending so replication cannot discard the command and its staged bytes. + status.setCode(TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode()); + } + return status; + } + @Override public TSStatus write(IConsensusRequest request) { try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 7e373d774668c..3cb4319d4f232 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -99,7 +99,7 @@ public class LoadTsFileManager { private final LoadTaskRetention retention; /** - * The tasks this region finished committing, most recent last, to answer a repeated COMMIT. + * The tasks this region finished, most recent last, to answer repeated consensus commands. * *

The coordinator cannot tell a lost answer from a lost command - an RPC that timed out may * have applied - so it has to be able to re-send the COMMIT that ended a task. A task this region @@ -110,14 +110,15 @@ public class LoadTsFileManager { * forget it: a repeated COMMIT of a task that committed before the restart is then reported as * unknown again, which is what it is. */ - private static final int COMMITTED_TASK_HISTORY = 256; + private static final int FINISHED_TASK_HISTORY = 256; - private final Map committedLoadIds = + private final Map finishedLoadIds = Collections.synchronizedMap( - new LinkedHashMap(16, 0.75f, true) { + new LinkedHashMap(16, 0.75f, true) { @Override - protected boolean removeEldestEntry(final Map.Entry eldest) { - return size() > COMMITTED_TASK_HISTORY; + protected boolean removeEldestEntry( + final Map.Entry eldest) { + return size() > FINISHED_TASK_HISTORY; } }); @@ -384,10 +385,11 @@ public List getActiveTaskDirs() { return result; } - /** Whether this region already committed the task, so that a repeated COMMIT is a no-op. */ - private boolean isAlreadyCommitted(final String loadId) { - if (committedLoadIds.containsKey(loadId)) { - return true; + /** The terminal command this region already applied to the task, or null if it is unknown. */ + private LoadTsFileConsensusOp getTerminalOp(final String loadId) { + final LoadTsFileConsensusOp finishedOp = finishedLoadIds.get(loadId); + if (finishedOp != null) { + return finishedOp; } for (final String baseDir : LoadStagingDirs.baseDirs()) { final File taskDir = @@ -404,8 +406,8 @@ private boolean isAlreadyCommitted(final String loadId) { for (final File file : files) { try { final LoadTsFileProgress.TerminalRecord terminal = LoadTsFileProgress.readTerminal(file); - if (terminal != null && terminal.op() == LoadTsFileConsensusOp.COMMIT) { - return true; + if (terminal != null) { + return terminal.op(); } } catch (final IOException e) { LOGGER.warn( @@ -415,7 +417,7 @@ private boolean isAlreadyCommitted(final String loadId) { } } } - return false; + return null; } public boolean prepare( @@ -423,7 +425,8 @@ public boolean prepare( final Map timePartitionProgressIndexMap) throws IOException, LoadFileException { if (!uuid2WriterManager.containsKey(node.getLoadId())) { - return false; + // A lost PREPARE acknowledgement can be retried after the follower applied COMMIT or ABORT. + return node.isGeneratedByRemoteConsensusLeader() && getTerminalOp(node.getLoadId()) != null; } if (!uuid2WriterManager.get(node.getLoadId()).hasStagedData()) { throw new LoadFileException( @@ -454,7 +457,7 @@ public boolean loadAll( final Map timePartitionProgressIndexMap) throws IOException, LoadFileException { if (!uuid2WriterManager.containsKey(node.getLoadId())) { - return isAlreadyCommitted(node.getLoadId()); + return getTerminalOp(node.getLoadId()) == LoadTsFileConsensusOp.COMMIT; } LOGGER.info( StorageEngineMessages @@ -470,7 +473,7 @@ public boolean loadAll( .get(node.getLoadId()) .loadAll(node.isGeneratedByPipe(), !mustRetainStagedFiles); finishConsensusTask(node, mustRetainStagedFiles, LoadTsFileConsensusOp.COMMIT); - committedLoadIds.put(node.getLoadId(), Boolean.TRUE); + finishedLoadIds.put(node.getLoadId(), LoadTsFileConsensusOp.COMMIT); logLoadNodeToWAL(node); dataRegion.insertSeparatorToWAL(node); return true; @@ -482,6 +485,7 @@ public boolean deleteAll(final LoadTsFileConsensusNode node) throws IOException } finishConsensusTask( node, retention.mustRetain(node.getSearchIndex()), LoadTsFileConsensusOp.ABORT); + finishedLoadIds.put(node.getLoadId(), LoadTsFileConsensusOp.ABORT); logLoadNodeToWAL(node); dataRegion.insertSeparatorToWAL(node); return true; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java index b77765301ecc8..493125f75d334 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java @@ -25,14 +25,19 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.IPlanVisitor; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.utils.RetryUtils; +import org.apache.iotdb.db.exception.load.LoadFileException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.io.DataOutputStream; import java.io.IOException; @@ -43,6 +48,36 @@ public class DataRegionStateMachineTest { + @Test + public void testOnlyReplicatedLoadFailuresAreRetried() throws Exception { + final DataRegion dataRegion = Mockito.mock(DataRegion.class); + final LoadFileException failure = new LoadFileException("injected incomplete LOAD"); + Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap())) + .thenThrow(failure); + final IoTConsensusDataRegionStateMachine stateMachine = + new IoTConsensusDataRegionStateMachine(dataRegion); + final LoadTsFileConsensusNode prepare = + LoadTsFileConsensusNode.prepare( + new PlanNodeId("prepare"), "load", 1, 1L, false, Collections.emptyMap()); + final LoadTsFileConsensusNode commit = + LoadTsFileConsensusNode.commit( + new PlanNodeId("commit"), "load", false, Collections.emptyMap()); + for (final LoadTsFileConsensusNode node : new LoadTsFileConsensusNode[] {prepare, commit}) { + final TSStatus localStatus = stateMachine.write(node); + Assert.assertEquals(TSStatusCode.LOAD_FILE_ERROR.getStatusCode(), localStatus.getCode()); + Assert.assertFalse(RetryUtils.needRetryForWrite(localStatus.getCode())); + + node.markAsGeneratedByRemoteConsensusLeader(); + final TSStatus remoteStatus = stateMachine.write(node); + Assert.assertTrue(RetryUtils.needRetryForWrite(remoteStatus.getCode())); + Assert.assertEquals(localStatus.getMessage(), remoteStatus.getMessage()); + } + Mockito.verify(dataRegion, Mockito.times(2)) + .writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap()); + Mockito.verify(dataRegion, Mockito.times(2)) + .writeLoadTsFileCommit(Mockito.any(), Mockito.anyMap()); + } + @Test public void testMergeInsertRowNodes() throws IllegalPathException { List list = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java index 9c0060c5fa9f2..933070908bec2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java @@ -23,9 +23,12 @@ import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.RecoverProgressIndex; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.request.IConsensusRequest; +import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.consensus.iot.log.ConsensusReqReader; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.consensus.statemachine.dataregion.IoTConsensusDataRegionStateMachine; import org.apache.iotdb.db.exception.load.LoadFileException; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; @@ -45,6 +48,7 @@ import org.apache.iotdb.db.storageengine.load.splitter.ChunkPayloadUnavailableException; import org.apache.iotdb.db.storageengine.load.splitter.NonAlignedChunkData; import org.apache.iotdb.db.storageengine.load.splitter.TsFileData; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.header.ChunkHeader; @@ -501,6 +505,88 @@ public void testPrepareFailsWhileAPieceOfTheTaskIsStillMissing() throws Exceptio assertTrue(manager.prepare(prepareNode(loadId), Collections.emptyMap())); } + @Test + public void testReplicatedLoadRetriesCommandsThatOvertakePieces() throws Exception { + final String loadId = "replicated-out-of-order-load"; + final LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + final IoTConsensusDataRegionStateMachine stateMachine = + new IoTConsensusDataRegionStateMachine(dataRegion); + Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap())) + .thenAnswer( + invocation -> manager.prepare(invocation.getArgument(0), invocation.getArgument(1))); + Mockito.when(dataRegion.writeLoadTsFileCommit(Mockito.any(), Mockito.anyMap())) + .thenAnswer( + invocation -> manager.loadAll(invocation.getArgument(0), invocation.getArgument(1))); + emulateStagedFileImport(); + + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + final LoadTsFileConsensusNode commit = commitNode(loadId); + prepare.markAsGeneratedByRemoteConsensusLeader(); + commit.markAsGeneratedByRemoteConsensusLeader(); + try { + // Neither command may be acknowledged while the piece is still retrying after reconnect. + assertTrue( + RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) prepare).getCode())); + assertTrue( + RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) commit).getCode())); + + final NonAlignedChunkData[] chunks = twoChunksOfTheSamePartition(); + final List refs = + manager.writePiece(loadId, toTsFileDataList(Collections.singletonList(chunks[1]))); + final File staged = new File(tempDir, refs.get(0).getRelativePath()); + // A partially received piece set must remain retryable as well. + assertTrue( + RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) prepare).getCode())); + manager.writePiece(loadId, toTsFileDataList(Collections.singletonList(chunks[0]))); + assertTrue( + RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) commit).getCode())); + + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + stateMachine.write((IConsensusRequest) prepare).getCode()); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + stateMachine.write((IConsensusRequest) commit).getCode()); + final File imported = new File(new File(tempDir, "data"), staged.getName()); + assertAllSeriesReadable( + imported, + java.util.Arrays.asList( + new SeriesExpectation( + new StringArrayDeviceID("root", "load_manager_test", "d0"), "s0", 10, 0, 9), + new SeriesExpectation( + new StringArrayDeviceID("root", "load_manager_test", "d1"), "s1", 10, 100, 109))); + + // Losing an acknowledgement must not leave PREPARE retrying forever after COMMIT succeeds. + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + stateMachine.write((IConsensusRequest) prepare).getCode()); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + stateMachine.write((IConsensusRequest) commit).getCode()); + } finally { + manager.stop(); + } + } + + @Test + public void testReplicatedPrepareAfterAbortIsANoOp() throws Exception { + final String loadId = "replicated-aborted-load"; + final LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + try { + manager.writePiece(loadId, Collections.singletonList(laidOutChunk())); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertTrue(manager.deleteAll(abortNode(loadId))); + + assertFalse(manager.prepare(prepare, Collections.emptyMap())); + prepare.markAsGeneratedByRemoteConsensusLeader(); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertFalse(manager.loadAll(commitNode(loadId), Collections.emptyMap())); + } finally { + manager.stop(); + } + } + /** * A progress file that ends in the middle of an entry is what a snapshot of the staging directory * holds when it was taken while a piece was being appended. The fragment describes nothing, while From 001b1e23b1724fca36aab5ca102d021bd5260db7 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:56:49 +0800 Subject: [PATCH 02/10] Retry only incomplete replicated LOAD commands --- .../dataregion/DataExecutionVisitor.java | 14 +++++++ .../dataregion/DataRegionStateMachine.java | 5 +++ .../IoTConsensusDataRegionStateMachine.java | 14 ------- .../load/LoadFileNotReadyException.java | 32 ++++++++++++++ .../storageengine/load/LoadTsFileManager.java | 26 ++++++++++-- .../load/TsFileWriterManager.java | 5 ++- .../DataRegionStateMachineTest.java | 42 +++++++++++++++++-- 7 files changed, 115 insertions(+), 23 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileNotReadyException.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java index 4f8eca8f27551..461ee1daabf3a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.exception.BatchProcessException; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.WriteProcessRejectException; +import org.apache.iotdb.db.exception.load.LoadFileNotReadyException; import org.apache.iotdb.db.exception.query.OutOfTTLException; import org.apache.iotdb.db.exception.runtime.TableLostRuntimeException; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; @@ -90,6 +91,8 @@ public TSStatus visitLoadTsFileConsensus( final boolean prepared = dataRegion.writeLoadTsFilePrepare(node, deserializeProgressIndexes(node)); yield prepared ? StatusUtils.OK : RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR); + } catch (final LoadFileNotReadyException e) { + yield getLoadFileNotReadyStatus(node, e); } catch (final Exception e) { LOGGER.error(DataNodeMiscMessages.ERROR_EXECUTING_PLAN_NODE, node, e); yield RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, e.getMessage()); @@ -100,6 +103,8 @@ public TSStatus visitLoadTsFileConsensus( final boolean committed = dataRegion.writeLoadTsFileCommit(node, deserializeProgressIndexes(node)); yield committed ? StatusUtils.OK : RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR); + } catch (final LoadFileNotReadyException e) { + yield getLoadFileNotReadyStatus(node, e); } catch (final Exception e) { LOGGER.error(DataNodeMiscMessages.ERROR_EXECUTING_PLAN_NODE, node, e); yield RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, e.getMessage()); @@ -117,6 +122,15 @@ public TSStatus visitLoadTsFileConsensus( }; } + private TSStatus getLoadFileNotReadyStatus( + final LoadTsFileConsensusNode node, final LoadFileNotReadyException exception) { + if (node.isGeneratedByRemoteConsensusLeader()) { + return RpcUtils.getStatus(TSStatusCode.WRITE_PROCESS_REJECT, exception.getMessage()); + } + LOGGER.error(DataNodeMiscMessages.ERROR_EXECUTING_PLAN_NODE, node, exception); + return RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exception.getMessage()); + } + /** Restores the pipe progress a PREPARE or COMMIT carries, per time partition. */ private Map deserializeProgressIndexes( final LoadTsFileConsensusNode node) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java index 3d2fb3d2006b3..f3c833f4b9b92 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceManager; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.SearchNode; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache; @@ -343,6 +344,10 @@ public static boolean needRetryForSpecificCases(int statusCode, PlanNode planNod // consensus layer. if (statusCode == TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode() && !planNode.isGeneratedByPipe()) { + if (planNode instanceof LoadTsFileConsensusNode + && ((LoadTsFileConsensusNode) planNode).isGeneratedByRemoteConsensusLeader()) { + return false; + } return true; } return statusCode == TSStatusCode.METADATA_LEASE_FENCED_RETRY_REQUIRED.getStatusCode() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java index 896241265da91..f3fecdf69f38b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusDataRegionStateMachine.java @@ -31,7 +31,6 @@ import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest; import org.apache.iotdb.consensus.common.request.IoTConsensusRequest; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; import org.apache.iotdb.rpc.TSStatusCode; @@ -51,19 +50,6 @@ public IoTConsensusDataRegionStateMachine(DataRegion region) { super(region); } - @Override - protected TSStatus write(final PlanNode planNode) { - final TSStatus status = super.write(planNode); - if (planNode instanceof LoadTsFileConsensusNode - && ((LoadTsFileConsensusNode) planNode).isGeneratedByRemoteConsensusLeader() - && status.getCode() == TSStatusCode.LOAD_FILE_ERROR.getStatusCode()) { - // A reconnect can deliver PREPARE or COMMIT before an earlier piece finishes retrying. - // Keep the batch pending so replication cannot discard the command and its staged bytes. - status.setCode(TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode()); - } - return status; - } - @Override public TSStatus write(IConsensusRequest request) { try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileNotReadyException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileNotReadyException.java new file mode 100644 index 0000000000000..213b039598a9c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileNotReadyException.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.exception.load; + +/** A LOAD command arrived before the staged data needed to apply it. */ +public class LoadFileNotReadyException extends LoadFileException { + + public LoadFileNotReadyException() { + super(""); + } + + public LoadFileNotReadyException(final String message) { + super(message); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 3cb4319d4f232..23aa43c1f283d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.load.LoadFileException; +import org.apache.iotdb.db.exception.load.LoadFileNotReadyException; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusOp; @@ -426,13 +427,23 @@ public boolean prepare( throws IOException, LoadFileException { if (!uuid2WriterManager.containsKey(node.getLoadId())) { // A lost PREPARE acknowledgement can be retried after the follower applied COMMIT or ABORT. - return node.isGeneratedByRemoteConsensusLeader() && getTerminalOp(node.getLoadId()) != null; + if (node.isGeneratedByRemoteConsensusLeader()) { + if (getTerminalOp(node.getLoadId()) != null) { + return true; + } + throw new LoadFileNotReadyException(); + } + return false; } if (!uuid2WriterManager.get(node.getLoadId()).hasStagedData()) { - throw new LoadFileException( + final String message = String.format( StorageEngineMessages.MESSAGE_LOAD_CONSENSUS_PREPARE_WITHOUT_STAGED_DATA_FE8ADC37, - node.getLoadId())); + node.getLoadId()); + if (node.isGeneratedByRemoteConsensusLeader()) { + throw new LoadFileNotReadyException(message); + } + throw new LoadFileException(message); } LOGGER.info( StorageEngineMessages @@ -457,7 +468,14 @@ public boolean loadAll( final Map timePartitionProgressIndexMap) throws IOException, LoadFileException { if (!uuid2WriterManager.containsKey(node.getLoadId())) { - return getTerminalOp(node.getLoadId()) == LoadTsFileConsensusOp.COMMIT; + final LoadTsFileConsensusOp terminalOp = getTerminalOp(node.getLoadId()); + if (terminalOp == LoadTsFileConsensusOp.COMMIT) { + return true; + } + if (terminalOp == null && node.isGeneratedByRemoteConsensusLeader()) { + throw new LoadFileNotReadyException(); + } + return false; } LOGGER.info( StorageEngineMessages diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java index ca30a78b6df23..55e4539372098 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.load.LoadFileException; +import org.apache.iotdb.db.exception.load.LoadFileNotReadyException; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; @@ -497,7 +498,7 @@ private void validateStagedFiles( || progress == null || !progress.isReady(fileLength) || (expectedDataEnd >= 0L && fileLength < expectedDataEnd)) { - throw new LoadFileException( + throw new LoadFileNotReadyException( String.format( StorageEngineMessages.EXCEPTION_LOAD_CONSENSUS_STAGED_FILE_NOT_CONTINUOUS_F9408C19, stagedFile.getAbsolutePath(), @@ -570,7 +571,7 @@ void loadAll(final boolean isGeneratedByPipe, final boolean deleteStagedSource) final TsFilePrecalculatedChunkWriter writer = timePartition2Writer.get(entry.getKey()); if (writer != null && !writer.isSealed()) { - throw new LoadFileException( + throw new LoadFileNotReadyException( String.format( StorageEngineMessages.EXCEPTION_LOAD_CONSENSUS_STAGED_FILE_INCOMPLETE_1CDE954B, tsFileResource.getTsFilePath(), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java index 493125f75d334..04302d520cc82 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.db.exception.load.LoadFileException; +import org.apache.iotdb.db.exception.load.LoadFileNotReadyException; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -49,9 +50,9 @@ public class DataRegionStateMachineTest { @Test - public void testOnlyReplicatedLoadFailuresAreRetried() throws Exception { + public void testGenericReplicatedLoadFailuresAreNotRetried() throws Exception { final DataRegion dataRegion = Mockito.mock(DataRegion.class); - final LoadFileException failure = new LoadFileException("injected incomplete LOAD"); + final LoadFileException failure = new LoadFileException("injected permanent LOAD failure"); Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap())) .thenThrow(failure); final IoTConsensusDataRegionStateMachine stateMachine = @@ -69,7 +70,8 @@ public void testOnlyReplicatedLoadFailuresAreRetried() throws Exception { node.markAsGeneratedByRemoteConsensusLeader(); final TSStatus remoteStatus = stateMachine.write(node); - Assert.assertTrue(RetryUtils.needRetryForWrite(remoteStatus.getCode())); + Assert.assertEquals(TSStatusCode.LOAD_FILE_ERROR.getStatusCode(), remoteStatus.getCode()); + Assert.assertFalse(RetryUtils.needRetryForWrite(remoteStatus.getCode())); Assert.assertEquals(localStatus.getMessage(), remoteStatus.getMessage()); } Mockito.verify(dataRegion, Mockito.times(2)) @@ -78,6 +80,40 @@ public void testOnlyReplicatedLoadFailuresAreRetried() throws Exception { .writeLoadTsFileCommit(Mockito.any(), Mockito.anyMap()); } + @Test + public void testRemoteLoadNotReadyIsRetriedByConsensus() throws Exception { + final DataRegion dataRegion = Mockito.mock(DataRegion.class); + Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap())) + .thenThrow(new LoadFileNotReadyException()); + final CountingIoTConsensusDataRegionStateMachine stateMachine = + new CountingIoTConsensusDataRegionStateMachine(dataRegion); + final LoadTsFileConsensusNode prepare = + LoadTsFileConsensusNode.prepare( + new PlanNodeId("prepare"), "load", 1, 1L, false, Collections.emptyMap()); + prepare.markAsGeneratedByRemoteConsensusLeader(); + + final TSStatus status = stateMachine.write(prepare); + + Assert.assertEquals(TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode(), status.getCode()); + Assert.assertTrue(RetryUtils.needRetryForWrite(status.getCode())); + Assert.assertEquals(0, stateMachine.localRetries); + } + + private static class CountingIoTConsensusDataRegionStateMachine + extends IoTConsensusDataRegionStateMachine { + + private int localRetries; + + private CountingIoTConsensusDataRegionStateMachine(final DataRegion region) { + super(region); + } + + @Override + protected void waitBeforeNextWriteRetry() { + localRetries++; + } + } + @Test public void testMergeInsertRowNodes() throws IllegalPathException { List list = new ArrayList<>(); From a0fe99b6319a4b5858cdd4dc14eaf5ad95cd7cb6 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:30:52 +0800 Subject: [PATCH 03/10] Preserve LOAD terminal decisions across consensus retries --- .../storageengine/dataregion/DataRegion.java | 1 + .../storageengine/load/LoadTsFileCleaner.java | 4 +- .../storageengine/load/LoadTsFileManager.java | 135 ++++++++++++++---- .../load/LoadTsFileSnapshot.java | 7 +- .../load/LoadTsFileManagerTest.java | 101 +++++++++++++ 5 files changed, 218 insertions(+), 30 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 94f44022ceffe..e8e7ea8ef66fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -5664,6 +5664,7 @@ public void markDeleted() { try { deleted = true; stopLoadTsFileManager(); + LoadTsFileManager.clearTerminalHistory(databaseName, dataRegionIdString); shutdownUpgradeModFileThreadPool(); releaseDirectBufferMemory(); MetricService.getInstance().removeMetricSet(metrics); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java index 93a537c1e4aca..8df83a79ada0e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java @@ -180,7 +180,9 @@ private void scanRegions() { } for (final File taskDir : taskDirs) { - if (!taskDir.isDirectory() || loadId2RetainedTask.containsKey(taskDir.getName())) { + if (!taskDir.isDirectory() + || LoadTsFileManager.TERMINAL_HISTORY_DIR.equals(taskDir.getName()) + || loadId2RetainedTask.containsKey(taskDir.getName())) { continue; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 23aa43c1f283d..05b6f8831a7b7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -48,7 +48,9 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -57,6 +59,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; @@ -79,6 +82,8 @@ public class LoadTsFileManager { private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); + static final String TERMINAL_HISTORY_DIR = ".finished"; + public static final Cache MEASUREMENT_ID_CACHE = Caffeine.newBuilder() .maximumWeight(CONFIG.getLoadMeasurementIdCacheSizeInBytes()) @@ -102,14 +107,10 @@ public class LoadTsFileManager { /** * The tasks this region finished, most recent last, to answer repeated consensus commands. * - *

The coordinator cannot tell a lost answer from a lost command - an RPC that timed out may - * have applied - so it has to be able to re-send the COMMIT that ended a task. A task this region - * already imported is then a no-op and not a failure, otherwise a retry that follows a lost - * answer would report a load that committed everywhere as failed. A task whose staged directory - * is still there is recognized from the terminal record of its progress files as well; the record - * below covers the tasks that were released right away, and it is bounded because a restart may - * forget it: a repeated COMMIT of a task that committed before the restart is then reported as - * unknown again, which is what it is. + *

This is only a bounded cache. Terminal records also live outside the staged task directories + * for the lifetime of the region, and are included in its LOAD snapshot. A replica must still + * recognize a command whose successful response was lost after staging cleanup or a restart; + * treating that command as missing data would otherwise retry it forever. */ private static final int FINISHED_TASK_HISTORY = 256; @@ -240,7 +241,7 @@ private void recover() { continue; } for (final File uuidDir : uuidDirs) { - if (!uuidDir.isDirectory()) { + if (!uuidDir.isDirectory() || TERMINAL_HISTORY_DIR.equals(uuidDir.getName())) { continue; } final String uuid = uuidDir.getName(); @@ -343,6 +344,9 @@ public List writePiece(final LoadTsFileConsens if (stopped) { throw newStoppedIOException(); } + if (node.isGeneratedByRemoteConsensusLeader() && getTerminalOp(node.getLoadId()) != null) { + return Collections.emptyList(); + } final Lock readLock = writersLock.readLock(); readLock.lock(); try { @@ -386,13 +390,78 @@ public List getActiveTaskDirs() { return result; } + /** Terminal metadata is snapshotted separately from active tasks and their staged bytes. */ + List getTerminalHistoryDirs() { + final List result = new ArrayList<>(); + for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) { + final File historyDir = + new File(getDataRegionLoadDir(new File(baseDir)), TERMINAL_HISTORY_DIR); + if (historyDir.isDirectory()) { + result.add(historyDir); + } + } + return result; + } + + /** Region deletion ends consensus retries; ordinary shutdown must keep these records. */ + public static void clearTerminalHistory(final String databaseName, final String dataRegionId) { + for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) { + LoadStagingDirs.deleteTaskDir( + new File( + LoadStagingDirs.regionLoadDir(new File(baseDir), databaseName, dataRegionId), + TERMINAL_HISTORY_DIR)); + } + } + + private void rememberTerminal(final LoadTsFileConsensusNode node, final LoadTsFileConsensusOp op) + throws IOException { + rememberTerminal(node.getLoadId(), op, node.getSearchIndex()); + } + + private void rememberTerminal( + final String loadId, final LoadTsFileConsensusOp op, final long searchIndex) + throws IOException { + final File historyDir = + new File( + getDataRegionLoadDir(new File(LoadStagingDirs.configuredBaseDirs()[0])), + TERMINAL_HISTORY_DIR); + final LoadTsFileProgress terminal = + new LoadTsFileProgress(new File(historyDir, loadId + ".copying." + UUID.randomUUID())); + final File temporaryRecord = terminal.getProgressFile(); + final File record = LoadTsFileProgress.progressFileFor(new File(historyDir, loadId)); + try { + terminal.recordTerminal(op, searchIndex); + try { + Files.move( + temporaryRecord.toPath(), + record.toPath(), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (final AtomicMoveNotSupportedException e) { + Files.move(temporaryRecord.toPath(), record.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } finally { + Files.deleteIfExists(temporaryRecord.toPath()); + } + finishedLoadIds.put(loadId, op); + } + /** The terminal command this region already applied to the task, or null if it is unknown. */ - private LoadTsFileConsensusOp getTerminalOp(final String loadId) { + private LoadTsFileConsensusOp getTerminalOp(final String loadId) throws IOException { final LoadTsFileConsensusOp finishedOp = finishedLoadIds.get(loadId); if (finishedOp != null) { return finishedOp; } - for (final String baseDir : LoadStagingDirs.baseDirs()) { + for (final File historyDir : getTerminalHistoryDirs()) { + final LoadTsFileProgress.TerminalRecord terminal = + LoadTsFileProgress.readTerminal( + LoadTsFileProgress.progressFileFor(new File(historyDir, loadId))); + if (terminal != null) { + finishedLoadIds.put(loadId, terminal.op()); + return terminal.op(); + } + } + for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) { final File taskDir = new File( LoadStagingDirs.regionLoadDir( @@ -405,16 +474,19 @@ private LoadTsFileConsensusOp getTerminalOp(final String loadId) { continue; } for (final File file : files) { + final LoadTsFileProgress.TerminalRecord terminal; try { - final LoadTsFileProgress.TerminalRecord terminal = LoadTsFileProgress.readTerminal(file); - if (terminal != null) { - return terminal.op(); - } + terminal = LoadTsFileProgress.readTerminal(file); } catch (final IOException e) { LOGGER.warn( StorageEngineMessages.LOG_LOAD_CONSENSUS_RECOVER_TASK_META_FAILED_C39E04BB, file.getPath(), e); + continue; + } + if (terminal != null) { + rememberTerminal(loadId, terminal.op(), terminal.searchIndex()); + return terminal.op(); } } } @@ -425,12 +497,13 @@ public boolean prepare( final LoadTsFileConsensusNode node, final Map timePartitionProgressIndexMap) throws IOException, LoadFileException { + // A terminal decision also fences late pieces and retries from an older consensus batch. + if (node.isGeneratedByRemoteConsensusLeader() && getTerminalOp(node.getLoadId()) != null) { + return true; + } if (!uuid2WriterManager.containsKey(node.getLoadId())) { // A lost PREPARE acknowledgement can be retried after the follower applied COMMIT or ABORT. if (node.isGeneratedByRemoteConsensusLeader()) { - if (getTerminalOp(node.getLoadId()) != null) { - return true; - } throw new LoadFileNotReadyException(); } return false; @@ -467,12 +540,12 @@ public boolean loadAll( final LoadTsFileConsensusNode node, final Map timePartitionProgressIndexMap) throws IOException, LoadFileException { + final LoadTsFileConsensusOp terminalOp = getTerminalOp(node.getLoadId()); + if (terminalOp != null) { + return terminalOp == LoadTsFileConsensusOp.COMMIT; + } if (!uuid2WriterManager.containsKey(node.getLoadId())) { - final LoadTsFileConsensusOp terminalOp = getTerminalOp(node.getLoadId()); - if (terminalOp == LoadTsFileConsensusOp.COMMIT) { - return true; - } - if (terminalOp == null && node.isGeneratedByRemoteConsensusLeader()) { + if (node.isGeneratedByRemoteConsensusLeader()) { throw new LoadFileNotReadyException(); } return false; @@ -489,9 +562,13 @@ public boolean loadAll( final boolean mustRetainStagedFiles = retention.mustRetain(node.getSearchIndex()); uuid2WriterManager .get(node.getLoadId()) - .loadAll(node.isGeneratedByPipe(), !mustRetainStagedFiles); + .loadAll( + node.isGeneratedByPipe(), + !mustRetainStagedFiles && !node.isGeneratedByRemoteConsensusLeader()); + // Keep a follower's staged copy until the terminal answer is durable, even without a local WAL + // watermark to retain it. A restart before this record can then resume the prepared task. + rememberTerminal(node, LoadTsFileConsensusOp.COMMIT); finishConsensusTask(node, mustRetainStagedFiles, LoadTsFileConsensusOp.COMMIT); - finishedLoadIds.put(node.getLoadId(), LoadTsFileConsensusOp.COMMIT); logLoadNodeToWAL(node); dataRegion.insertSeparatorToWAL(node); return true; @@ -499,11 +576,17 @@ public boolean loadAll( public boolean deleteAll(final LoadTsFileConsensusNode node) throws IOException { if (!uuid2WriterManager.containsKey(node.getLoadId())) { + if (getTerminalOp(node.getLoadId()) == null) { + // ABORT may overtake every PIECE. Its success must still settle pending PREPARE retries. + rememberTerminal(node, LoadTsFileConsensusOp.ABORT); + logLoadNodeToWAL(node); + dataRegion.insertSeparatorToWAL(node); + } return false; } + rememberTerminal(node, LoadTsFileConsensusOp.ABORT); finishConsensusTask( node, retention.mustRetain(node.getSearchIndex()), LoadTsFileConsensusOp.ABORT); - finishedLoadIds.put(node.getLoadId(), LoadTsFileConsensusOp.ABORT); logLoadNodeToWAL(node); dataRegion.insertSeparatorToWAL(node); return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java index 64e3e44a524fc..08ded6de6805f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java @@ -107,7 +107,8 @@ public static boolean snapshot(final DataRegion dataRegion, final File snapshotD return true; } - final List taskDirs = manager.getActiveTaskDirs(); + final List taskDirs = new ArrayList<>(manager.getActiveTaskDirs()); + taskDirs.addAll(manager.getTerminalHistoryDirs()); if (taskDirs.isEmpty()) { return true; } @@ -146,14 +147,14 @@ public static boolean snapshot(final DataRegion dataRegion, final File snapshotD private static void copyTask(final File taskDir, final File targetDir) throws IOException { // 1. Snapshot raw staged and modification files first for (final File file : listFiles(taskDir)) { - if (file.isFile() && !isProgressLog(file)) { + if (file.isFile() && !file.getName().contains(TEMP_FILE_SUFFIX) && !isProgressLog(file)) { publishAtomicCopy(file, new File(targetDir, file.getName())); } } // 2. Snapshot progress logs afterwards so they describe an existing snapshot boundary for (final File file : listFiles(taskDir)) { - if (file.isFile() && isProgressLog(file)) { + if (file.isFile() && !file.getName().contains(TEMP_FILE_SUFFIX) && isProgressLog(file)) { publishAtomicCopy(file, new File(targetDir, file.getName())); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java index 933070908bec2..6448d5f2c83d8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.request.IConsensusRequest; import org.apache.iotdb.commons.utils.RetryUtils; +import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.consensus.iot.log.ConsensusReqReader; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -587,6 +588,106 @@ public void testReplicatedPrepareAfterAbortIsANoOp() throws Exception { } } + @Test + public void testReplicatedCommitRetrySurvivesRestartAndSnapshot() throws Exception { + final String loadId = "replicated-committed-retry"; + LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + emulateStagedFileImport(); + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + final LoadTsFileConsensusNode commit = commitNode(loadId); + prepare.markAsGeneratedByRemoteConsensusLeader(); + commit.markAsGeneratedByRemoteConsensusLeader(); + try { + manager.writePiece(loadId, Collections.singletonList(laidOutChunk())); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertTrue(manager.loadAll(commit, Collections.emptyMap())); + assertTrue(manager.getActiveTaskDirs().isEmpty()); + + // The COMMIT response was lost, and the follower restarts before it is retried. + manager.stop(); + manager = new LoadTsFileManager(dataRegion); + assertTrue(manager.loadAll(commit, Collections.emptyMap())); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + + final String originalProtocol = config.getDataRegionConsensusProtocolClass(); + try { + config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager)); + final File snapshot = new File(tempDir, "finished-load-snapshot"); + assertTrue(LoadTsFileSnapshot.snapshot(dataRegion, snapshot)); + manager.stop(); + LoadTsFileSnapshot.clear(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + LoadTsFileSnapshot.restore( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), snapshot); + manager = new LoadTsFileManager(dataRegion); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertTrue(manager.loadAll(commit, Collections.emptyMap())); + } finally { + config.setDataRegionConsensusProtocolClass(originalProtocol); + } + Mockito.verify(dataRegion, Mockito.times(1)) + .loadNewTsFile( + Mockito.any(), + Mockito.anyBoolean(), + Mockito.anyBoolean(), + Mockito.anyBoolean(), + Mockito.any()); + } finally { + manager.stop(); + } + } + + @Test + public void testReplicatedAbortBeforePiecesStopsRetries() throws Exception { + final String loadId = "replicated-abort-before-pieces"; + LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + final LoadTsFileConsensusNode commit = commitNode(loadId); + final LoadTsFileConsensusNode abort = abortNode(loadId); + final LoadTsFileConsensusNode piece = stagedPiece(loadId, laidOutChunk()); + prepare.markAsGeneratedByRemoteConsensusLeader(); + commit.markAsGeneratedByRemoteConsensusLeader(); + abort.markAsGeneratedByRemoteConsensusLeader(); + piece.markAsGeneratedByRemoteConsensusLeader(); + try { + Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap())) + .thenAnswer( + invocation -> + dataRegion + .getLoadTsFileManagerIfPresent() + .get() + .prepare(invocation.getArgument(0), invocation.getArgument(1))); + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager)); + final IoTConsensusDataRegionStateMachine stateMachine = + new IoTConsensusDataRegionStateMachine(dataRegion); + assertTrue( + RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) prepare).getCode())); + + // ABORT can overtake the same missing PIECE as PREPARE did. + assertFalse(manager.deleteAll(abort)); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + stateMachine.write((IConsensusRequest) prepare).getCode()); + manager.writePiece(piece); + assertTrue(manager.getActiveTaskDirs().isEmpty()); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertFalse(manager.loadAll(commit, Collections.emptyMap())); + + manager.stop(); + manager = new LoadTsFileManager(dataRegion); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertFalse(manager.loadAll(commit, Collections.emptyMap())); + manager.writePiece(piece); + assertTrue(manager.getActiveTaskDirs().isEmpty()); + manager.stop(); + LoadTsFileManager.clearTerminalHistory( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + assertTrue(manager.getTerminalHistoryDirs().isEmpty()); + } finally { + manager.stop(); + } + } + /** * A progress file that ends in the middle of an entry is what a snapshot of the staging directory * holds when it was taken while a piece was being appended. The fragment describes nothing, while From 0b872f14f661b3971419481bb6ccfcbb8417ea9d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:00:15 +0800 Subject: [PATCH 04/10] Preserve LOAD terminal decisions during snapshot and recovery --- .../dataregion/snapshot/SnapshotTaker.java | 4 +- .../storageengine/load/LoadTsFileManager.java | 107 ++++++++++------ .../load/LoadTsFileSnapshot.java | 68 ++++++++-- .../load/LoadTsFileManagerTest.java | 119 ++++++++++++++++-- 4 files changed, 240 insertions(+), 58 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java index d32615fec33b6..e0db011fc470a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java @@ -94,6 +94,7 @@ public boolean takeFullSnapshot( boolean success; snapshotLogger.logSnapshotId(finalSnapshotId); + final List loadTerminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion); try { readLockTheFile(); if (flushBeforeSnapshot) { @@ -107,7 +108,8 @@ public boolean takeFullSnapshot( success = createSnapshot(seqFiles, tempSnapshotId); success = success && createSnapshot(unseqFiles, tempSnapshotId); success = success && snapshotCompressionRatio(snapshotDirPath); - success = success && LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir); + success = + success && LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir, loadTerminalHistory); } finally { readUnlockTheFile(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 05b6f8831a7b7..4bc1cbbd1ccfa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -245,10 +245,18 @@ private void recover() { continue; } final String uuid = uuidDir.getName(); - if (retention.recoverTerminalTask(uuid, uuidDir)) { - continue; - } try { + final LoadTsFileProgress.TerminalRecord terminal = readTerminalHistory(uuid); + if (terminal != null) { + // The process may have stopped after publishing the decision but before closing or + // marking staging. That directory belongs to a finished task, never an active writer. + finishedLoadIds.put(uuid, terminal.op()); + retention.retain(uuid, uuidDir, terminal.op(), terminal.searchIndex()); + continue; + } + if (retention.recoverTerminalTask(uuid, uuidDir)) { + continue; + } final TsFileWriterManager recovered = new TsFileWriterManager(dataRegion, uuidDir).recoverFromDisk(); if (recovered.hasPendingTsFiles()) { @@ -392,10 +400,16 @@ public List getActiveTaskDirs() { /** Terminal metadata is snapshotted separately from active tasks and their staged bytes. */ List getTerminalHistoryDirs() { + return getTerminalHistoryDirs(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + } + + static List getTerminalHistoryDirs(final String databaseName, final String dataRegionId) { final List result = new ArrayList<>(); for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) { final File historyDir = - new File(getDataRegionLoadDir(new File(baseDir)), TERMINAL_HISTORY_DIR); + new File( + LoadStagingDirs.regionLoadDir(new File(baseDir), databaseName, dataRegionId), + TERMINAL_HISTORY_DIR); if (historyDir.isDirectory()) { result.add(historyDir); } @@ -421,46 +435,66 @@ private void rememberTerminal(final LoadTsFileConsensusNode node, final LoadTsFi private void rememberTerminal( final String loadId, final LoadTsFileConsensusOp op, final long searchIndex) throws IOException { - final File historyDir = - new File( - getDataRegionLoadDir(new File(LoadStagingDirs.configuredBaseDirs()[0])), - TERMINAL_HISTORY_DIR); - final LoadTsFileProgress terminal = - new LoadTsFileProgress(new File(historyDir, loadId + ".copying." + UUID.randomUUID())); - final File temporaryRecord = terminal.getProgressFile(); - final File record = LoadTsFileProgress.progressFileFor(new File(historyDir, loadId)); + // Region deletion clears history after stop(), so no terminal publication may outlive it. + final Lock readLock = writersLock.readLock(); + readLock.lock(); try { - terminal.recordTerminal(op, searchIndex); + if (stopped) { + throw newStoppedIOException(); + } + final File historyDir = + new File( + getDataRegionLoadDir(new File(LoadStagingDirs.configuredBaseDirs()[0])), + TERMINAL_HISTORY_DIR); + final LoadTsFileProgress terminal = + new LoadTsFileProgress(new File(historyDir, loadId + ".copying." + UUID.randomUUID())); + final File temporaryRecord = terminal.getProgressFile(); + final File record = LoadTsFileProgress.progressFileFor(new File(historyDir, loadId)); try { - Files.move( - temporaryRecord.toPath(), - record.toPath(), - StandardCopyOption.ATOMIC_MOVE, - StandardCopyOption.REPLACE_EXISTING); - } catch (final AtomicMoveNotSupportedException e) { - Files.move(temporaryRecord.toPath(), record.toPath(), StandardCopyOption.REPLACE_EXISTING); + terminal.recordTerminal(op, searchIndex); + try { + Files.move( + temporaryRecord.toPath(), + record.toPath(), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (final AtomicMoveNotSupportedException e) { + Files.move( + temporaryRecord.toPath(), record.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } finally { + Files.deleteIfExists(temporaryRecord.toPath()); } + finishedLoadIds.put(loadId, op); } finally { - Files.deleteIfExists(temporaryRecord.toPath()); + readLock.unlock(); } - finishedLoadIds.put(loadId, op); } - /** The terminal command this region already applied to the task, or null if it is unknown. */ - private LoadTsFileConsensusOp getTerminalOp(final String loadId) throws IOException { - final LoadTsFileConsensusOp finishedOp = finishedLoadIds.get(loadId); - if (finishedOp != null) { - return finishedOp; - } + private LoadTsFileProgress.TerminalRecord readTerminalHistory(final String loadId) + throws IOException { for (final File historyDir : getTerminalHistoryDirs()) { final LoadTsFileProgress.TerminalRecord terminal = LoadTsFileProgress.readTerminal( LoadTsFileProgress.progressFileFor(new File(historyDir, loadId))); if (terminal != null) { - finishedLoadIds.put(loadId, terminal.op()); - return terminal.op(); + return terminal; } } + return null; + } + + /** The terminal command this region already applied to the task, or null if it is unknown. */ + private LoadTsFileConsensusOp getTerminalOp(final String loadId) throws IOException { + final LoadTsFileConsensusOp finishedOp = finishedLoadIds.get(loadId); + if (finishedOp != null) { + return finishedOp; + } + final LoadTsFileProgress.TerminalRecord history = readTerminalHistory(loadId); + if (history != null) { + finishedLoadIds.put(loadId, history.op()); + return history.op(); + } for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) { final File taskDir = new File( @@ -575,13 +609,14 @@ public boolean loadAll( } public boolean deleteAll(final LoadTsFileConsensusNode node) throws IOException { + if (getTerminalOp(node.getLoadId()) != null) { + return false; + } if (!uuid2WriterManager.containsKey(node.getLoadId())) { - if (getTerminalOp(node.getLoadId()) == null) { - // ABORT may overtake every PIECE. Its success must still settle pending PREPARE retries. - rememberTerminal(node, LoadTsFileConsensusOp.ABORT); - logLoadNodeToWAL(node); - dataRegion.insertSeparatorToWAL(node); - } + // ABORT may overtake every PIECE. Its success must still settle pending PREPARE retries. + rememberTerminal(node, LoadTsFileConsensusOp.ABORT); + logLoadNodeToWAL(node); + dataRegion.insertSeparatorToWAL(node); return false; } rememberTerminal(node, LoadTsFileConsensusOp.ABORT); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java index 08ded6de6805f..c0a23843c0306 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java @@ -89,37 +89,73 @@ private LoadTsFileSnapshot() {} // Snapshot Entrypoint // ------------------------------------------------------------------------- - /** Captures an atomic snapshot of active staged tasks under the DataRegion. */ + /** Captures the LOAD state when no data-file snapshot boundary needs to be supplied. */ public static boolean snapshot(final DataRegion dataRegion, final File snapshotDir) { + return snapshot(dataRegion, snapshotDir, null); + } + + /** + * Selects terminal records before the data-file snapshot captures its resource lists. A COMMIT + * published afterwards may describe data absent from those lists, so replay must not be fenced by + * that decision. Records are immutable and can be copied after the data files. + */ + public static List captureTerminalHistory(final DataRegion dataRegion) throws IOException { + Objects.requireNonNull( + dataRegion, StorageEngineMessages.EXCEPTION_DATAREGION_CANNOT_BE_NULL_0B936879); + final List result = new ArrayList<>(); + if (isSupportedProtocol()) { + // A restarted region need not have initialized its lazy LOAD manager yet. + for (final File historyDir : + LoadTsFileManager.getTerminalHistoryDirs( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString())) { + for (final File file : listFiles(historyDir)) { + if (file.isFile() && isProgressLog(file) && !file.getName().contains(TEMP_FILE_SUFFIX)) { + result.add(file); + } + } + } + } + return result; + } + + /** Captures active staging and only the terminal decisions covered by the data-file snapshot. */ + public static boolean snapshot( + final DataRegion dataRegion, + final File snapshotDir, + final List capturedTerminalHistory) { Objects.requireNonNull( dataRegion, StorageEngineMessages.EXCEPTION_DATAREGION_CANNOT_BE_NULL_0B936879); Objects.requireNonNull( snapshotDir, StorageEngineMessages.EXCEPTION_SNAPSHOTDIR_CANNOT_BE_NULL_283211CB); - if (!SUPPORTED_PROTOCOLS.contains( - ConsensusFactory.normalizeConsensusProtocolClass( - IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass()))) { + if (!isSupportedProtocol()) { return true; } final LoadTsFileManager manager = dataRegion.getLoadTsFileManagerIfPresent().orElse(null); - if (manager == null) { - return true; - } - - final List taskDirs = new ArrayList<>(manager.getActiveTaskDirs()); - taskDirs.addAll(manager.getTerminalHistoryDirs()); - if (taskDirs.isEmpty()) { - return true; - } + final List taskDirs = + manager == null ? Collections.emptyList() : manager.getActiveTaskDirs(); final File loadSnapshotDir = new File(snapshotDir, SNAPSHOT_SUBDIR_NAME); try { + final List terminalHistory = + capturedTerminalHistory == null + ? captureTerminalHistory(dataRegion) + : capturedTerminalHistory; + if (taskDirs.isEmpty() && terminalHistory.isEmpty()) { + return true; + } writeRootsManifest(loadSnapshotDir, taskDirs); for (final File taskDir : taskDirs) { final File targetDir = new File(loadSnapshotDir, taskDir.getName()); copyTask(taskDir, targetDir); } + // Terminal records have no payload references and may share any restored staging root. + final File historyTargetDir = + new File(loadSnapshotDir, LoadTsFileManager.TERMINAL_HISTORY_DIR); + for (final File terminal : terminalHistory) { + publishAtomicCopy(terminal, new File(historyTargetDir, terminal.getName())); + } LOGGER.info( String.format( @@ -137,6 +173,12 @@ public static boolean snapshot(final DataRegion dataRegion, final File snapshotD } } + private static boolean isSupportedProtocol() { + return SUPPORTED_PROTOCOLS.contains( + ConsensusFactory.normalizeConsensusProtocolClass( + IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass())); + } + // ------------------------------------------------------------------------- // Task Copy Pipeline & Live Catch-Up // ------------------------------------------------------------------------- diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java index 6448d5f2c83d8..5ba11443062ac 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java @@ -38,6 +38,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusOp; import org.apache.iotdb.db.queryengine.plan.scheduler.load.ChunkOffsetCalculator; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.apache.iotdb.db.storageengine.dataregion.snapshot.SnapshotTaker; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.wal.node.IWALNode; @@ -49,6 +51,7 @@ import org.apache.iotdb.db.storageengine.load.splitter.ChunkPayloadUnavailableException; import org.apache.iotdb.db.storageengine.load.splitter.NonAlignedChunkData; import org.apache.iotdb.db.storageengine.load.splitter.TsFileData; +import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; @@ -123,6 +126,8 @@ public class LoadTsFileManagerTest { public void setUp() throws Exception { tempDir = Files.createTempDirectory("load-tsfile-manager-test").toFile(); config = IoTDBDescriptor.getInstance().getConfig(); + // Startup resolves LOAD roots once. Do that before installing this test's isolated roots. + TierManager.getInstance(); originalLoadBaseDirs = config.getLoadTsFileDirs(); config.setLoadTsFileDirs(new String[] {tempDir.getAbsolutePath()}); @@ -251,12 +256,6 @@ public void testOutOfOrderPiecesSurviveTheLossOfTheInMemoryWriters() throws Exce // progress records stays on disk. dropInMemoryWriters(firstRun); - // The first lookup of a staging directory makes the tier manager reformulate the configured - // LOAD directories from the data directories, which in a test is not the directory the pieces - // were just staged in. A DataNode does that once at startup, before any load, so pinning the - // directory here is what keeps the restarted manager looking at the very same staged file. - config.setLoadTsFileDirs(new String[] {tempDir.getAbsolutePath()}); - // A new manager resumes the very same staged directory from those records. final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); @@ -612,10 +611,11 @@ public void testReplicatedCommitRetrySurvivesRestartAndSnapshot() throws Excepti final String originalProtocol = config.getDataRegionConsensusProtocolClass(); try { config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); - Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager)); + // After another restart no LOAD has initialized this region's lazy manager yet. + manager.stop(); + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.empty()); final File snapshot = new File(tempDir, "finished-load-snapshot"); assertTrue(LoadTsFileSnapshot.snapshot(dataRegion, snapshot)); - manager.stop(); LoadTsFileSnapshot.clear(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); LoadTsFileSnapshot.restore( dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), snapshot); @@ -637,6 +637,99 @@ public void testReplicatedCommitRetrySurvivesRestartAndSnapshot() throws Excepti } } + @Test + public void testSnapshotDoesNotFenceALoadCommittedAfterDataFileCapture() throws Exception { + final String loadId = "commit-during-snapshot"; + final String originalProtocol = config.getDataRegionConsensusProtocolClass(); + final LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + try { + config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager)); + emulateStagedFileImport(); + manager.writePiece(loadId, Collections.singletonList(laidOutChunk())); + assertTrue(manager.prepare(prepareNode(loadId), Collections.emptyMap())); + + final TsFileManager tsFileManager = Mockito.mock(TsFileManager.class); + Mockito.when(dataRegion.getTsFileManager()).thenReturn(tsFileManager); + Mockito.when(tsFileManager.getTsFileList(Mockito.anyBoolean())) + .thenAnswer(invocation -> new ArrayList<>()); + Mockito.doAnswer( + invocation -> { + // SnapshotTaker already captured its data-file lists before flushing. + assertTrue(manager.loadAll(commitNode(loadId), Collections.emptyMap())); + return null; + }) + .when(dataRegion) + .syncCloseAllWorkingTsFileProcessors(); + + final File snapshot = new File(tempDir, "concurrent-commit-snapshot"); + assertTrue(new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true)); + final File history = + new File( + new File(snapshot, LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME), + LoadTsFileManager.TERMINAL_HISTORY_DIR); + assertFalse(LoadTsFileProgress.progressFileFor(new File(history, loadId)).exists()); + + // The data-file snapshot predates COMMIT. Its replay must still stage the missing data. + manager.stop(); + LoadTsFileSnapshot.clear(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + LoadTsFileSnapshot.restore( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), snapshot); + final LoadTsFileManager restored = new LoadTsFileManager(dataRegion); + try { + final LoadTsFileConsensusNode piece = stagedPiece(loadId, laidOutChunk()); + piece.markAsGeneratedByRemoteConsensusLeader(); + restored.writePiece(piece); + assertEquals(1, restored.getActiveTaskDirs().size()); + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + prepare.markAsGeneratedByRemoteConsensusLeader(); + assertTrue(restored.prepare(prepare, Collections.emptyMap())); + } finally { + restored.stop(); + } + } finally { + manager.stop(); + config.setDataRegionConsensusProtocolClass(originalProtocol); + } + } + + @Test + public void testAbortCannotReplaceACommittedDecisionBeforeStagingCleanup() throws Exception { + final String loadId = "commit-before-staging-cleanup"; + final LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + try { + emulateStagedFileImport(); + manager.writePiece(loadId, Collections.singletonList(laidOutChunk())); + assertTrue(manager.prepare(prepareNode(loadId), Collections.emptyMap())); + final TsFileWriterManager writer = writerOf(manager, loadId); + writer.loadAll(false, false); + final File historyDir = + new File(writer.getTaskDir().getParentFile(), LoadTsFileManager.TERMINAL_HISTORY_DIR); + new LoadTsFileProgress(new File(historyDir, loadId)) + .recordTerminal(LoadTsFileConsensusOp.COMMIT, 10L); + // A crash after the durable decision leaves staging without its per-file terminal tails. + writer.close(true); + + final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); + try { + assertFalse(recovered.deleteAll(abortNode(loadId))); + assertTrue(recovered.loadAll(commitNode(loadId), Collections.emptyMap())); + assertTrue(recovered.getActiveTaskDirs().isEmpty()); + Mockito.verify(dataRegion, Mockito.times(1)) + .loadNewTsFile( + Mockito.any(), + Mockito.anyBoolean(), + Mockito.anyBoolean(), + Mockito.anyBoolean(), + Mockito.any()); + } finally { + recovered.stop(); + } + } finally { + manager.stop(); + } + } + @Test public void testReplicatedAbortBeforePiecesStopsRetries() throws Exception { final String loadId = "replicated-abort-before-pieces"; @@ -1299,6 +1392,16 @@ public void testStopIsTerminalForWrites() throws Exception { } catch (final IOException e) { assertTrue(e.getMessage().contains("has been stopped")); } + LoadTsFileManager.clearTerminalHistory( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + try { + // A command that already obtained the manager must not recreate a deleted region's history. + manager.deleteAll(abortNode("abort-after-region-deletion")); + fail("A stopped LOAD manager has to refuse terminal publication"); + } catch (final IOException e) { + assertTrue(e.getMessage().contains("has been stopped")); + } + assertTrue(manager.getTerminalHistoryDirs().isEmpty()); } /** A stop waits for the piece in flight, and still closes the writer that piece created. */ From e2a11bdc021a6f013e17a4fb0f2227d0f8b2737d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:35:39 +0800 Subject: [PATCH 05/10] Keep LOAD snapshot state consistent across commits --- .../dataregion/snapshot/SnapshotTaker.java | 28 ++++- .../storageengine/load/LoadTsFileManager.java | 103 +++++++++++------ .../load/LoadTsFileManagerTest.java | 105 ++++++++++++++++++ 3 files changed, 196 insertions(+), 40 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java index e0db011fc470a..50f9aebbfaab3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.load.LoadTsFileManager; import org.apache.iotdb.db.storageengine.load.LoadTsFileSnapshot; import org.slf4j.Logger; @@ -94,15 +95,30 @@ public boolean takeFullSnapshot( boolean success; snapshotLogger.logSnapshotId(finalSnapshotId); - final List loadTerminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion); + final List loadTerminalHistory; try { - readLockTheFile(); - if (flushBeforeSnapshot) { + synchronized (dataRegion) { + // Prevent lazy manager creation while selecting the LOAD and data snapshot boundary. + final LoadTsFileManager loadManager = + dataRegion.getLoadTsFileManagerIfPresent().orElse(null); + if (loadManager != null) { + loadManager.lockSnapshotBoundary(); + } try { - dataRegion.writeLock("snapshotTaker"); - dataRegion.syncCloseAllWorkingTsFileProcessors(); + loadTerminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion); + readLockTheFile(); + if (flushBeforeSnapshot) { + dataRegion.writeLock("snapshotTaker"); + try { + dataRegion.syncCloseAllWorkingTsFileProcessors(); + } finally { + dataRegion.writeUnlock(); + } + } } finally { - dataRegion.writeUnlock(); + if (loadManager != null) { + loadManager.unlockSnapshotBoundary(); + } } } success = createSnapshot(seqFiles, tempSnapshotId); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 4bc1cbbd1ccfa..7d67ef8b78a5e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -129,6 +129,9 @@ protected boolean removeEldestEntry( /** Pieces read, stop() and closing a writer write. Fair, so a stop is not starved. */ private final ReentrantReadWriteLock writersLock = new ReentrantReadWriteLock(true); + /** Fences COMMIT/ABORT publication against the data and LOAD-state snapshot boundary. */ + private final ReentrantReadWriteLock snapshotBoundaryLock = new ReentrantReadWriteLock(true); + /** Set under the write side, read as the fast path of a piece. */ private volatile boolean stopped = false; @@ -255,6 +258,13 @@ private void recover() { continue; } if (retention.recoverTerminalTask(uuid, uuidDir)) { + // Older versions kept the decision only in each staged file's progress tail. + final LoadTsFileProgress.TerminalRecord retainedTerminal = + LoadTsFileProgress.readTerminalOfTaskDir(uuidDir); + if (retainedTerminal != null) { + rememberTerminal(uuid, retainedTerminal.op(), retainedTerminal.searchIndex()); + finishedLoadIds.put(uuid, retainedTerminal.op()); + } continue; } final TsFileWriterManager recovered = @@ -398,6 +408,15 @@ public List getActiveTaskDirs() { return result; } + /** Holds the write side while a region snapshot selects its data and LOAD state. */ + public void lockSnapshotBoundary() { + snapshotBoundaryLock.writeLock().lock(); + } + + public void unlockSnapshotBoundary() { + snapshotBoundaryLock.writeLock().unlock(); + } + /** Terminal metadata is snapshotted separately from active tasks and their staged bytes. */ List getTerminalHistoryDirs() { return getTerminalHistoryDirs(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); @@ -574,54 +593,70 @@ public boolean loadAll( final LoadTsFileConsensusNode node, final Map timePartitionProgressIndexMap) throws IOException, LoadFileException { - final LoadTsFileConsensusOp terminalOp = getTerminalOp(node.getLoadId()); - if (terminalOp != null) { - return terminalOp == LoadTsFileConsensusOp.COMMIT; - } - if (!uuid2WriterManager.containsKey(node.getLoadId())) { - if (node.isGeneratedByRemoteConsensusLeader()) { - throw new LoadFileNotReadyException(); + final Lock snapshotReadLock = snapshotBoundaryLock.readLock(); + snapshotReadLock.lock(); + final boolean mustRetainStagedFiles; + try { + final LoadTsFileConsensusOp terminalOp = getTerminalOp(node.getLoadId()); + if (terminalOp != null) { + return terminalOp == LoadTsFileConsensusOp.COMMIT; } - return false; + if (!uuid2WriterManager.containsKey(node.getLoadId())) { + if (node.isGeneratedByRemoteConsensusLeader()) { + throw new LoadFileNotReadyException(); + } + return false; + } + LOGGER.info( + StorageEngineMessages + .LOG_COMMITTING_LOAD_TSFILE_ARG_LOADING_PREPARED_RESOURCES_INTO_DATAREGION_EA1D6335, + node.getLoadId()); + // Whether the staged bytes are still needed has to be decided before they are imported: the + // WAL entries of the pieces of this task only reference those bytes, and they are expanded on + // this node for a replica that has not applied them yet. Importing them by moving the staged + // file away would leave those entries unexpandable, so a copy is imported while the watermark + // of the WAL has not got past this command yet. + mustRetainStagedFiles = retention.mustRetain(node.getSearchIndex()); + uuid2WriterManager + .get(node.getLoadId()) + .loadAll( + node.isGeneratedByPipe(), + !mustRetainStagedFiles && !node.isGeneratedByRemoteConsensusLeader()); + // Keep a follower's staged copy until the terminal answer is durable, even without a local + // WAL watermark to retain it. A restart before this record can then resume the prepared task. + rememberTerminal(node, LoadTsFileConsensusOp.COMMIT); + finishConsensusTask(node, mustRetainStagedFiles, LoadTsFileConsensusOp.COMMIT); + } finally { + snapshotReadLock.unlock(); } - LOGGER.info( - StorageEngineMessages - .LOG_COMMITTING_LOAD_TSFILE_ARG_LOADING_PREPARED_RESOURCES_INTO_DATAREGION_EA1D6335, - node.getLoadId()); - // Whether the staged bytes are still needed has to be decided before they are imported: the WAL - // entries of the pieces of this task only reference those bytes, and they are expanded on this - // node for a replica that has not applied them yet. Importing them by moving the staged file - // away would leave those entries unexpandable, so a copy is imported while the watermark of the - // WAL has not got past this command yet. - final boolean mustRetainStagedFiles = retention.mustRetain(node.getSearchIndex()); - uuid2WriterManager - .get(node.getLoadId()) - .loadAll( - node.isGeneratedByPipe(), - !mustRetainStagedFiles && !node.isGeneratedByRemoteConsensusLeader()); - // Keep a follower's staged copy until the terminal answer is durable, even without a local WAL - // watermark to retain it. A restart before this record can then resume the prepared task. - rememberTerminal(node, LoadTsFileConsensusOp.COMMIT); - finishConsensusTask(node, mustRetainStagedFiles, LoadTsFileConsensusOp.COMMIT); logLoadNodeToWAL(node); dataRegion.insertSeparatorToWAL(node); return true; } public boolean deleteAll(final LoadTsFileConsensusNode node) throws IOException { - if (getTerminalOp(node.getLoadId()) != null) { - return false; - } - if (!uuid2WriterManager.containsKey(node.getLoadId())) { + final Lock snapshotReadLock = snapshotBoundaryLock.readLock(); + snapshotReadLock.lock(); + final boolean hasWriter; + try { + if (getTerminalOp(node.getLoadId()) != null) { + return false; + } + hasWriter = uuid2WriterManager.containsKey(node.getLoadId()); // ABORT may overtake every PIECE. Its success must still settle pending PREPARE retries. rememberTerminal(node, LoadTsFileConsensusOp.ABORT); + if (hasWriter) { + finishConsensusTask( + node, retention.mustRetain(node.getSearchIndex()), LoadTsFileConsensusOp.ABORT); + } + } finally { + snapshotReadLock.unlock(); + } + if (!hasWriter) { logLoadNodeToWAL(node); dataRegion.insertSeparatorToWAL(node); return false; } - rememberTerminal(node, LoadTsFileConsensusOp.ABORT); - finishConsensusTask( - node, retention.mustRetain(node.getSearchIndex()), LoadTsFileConsensusOp.ABORT); logLoadNodeToWAL(node); dataRegion.insertSeparatorToWAL(node); return true; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java index 5ba11443062ac..9b84d0dffce3e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java @@ -89,6 +89,7 @@ import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; @@ -730,6 +731,110 @@ public void testAbortCannotReplaceACommittedDecisionBeforeStagingCleanup() throw } } + @Test + public void testRecoveryMigratesLegacyTerminalRecordIntoSnapshotHistory() throws Exception { + final String originalProtocol = config.getDataRegionConsensusProtocolClass(); + config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + final String loadId = "legacy-terminal-load"; + try { + final LoadTsFileManager beforeUpgrade = new LoadTsFileManager(dataRegion); + try { + beforeUpgrade.writePiece(loadId, Collections.singletonList(laidOutChunk())); + assertTrue(beforeUpgrade.prepare(prepareNode(loadId), Collections.emptyMap())); + final TsFileWriterManager writer = writerOf(beforeUpgrade, loadId); + writer.recordTerminal(LoadTsFileConsensusOp.COMMIT, 10L, Collections.emptyMap()); + writer.close(true); + + final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); + try { + final List terminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion); + assertEquals(1, terminalHistory.size()); + assertEquals( + loadId + LoadTsFileProgress.PROGRESS_SUFFIX, terminalHistory.get(0).getName()); + } finally { + recovered.stop(); + } + } finally { + beforeUpgrade.stop(); + } + } finally { + config.setDataRegionConsensusProtocolClass(originalProtocol); + } + } + + @Test + public void testCommitCannotSplitSnapshotDataAndTerminalBoundary() throws Exception { + final String originalProtocol = config.getDataRegionConsensusProtocolClass(); + config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + final String loadId = "commit-at-snapshot-boundary"; + try { + final LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + final AtomicBoolean commitStarted = new AtomicBoolean(); + final CountDownLatch commitFinished = new CountDownLatch(1); + final AtomicReference commitFailure = new AtomicReference<>(); + final AtomicReference commitThread = new AtomicReference<>(); + try { + final LoadTsFileConsensusNode commit = commitNode(loadId); + commit.markAsGeneratedByRemoteConsensusLeader(); + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager)); + emulateStagedFileImport(); + manager.writePiece(loadId, Collections.singletonList(laidOutChunk())); + assertTrue(manager.prepare(prepareNode(loadId), Collections.emptyMap())); + + final TsFileManager tsFileManager = Mockito.mock(TsFileManager.class); + Mockito.when(tsFileManager.getTsFileList(Mockito.anyBoolean())) + .thenAnswer(invocation -> new ArrayList<>()); + Mockito.when(dataRegion.getTsFileManager()) + .thenAnswer( + invocation -> { + if (commitStarted.compareAndSet(false, true)) { + final Thread thread = + new Thread( + () -> { + try { + assertTrue(manager.loadAll(commit, Collections.emptyMap())); + } catch (final Throwable t) { + commitFailure.set(t); + } finally { + commitFinished.countDown(); + } + }); + commitThread.set(thread); + thread.start(); + assertFalse( + "COMMIT must wait for snapshot boundary capture", + commitFinished.await(1, TimeUnit.SECONDS)); + } + return tsFileManager; + }); + + final File snapshot = new File(tempDir, "commit-boundary-snapshot"); + assertTrue( + new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true)); + assertTrue( + "COMMIT must continue after snapshot boundary capture", + commitFinished.await(10, TimeUnit.SECONDS)); + commitThread.get().join(10_000L); + assertFalse(commitThread.get().isAlive()); + assertNull(commitFailure.get()); + + final File history = + new File( + new File(snapshot, LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME), + LoadTsFileManager.TERMINAL_HISTORY_DIR); + assertFalse(LoadTsFileProgress.progressFileFor(new File(history, loadId)).exists()); + } finally { + final Thread thread = commitThread.get(); + if (thread != null) { + thread.join(10_000L); + } + manager.stop(); + } + } finally { + config.setDataRegionConsensusProtocolClass(originalProtocol); + } + } + @Test public void testReplicatedAbortBeforePiecesStopsRetries() throws Exception { final String loadId = "replicated-abort-before-pieces"; From 9a4701faf79e7a4f2949a5a1c76b998d845f35b4 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:00:26 +0800 Subject: [PATCH 06/10] Preserve LOAD staging and legacy decisions through snapshots --- .../dataregion/snapshot/SnapshotTaker.java | 7 ++- .../storageengine/load/LoadTsFileManager.java | 15 +++-- .../load/LoadTsFileManagerTest.java | 61 ++++++++++++++----- 3 files changed, 58 insertions(+), 25 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java index 50f9aebbfaab3..b2397c0088bb8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java @@ -115,17 +115,18 @@ public boolean takeFullSnapshot( dataRegion.writeUnlock(); } } + // A COMMIT after data-file selection must not remove staging before it is copied. + // Its pieces may predate the new replica's sync channel and never be replayed there. + success = LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir, loadTerminalHistory); } finally { if (loadManager != null) { loadManager.unlockSnapshotBoundary(); } } } - success = createSnapshot(seqFiles, tempSnapshotId); + success = success && createSnapshot(seqFiles, tempSnapshotId); success = success && createSnapshot(unseqFiles, tempSnapshotId); success = success && snapshotCompressionRatio(snapshotDirPath); - success = - success && LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir, loadTerminalHistory); } finally { readUnlockTheFile(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 7d67ef8b78a5e..43a1ee954e284 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -129,7 +129,7 @@ protected boolean removeEldestEntry( /** Pieces read, stop() and closing a writer write. Fair, so a stop is not starved. */ private final ReentrantReadWriteLock writersLock = new ReentrantReadWriteLock(true); - /** Fences COMMIT/ABORT publication against the data and LOAD-state snapshot boundary. */ + /** Fences COMMIT/ABORT against selection and copying of a data and LOAD-state snapshot. */ private final ReentrantReadWriteLock snapshotBoundaryLock = new ReentrantReadWriteLock(true); /** Set under the write side, read as the fast path of a piece. */ @@ -257,14 +257,13 @@ private void recover() { retention.retain(uuid, uuidDir, terminal.op(), terminal.searchIndex()); continue; } - if (retention.recoverTerminalTask(uuid, uuidDir)) { + final LoadTsFileProgress.TerminalRecord retainedTerminal = + LoadTsFileProgress.readTerminalOfTaskDir(uuidDir); + if (retainedTerminal != null) { // Older versions kept the decision only in each staged file's progress tail. - final LoadTsFileProgress.TerminalRecord retainedTerminal = - LoadTsFileProgress.readTerminalOfTaskDir(uuidDir); - if (retainedTerminal != null) { - rememberTerminal(uuid, retainedTerminal.op(), retainedTerminal.searchIndex()); - finishedLoadIds.put(uuid, retainedTerminal.op()); - } + // Migrate it before registering a listener that may release the only staged copy. + rememberTerminal(uuid, retainedTerminal.op(), retainedTerminal.searchIndex()); + retention.retain(uuid, uuidDir, retainedTerminal.op(), retainedTerminal.searchIndex()); continue; } final TsFileWriterManager recovered = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java index 9b84d0dffce3e..f4af3318fe629 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java @@ -95,6 +95,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.LongConsumer; +import java.util.function.Predicate; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -639,7 +640,7 @@ public void testReplicatedCommitRetrySurvivesRestartAndSnapshot() throws Excepti } @Test - public void testSnapshotDoesNotFenceALoadCommittedAfterDataFileCapture() throws Exception { + public void testSnapshotRetainsStagingForLoadCommittedAfterDataFileCapture() throws Exception { final String loadId = "commit-during-snapshot"; final String originalProtocol = config.getDataRegionConsensusProtocolClass(); final LoadTsFileManager manager = new LoadTsFileManager(dataRegion); @@ -652,39 +653,55 @@ public void testSnapshotDoesNotFenceALoadCommittedAfterDataFileCapture() throws final TsFileManager tsFileManager = Mockito.mock(TsFileManager.class); Mockito.when(dataRegion.getTsFileManager()).thenReturn(tsFileManager); + final AtomicBoolean committed = new AtomicBoolean(); Mockito.when(tsFileManager.getTsFileList(Mockito.anyBoolean())) - .thenAnswer(invocation -> new ArrayList<>()); - Mockito.doAnswer( - invocation -> { - // SnapshotTaker already captured its data-file lists before flushing. - assertTrue(manager.loadAll(commitNode(loadId), Collections.emptyMap())); - return null; - }) - .when(dataRegion) - .syncCloseAllWorkingTsFileProcessors(); + .thenAnswer( + invocation -> + new ArrayList() { + @Override + public boolean removeIf(final Predicate filter) { + // Data-file copying starts after the snapshot boundary lock is released. + if (committed.compareAndSet(false, true)) { + try { + assertTrue(manager.loadAll(commitNode(loadId), Collections.emptyMap())); + } catch (final Exception e) { + throw new AssertionError(e); + } + } + return super.removeIf(filter); + } + }); final File snapshot = new File(tempDir, "concurrent-commit-snapshot"); assertTrue(new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true)); + assertTrue(committed.get()); final File history = new File( new File(snapshot, LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME), LoadTsFileManager.TERMINAL_HISTORY_DIR); assertFalse(LoadTsFileProgress.progressFileFor(new File(history, loadId)).exists()); - // The data-file snapshot predates COMMIT. Its replay must still stage the missing data. + // PIECE may predate the new replica's sync channel, so it cannot be replayed after restore. + // The snapshot itself must retain the staging that its later COMMIT still needs. manager.stop(); LoadTsFileSnapshot.clear(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); LoadTsFileSnapshot.restore( dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), snapshot); final LoadTsFileManager restored = new LoadTsFileManager(dataRegion); try { - final LoadTsFileConsensusNode piece = stagedPiece(loadId, laidOutChunk()); - piece.markAsGeneratedByRemoteConsensusLeader(); - restored.writePiece(piece); assertEquals(1, restored.getActiveTaskDirs().size()); final LoadTsFileConsensusNode prepare = prepareNode(loadId); prepare.markAsGeneratedByRemoteConsensusLeader(); assertTrue(restored.prepare(prepare, Collections.emptyMap())); + final File[] stagedFiles = + restored.getActiveTaskDirs().get(0).listFiles((dir, name) -> name.endsWith(".tsfile")); + assertNotNull(stagedFiles); + assertEquals(1, stagedFiles.length); + assertAllSeriesReadable( + stagedFiles[0], + Collections.singletonList( + new SeriesExpectation( + new StringArrayDeviceID("root", "load_manager_test", "d0"), "s0", 10, 0, 9))); } finally { restored.stop(); } @@ -737,6 +754,7 @@ public void testRecoveryMigratesLegacyTerminalRecordIntoSnapshotHistory() throws config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); final String loadId = "legacy-terminal-load"; try { + useWatermarkWALNode(5L); final LoadTsFileManager beforeUpgrade = new LoadTsFileManager(dataRegion); try { beforeUpgrade.writePiece(loadId, Collections.singletonList(laidOutChunk())); @@ -745,12 +763,27 @@ public void testRecoveryMigratesLegacyTerminalRecordIntoSnapshotHistory() throws writer.recordTerminal(LoadTsFileConsensusOp.COMMIT, 10L, Collections.emptyMap()); writer.close(true); + final IWALNode walNode = dataRegion.getWALNode().get(); + Mockito.doAnswer( + invocation -> { + // A watermark can advance as soon as recovery registers its release listener. + final LongConsumer listener = invocation.getArgument(0); + walWatermark.set(10L); + listener.accept(10L); + return null; + }) + .when(walNode) + .setSafeDeletedSearchIndexListener(Mockito.any()); final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); try { + assertFalse(writer.getTaskDir().exists()); final List terminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion); assertEquals(1, terminalHistory.size()); assertEquals( loadId + LoadTsFileProgress.PROGRESS_SUFFIX, terminalHistory.get(0).getName()); + final LoadTsFileConsensusNode commit = commitNode(loadId); + commit.markAsGeneratedByRemoteConsensusLeader(); + assertTrue(recovered.loadAll(commit, Collections.emptyMap())); } finally { recovered.stop(); } From c634776f157151c63022d249d854a6adb2dbd718 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:19:40 +0800 Subject: [PATCH 07/10] Protect LOAD retry metadata during cleanup and deletion --- .../storageengine/dataregion/DataRegion.java | 39 ++++---- .../storageengine/load/LoadTsFileCleaner.java | 93 +++++++++++-------- .../storageengine/load/LoadTsFileManager.java | 20 ++++ .../dataregion/DataRegionTest.java | 16 ++++ .../load/LoadTsFileCleanerTest.java | 87 +++++++++++++++++ 5 files changed, 199 insertions(+), 56 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index e8e7ea8ef66fe..270927aeb1655 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -570,28 +570,29 @@ public String getDatabaseName() { } public LoadTsFileManager getLoadTsFileManager() { - LoadTsFileManager manager = loadTsFileManager; - if (manager == null) { - synchronized (this) { - manager = loadTsFileManager; - if (manager == null) { - manager = new LoadTsFileManager(this); - loadTsFileManager = manager; + insertLock.readLock().lock(); + try { + if (deleted) { + throw new IllegalStateException( + String.format( + StorageEngineMessages + .STORAGE_EXCEPTION_REGION_ARG_IS_DELETED_SO_IT_TAKES_NO_LOAD_WORK_6E05108F, + databaseName + "-" + dataRegionIdString)); + } + LoadTsFileManager manager = loadTsFileManager; + if (manager == null) { + synchronized (this) { + manager = loadTsFileManager; + if (manager == null) { + manager = new LoadTsFileManager(this); + loadTsFileManager = manager; + } } } + return manager; + } finally { + insertLock.readLock().unlock(); } - if (deleted) { - // The region went away: it takes no LOAD work, and the manager is stopped for good. A manager - // created in this window stops itself here, the one stopLoadTsFileManager saw was stopped by - // it. - manager.stop(); - throw new IllegalStateException( - String.format( - StorageEngineMessages - .STORAGE_EXCEPTION_REGION_ARG_IS_DELETED_SO_IT_TAKES_NO_LOAD_WORK_6E05108F, - databaseName + "-" + dataRegionIdString)); - } - return manager; } public Optional getLoadTsFileManagerIfPresent() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java index 8df83a79ada0e..7fbfebe6a7d6d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java @@ -173,47 +173,66 @@ private void scanRegions() { new File(baseDir), dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + scanRegion(dataRegion, regionDir); + } + } + } + + private void scanRegion(final DataRegion dataRegion, final File regionDir) { + File[] taskDirs = regionDir.listFiles(); + if (taskDirs == null) { + return; + } + + for (final File taskDir : taskDirs) { + if (!taskDir.isDirectory() + || LoadTsFileManager.TERMINAL_HISTORY_DIR.equals(taskDir.getName()) + || loadId2RetainedTask.containsKey(taskDir.getName())) { + continue; + } - final File[] taskDirs = regionDir.listFiles(); - if (taskDirs == null) { + // A task the region still holds is in progress: the command path and the terminal path + // own it, and reading its records here would open every progress file of the directory on + // every scan to learn nothing. The idle timeout is still applied to it. + final long recordedCommandTimeMillis = + recordedCommandTimeMillis(dataRegion, taskDir.getName()); + if (recordedCommandTimeMillis >= 0L) { + reclaimIfIdle(dataRegion, taskDir.getName(), taskDir, recordedCommandTimeMillis); + continue; + } + + final LoadTsFileProgress.TerminalRecord terminal = + LoadTsFileProgress.readTerminalOfTaskDir(taskDir); + // A task that never reached COMMIT or ABORT belongs to a coordinator that died before it + // sent one: nothing else would ever revisit its staged data. + if (terminal == null) { + reclaimIfIdle( + dataRegion, taskDir.getName(), taskDir, lastCommandTimeMillis(dataRegion, taskDir)); + continue; + } + + // Do not release the only copy of a legacy terminal decision if its migration failed. This + // check also covers a manager that was initialized before this scan. + if (!LoadTsFileManager.hasTerminalHistory( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), taskDir.getName())) { + try { + dataRegion.getLoadTsFileManager(); + } catch (final IllegalStateException e) { + return; + } + if (!LoadTsFileManager.hasTerminalHistory( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), taskDir.getName()) + || loadId2RetainedTask.containsKey(taskDir.getName())) { continue; } + } - for (final File taskDir : taskDirs) { - if (!taskDir.isDirectory() - || LoadTsFileManager.TERMINAL_HISTORY_DIR.equals(taskDir.getName()) - || loadId2RetainedTask.containsKey(taskDir.getName())) { - continue; - } - - // A task the region still holds is in progress: the command path and the terminal path - // own it, and reading its records here would open every progress file of the directory on - // every scan to learn nothing. The idle timeout is still applied to it. - final long recordedCommandTimeMillis = - recordedCommandTimeMillis(dataRegion, taskDir.getName()); - if (recordedCommandTimeMillis >= 0L) { - reclaimIfIdle(dataRegion, taskDir.getName(), taskDir, recordedCommandTimeMillis); - continue; - } - - final LoadTsFileProgress.TerminalRecord terminal = - LoadTsFileProgress.readTerminalOfTaskDir(taskDir); - // A task that never reached COMMIT or ABORT belongs to a coordinator that died before it - // sent one: nothing else would ever revisit its staged data. - if (terminal == null) { - reclaimIfIdle( - dataRegion, taskDir.getName(), taskDir, lastCommandTimeMillis(dataRegion, taskDir)); - continue; - } - - final RetainedTask unmanagedTask = - new RetainedTask( - taskDir.getName(), dataRegion, taskDir, terminal.searchIndex(), terminal.op()); - - if (canDelete(unmanagedTask)) { - delete(unmanagedTask); - } - } + final RetainedTask unmanagedTask = + new RetainedTask( + taskDir.getName(), dataRegion, taskDir, terminal.searchIndex(), terminal.op()); + + if (canDelete(unmanagedTask)) { + delete(unmanagedTask); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 43a1ee954e284..c6d71f2663945 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -435,6 +435,26 @@ static List getTerminalHistoryDirs(final String databaseName, final String return result; } + static boolean hasTerminalHistory( + final String databaseName, final String dataRegionId, final String loadId) { + for (final File historyDir : getTerminalHistoryDirs(databaseName, dataRegionId)) { + try { + if (LoadTsFileProgress.readTerminal( + LoadTsFileProgress.progressFileFor(new File(historyDir, loadId))) + != null) { + return true; + } + } catch (final IOException e) { + LOGGER.warn( + StorageEngineMessages.LOG_LOAD_CONSENSUS_RECOVER_TASK_META_FAILED_C39E04BB, + historyDir.getPath(), + e); + return false; + } + } + return false; + } + /** Region deletion ends consensus retries; ordinary shutdown must keep these records. */ public static void clearTerminalHistory(final String databaseName, final String dataRegionId) { for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index e131ae6e468d7..6eeb57c279025 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -2523,6 +2523,22 @@ public void testDeletedRegionStopsItsLoadManagerAndRefusesANewOne() throws Excep } } + @Test + public void testDeletedRegionDoesNotInitializeItsLoadManager() throws Exception { + Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty()); + + StorageEngine.getInstance().deleteDataRegion(new DataRegionId(0)); + + Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty()); + try { + dataRegion.getLoadTsFileManager(); + Assert.fail("A deleted region must not initialize a LOAD manager"); + } catch (final IllegalStateException e) { + Assert.assertTrue(e.getMessage().contains("is deleted")); + } + Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty()); + } + public static class DummyDataRegion extends DataRegion { public DummyDataRegion(String systemInfoDir, String storageGroupName) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleanerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleanerTest.java index 01482f2dfe045..3d925c68d56ee 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleanerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleanerTest.java @@ -45,12 +45,14 @@ import org.mockito.Mockito; import java.io.File; +import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; /** How the staged directory of a LOAD that received no command is reclaimed. */ @@ -266,6 +268,91 @@ public void testAFinishedCommittedTaskIsReleasedOnceItsStagedFilesAreComplete() } } + @Test + public void testLegacyTerminalIsMigratedBeforeCleanerReclaimsTask() throws Exception { + final String loadId = "legacy-terminal-task"; + final File regionDir = LoadStagingDirs.regionLoadDir(tempDir, DATABASE, "0"); + final File taskDir = new File(regionDir, loadId); + Assert.assertTrue(taskDir.mkdirs()); + stageOneChunk(taskDir, 5L); + new LoadTsFileProgress(new File(taskDir, "root.sg-0-0.tsfile")) + .recordTerminal(LoadTsFileConsensusOp.ABORT, 7L); + walWatermark.set(10L); + + final AtomicReference manager = new AtomicReference<>(); + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()) + .thenAnswer(invocation -> Optional.ofNullable(manager.get())); + Mockito.when(dataRegion.getLoadTsFileManager()) + .thenAnswer( + invocation -> { + final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); + manager.set(recovered); + return recovered; + }); + + try { + final Method scanRegion = + LoadTsFileCleaner.class.getDeclaredMethod("scanRegion", DataRegion.class, File.class); + scanRegion.setAccessible(true); + scanRegion.invoke(LoadTsFileCleaner.getInstance(), dataRegion, regionDir); + + Assert.assertFalse("the replicated ABORT may release its old staging", taskDir.exists()); + final File historyRecord = + LoadTsFileProgress.progressFileFor( + new File(new File(regionDir, LoadTsFileManager.TERMINAL_HISTORY_DIR), loadId)); + final LoadTsFileProgress.TerminalRecord terminal = + LoadTsFileProgress.readTerminal(historyRecord); + Assert.assertNotNull(terminal); + Assert.assertEquals(LoadTsFileConsensusOp.ABORT, terminal.op()); + } finally { + if (manager.get() != null) { + manager.get().stop(); + } + } + } + + @Test + public void testLegacyTerminalIsNotReclaimedWhenMigrationCannotPersist() throws Exception { + final String loadId = "unmigratable-legacy-terminal-task"; + final File regionDir = LoadStagingDirs.regionLoadDir(tempDir, DATABASE, "0"); + final File taskDir = new File(regionDir, loadId); + Assert.assertTrue(taskDir.mkdirs()); + stageOneChunk(taskDir, 5L); + new LoadTsFileProgress(new File(taskDir, "root.sg-0-0.tsfile")) + .recordTerminal(LoadTsFileConsensusOp.ABORT, 7L); + final File historyDir = new File(regionDir, LoadTsFileManager.TERMINAL_HISTORY_DIR); + Files.write(historyDir.toPath(), new byte[0]); + walWatermark.set(10L); + + final AtomicReference manager = new AtomicReference<>(); + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()) + .thenAnswer(invocation -> Optional.ofNullable(manager.get())); + Mockito.when(dataRegion.getLoadTsFileManager()) + .thenAnswer( + invocation -> { + final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); + manager.set(recovered); + return recovered; + }); + + try { + final Method scanRegion = + LoadTsFileCleaner.class.getDeclaredMethod("scanRegion", DataRegion.class, File.class); + scanRegion.setAccessible(true); + scanRegion.invoke(LoadTsFileCleaner.getInstance(), dataRegion, regionDir); + + Assert.assertTrue( + "the staged legacy decision is the only retry record when migration fails", + taskDir.exists()); + Assert.assertTrue( + "the test must keep the history destination unavailable", historyDir.isFile()); + } finally { + if (manager.get() != null) { + manager.get().stop(); + } + } + } + @Test public void testALoadCommandReclaimsAnIdleTaskOfTheSameRegion() throws Exception { config.setLoadCleanupTaskIdleTimeoutSeconds(1L); From a7d199b65162c9d62bad1c4e64883e33472fc58f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:17:05 +0800 Subject: [PATCH 08/10] Avoid snapshot deadlock during LOAD manager initialization --- .../storageengine/dataregion/DataRegion.java | 34 ++++++++------- .../dataregion/DataRegionTest.java | 42 +++++++++++++++++++ 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 270927aeb1655..84e1a2b847647 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -570,29 +570,31 @@ public String getDatabaseName() { } public LoadTsFileManager getLoadTsFileManager() { - insertLock.readLock().lock(); - try { - if (deleted) { - throw new IllegalStateException( - String.format( - StorageEngineMessages - .STORAGE_EXCEPTION_REGION_ARG_IS_DELETED_SO_IT_TAKES_NO_LOAD_WORK_6E05108F, - databaseName + "-" + dataRegionIdString)); - } - LoadTsFileManager manager = loadTsFileManager; - if (manager == null) { - synchronized (this) { + LoadTsFileManager manager = loadTsFileManager; + if (manager == null) { + synchronized (this) { + // SnapshotTaker takes this monitor before insertLock. Keep that order while still draining + // initialization before markDeleted stops the manager and clears its terminal history. + insertLock.readLock().lock(); + try { manager = loadTsFileManager; - if (manager == null) { + if (manager == null && !deleted) { manager = new LoadTsFileManager(this); loadTsFileManager = manager; } + } finally { + insertLock.readLock().unlock(); } } - return manager; - } finally { - insertLock.readLock().unlock(); } + if (deleted) { + throw new IllegalStateException( + String.format( + StorageEngineMessages + .STORAGE_EXCEPTION_REGION_ARG_IS_DELETED_SO_IT_TAKES_NO_LOAD_WORK_6E05108F, + databaseName + "-" + dataRegionIdString)); + } + return manager; } public Optional getLoadTsFileManagerIfPresent() { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index 6eeb57c279025..7e7c2f2543c9a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -147,9 +147,12 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; import static org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils.genInsertRowNode; import static org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils.genInsertTabletNode; @@ -2539,6 +2542,45 @@ public void testDeletedRegionDoesNotInitializeItsLoadManager() throws Exception Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty()); } + @Test + public void testSnapshotDoesNotDeadlockWithLazyLoadManagerInitialization() throws Exception { + Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty()); + final Field insertLockField = DataRegion.class.getDeclaredField("insertLock"); + insertLockField.setAccessible(true); + final Lock insertWriteLock = ((ReadWriteLock) insertLockField.get(dataRegion)).writeLock(); + final FutureTask initialization = + new FutureTask<>(dataRegion::getLoadTsFileManager); + final Thread initializingThread = new Thread(initialization, "load-manager-initialization"); + try { + synchronized (dataRegion) { + // SnapshotTaker holds this monitor before it flushes under the region's insert write lock. + initializingThread.start(); + await() + .atMost(10, TimeUnit.SECONDS) + .until(() -> initializingThread.getState() == Thread.State.BLOCKED); + // Bound the acquisition so the regression fails instead of leaving deadlocked threads. + assertTrue( + "Snapshot must acquire insertLock while lazy initialization waits for its monitor", + insertWriteLock.tryLock(1, TimeUnit.SECONDS)); + try { + final File snapshot = + new File(TestConstant.OUTPUT_DATA_DIR, "snapshot-during-load-manager-initialization"); + assertTrue( + new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true)); + Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty()); + } finally { + insertWriteLock.unlock(); + } + } + Assert.assertSame( + initialization.get(10, TimeUnit.SECONDS), dataRegion.getLoadTsFileManager()); + } finally { + initializingThread.join(10_000L); + Assert.assertFalse(initializingThread.isAlive()); + dataRegion.stopLoadTsFileManager(); + } + } + public static class DummyDataRegion extends DataRegion { public DummyDataRegion(String systemInfoDir, String storageGroupName) From aa6b6acd01da446363f2e337c0163aaeeec0f4d6 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:28:02 +0800 Subject: [PATCH 09/10] Preserve legacy LOAD terminals across snapshots Capture preserved COMMIT and ABORT progress when terminal-history migration fails, including before lazy manager creation. Add snapshot and restore coverage, and exercise permanent COMMIT exceptions in the state-machine test. --- .../dataregion/snapshot/SnapshotTaker.java | 3 +- .../load/LoadTsFileSnapshot.java | 48 +++++++++-- .../DataRegionStateMachineTest.java | 3 + .../load/LoadTsFileManagerTest.java | 81 ++++++++++++++++++- 4 files changed, 124 insertions(+), 11 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java index b2397c0088bb8..7a580450e9069 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java @@ -41,6 +41,7 @@ import java.nio.file.Files; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -95,7 +96,7 @@ public boolean takeFullSnapshot( boolean success; snapshotLogger.logSnapshotId(finalSnapshotId); - final List loadTerminalHistory; + final Map loadTerminalHistory; try { synchronized (dataRegion) { // Prevent lazy manager creation while selecting the LOAD and data snapshot boundary. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java index c0a23843c0306..f75b312c93bcb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java @@ -99,10 +99,11 @@ public static boolean snapshot(final DataRegion dataRegion, final File snapshotD * published afterwards may describe data absent from those lists, so replay must not be fenced by * that decision. Records are immutable and can be copied after the data files. */ - public static List captureTerminalHistory(final DataRegion dataRegion) throws IOException { + public static Map captureTerminalHistory(final DataRegion dataRegion) + throws IOException { Objects.requireNonNull( dataRegion, StorageEngineMessages.EXCEPTION_DATAREGION_CANNOT_BE_NULL_0B936879); - final List result = new ArrayList<>(); + final Map result = new HashMap<>(); if (isSupportedProtocol()) { // A restarted region need not have initialized its lazy LOAD manager yet. for (final File historyDir : @@ -110,7 +111,40 @@ public static List captureTerminalHistory(final DataRegion dataRegion) thr dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString())) { for (final File file : listFiles(historyDir)) { if (file.isFile() && isProgressLog(file) && !file.getName().contains(TEMP_FILE_SUFFIX)) { - result.add(file); + result.put(file.getName(), file); + } + } + } + final Set activeLoadIds = new HashSet<>(); + dataRegion + .getLoadTsFileManagerIfPresent() + .ifPresent( + manager -> + manager.getActiveTaskDirs().forEach(dir -> activeLoadIds.add(dir.getName()))); + for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) { + final File regionDir = + LoadStagingDirs.regionLoadDir( + new File(baseDir), + dataRegion.getDatabaseName(), + dataRegion.getDataRegionIdString()); + for (final File taskDir : listFiles(regionDir)) { + final String recordName = taskDir.getName() + PROGRESS_SUFFIX; + if (!taskDir.isDirectory() + || LoadTsFileManager.TERMINAL_HISTORY_DIR.equals(taskDir.getName()) + || activeLoadIds.contains(taskDir.getName()) + || result.containsKey(recordName)) { + continue; + } + // A failed migration deliberately leaves the legacy decision in staging. Snapshot that + // record under its task identity, even when no lazy manager has recovered the directory. + for (final File file : listFiles(taskDir)) { + if (file.isFile() + && isProgressLog(file) + && !file.getName().contains(TEMP_FILE_SUFFIX) + && LoadTsFileProgress.readTerminal(file) != null) { + result.put(recordName, file); + break; + } } } } @@ -122,7 +156,7 @@ public static List captureTerminalHistory(final DataRegion dataRegion) thr public static boolean snapshot( final DataRegion dataRegion, final File snapshotDir, - final List capturedTerminalHistory) { + final Map capturedTerminalHistory) { Objects.requireNonNull( dataRegion, StorageEngineMessages.EXCEPTION_DATAREGION_CANNOT_BE_NULL_0B936879); Objects.requireNonNull( @@ -138,7 +172,7 @@ public static boolean snapshot( final File loadSnapshotDir = new File(snapshotDir, SNAPSHOT_SUBDIR_NAME); try { - final List terminalHistory = + final Map terminalHistory = capturedTerminalHistory == null ? captureTerminalHistory(dataRegion) : capturedTerminalHistory; @@ -153,8 +187,8 @@ public static boolean snapshot( // Terminal records have no payload references and may share any restored staging root. final File historyTargetDir = new File(loadSnapshotDir, LoadTsFileManager.TERMINAL_HISTORY_DIR); - for (final File terminal : terminalHistory) { - publishAtomicCopy(terminal, new File(historyTargetDir, terminal.getName())); + for (final Map.Entry terminal : terminalHistory.entrySet()) { + publishAtomicCopy(terminal.getValue(), new File(historyTargetDir, terminal.getKey())); } LOGGER.info( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java index 04302d520cc82..c0cc8416a7ca4 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java @@ -55,6 +55,8 @@ public void testGenericReplicatedLoadFailuresAreNotRetried() throws Exception { final LoadFileException failure = new LoadFileException("injected permanent LOAD failure"); Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap())) .thenThrow(failure); + Mockito.when(dataRegion.writeLoadTsFileCommit(Mockito.any(), Mockito.anyMap())) + .thenThrow(failure); final IoTConsensusDataRegionStateMachine stateMachine = new IoTConsensusDataRegionStateMachine(dataRegion); final LoadTsFileConsensusNode prepare = @@ -67,6 +69,7 @@ public void testGenericReplicatedLoadFailuresAreNotRetried() throws Exception { final TSStatus localStatus = stateMachine.write(node); Assert.assertEquals(TSStatusCode.LOAD_FILE_ERROR.getStatusCode(), localStatus.getCode()); Assert.assertFalse(RetryUtils.needRetryForWrite(localStatus.getCode())); + Assert.assertEquals(failure.getMessage(), localStatus.getMessage()); node.markAsGeneratedByRemoteConsensusLeader(); final TSStatus remoteStatus = stateMachine.write(node); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java index f4af3318fe629..f87d51cfb3329 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java @@ -777,10 +777,10 @@ public void testRecoveryMigratesLegacyTerminalRecordIntoSnapshotHistory() throws final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); try { assertFalse(writer.getTaskDir().exists()); - final List terminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion); + final Map terminalHistory = + LoadTsFileSnapshot.captureTerminalHistory(dataRegion); assertEquals(1, terminalHistory.size()); - assertEquals( - loadId + LoadTsFileProgress.PROGRESS_SUFFIX, terminalHistory.get(0).getName()); + assertTrue(terminalHistory.containsKey(loadId + LoadTsFileProgress.PROGRESS_SUFFIX)); final LoadTsFileConsensusNode commit = commitNode(loadId); commit.markAsGeneratedByRemoteConsensusLeader(); assertTrue(recovered.loadAll(commit, Collections.emptyMap())); @@ -795,6 +795,81 @@ public void testRecoveryMigratesLegacyTerminalRecordIntoSnapshotHistory() throws } } + @Test + public void testSnapshotPreservesLegacyTerminalsWhenMigrationCannotPersist() throws Exception { + final String originalProtocol = config.getDataRegionConsensusProtocolClass(); + config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + LoadStagingDirs.folderManager(); + final File regionDir = + LoadStagingDirs.regionLoadDir( + tempDir, dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + assertTrue(regionDir.mkdirs()); + final File historyDir = new File(regionDir, LoadTsFileManager.TERMINAL_HISTORY_DIR); + Files.write(historyDir.toPath(), new byte[0]); + final LoadTsFileConsensusOp[] outcomes = { + LoadTsFileConsensusOp.COMMIT, LoadTsFileConsensusOp.ABORT + }; + for (final LoadTsFileConsensusOp outcome : outcomes) { + final File taskDir = new File(regionDir, "legacy-" + outcome); + new LoadTsFileProgress(new File(taskDir, "legacy.tsfile")).recordTerminal(outcome, 10L); + } + + final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); + try { + assertTrue(recovered.getActiveTaskDirs().isEmpty()); + assertTrue(historyDir.isFile()); + final TsFileManager tsFileManager = Mockito.mock(TsFileManager.class); + Mockito.when(dataRegion.getTsFileManager()).thenReturn(tsFileManager); + Mockito.when(tsFileManager.getTsFileList(Mockito.anyBoolean())) + .thenAnswer(invocation -> new ArrayList<>()); + File snapshot = null; + for (final boolean hasManager : new boolean[] {false, true}) { + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()) + .thenReturn(hasManager ? Optional.of(recovered) : Optional.empty()); + snapshot = new File(tempDir, "legacy-terminal-snapshot-" + hasManager); + assertTrue( + new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true)); + for (final LoadTsFileConsensusOp outcome : outcomes) { + final File snapshotHistory = + new File( + new File(snapshot, LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME), + LoadTsFileManager.TERMINAL_HISTORY_DIR); + final LoadTsFileProgress.TerminalRecord terminal = + LoadTsFileProgress.readTerminal( + new File( + snapshotHistory, "legacy-" + outcome + LoadTsFileProgress.PROGRESS_SUFFIX)); + assertNotNull(terminal); + assertEquals(outcome, terminal.op()); + assertEquals(10L, terminal.searchIndex()); + } + } + + recovered.stop(); + LoadTsFileSnapshot.clear(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + LoadTsFileSnapshot.restore( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), snapshot); + final LoadTsFileManager restored = new LoadTsFileManager(dataRegion); + try { + for (final LoadTsFileConsensusOp outcome : outcomes) { + final String loadId = "legacy-" + outcome; + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + prepare.markAsGeneratedByRemoteConsensusLeader(); + assertTrue(restored.prepare(prepare, Collections.emptyMap())); + final LoadTsFileConsensusNode commit = commitNode(loadId); + commit.markAsGeneratedByRemoteConsensusLeader(); + assertEquals( + outcome == LoadTsFileConsensusOp.COMMIT, + restored.loadAll(commit, Collections.emptyMap())); + } + } finally { + restored.stop(); + } + } finally { + recovered.stop(); + config.setDataRegionConsensusProtocolClass(originalProtocol); + } + } + @Test public void testCommitCannotSplitSnapshotDataAndTerminalBoundary() throws Exception { final String originalProtocol = config.getDataRegionConsensusProtocolClass(); From a730680121d64c526ea2c41efc8961f9b6eced6b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:12:08 +0800 Subject: [PATCH 10/10] Honor legacy LOAD terminal when migration fails --- .../storageengine/load/LoadTsFileManager.java | 10 ++++- .../load/LoadTsFileManagerTest.java | 38 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index c6d71f2663945..47e9b79c7787e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -557,7 +557,15 @@ private LoadTsFileConsensusOp getTerminalOp(final String loadId) throws IOExcept continue; } if (terminal != null) { - rememberTerminal(loadId, terminal.op(), terminal.searchIndex()); + try { + rememberTerminal(loadId, terminal.op(), terminal.searchIndex()); + } catch (final IOException e) { + // The legacy terminal is still authoritative when its migration cannot be published. + LOGGER.warn( + StorageEngineMessages.LOG_LOAD_CONSENSUS_RECOVER_TASK_META_FAILED_C39E04BB, + file.getPath(), + e); + } return terminal.op(); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java index f87d51cfb3329..98a5facbc11c8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java @@ -795,6 +795,44 @@ public void testRecoveryMigratesLegacyTerminalRecordIntoSnapshotHistory() throws } } + @Test + public void testReplicatedRetriesUseLegacyTerminalWhenMigrationFails() throws Exception { + final String originalProtocol = config.getDataRegionConsensusProtocolClass(); + config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + final String loadId = "unmigratable-legacy-abort"; + final File regionDir = + LoadStagingDirs.regionLoadDir(tempDir, dataRegion.getDatabaseName(), "0"); + final File taskDir = new File(regionDir, loadId); + final File historyDir = new File(regionDir, LoadTsFileManager.TERMINAL_HISTORY_DIR); + try { + assertTrue(taskDir.mkdirs()); + new LoadTsFileProgress(new File(taskDir, "legacy.tsfile")) + .recordTerminal(LoadTsFileConsensusOp.ABORT, 10L); + Files.write(historyDir.toPath(), new byte[0]); + + final LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + try { + final LoadTsFileConsensusNode piece = stagedPiece(loadId, laidOutChunk()); + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + final LoadTsFileConsensusNode commit = commitNode(loadId); + piece.markAsGeneratedByRemoteConsensusLeader(); + prepare.markAsGeneratedByRemoteConsensusLeader(); + commit.markAsGeneratedByRemoteConsensusLeader(); + + assertTrue(manager.writePiece(piece).isEmpty()); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertFalse(manager.loadAll(commit, Collections.emptyMap())); + assertFalse(manager.deleteAll(abortNode(loadId))); + assertTrue(taskDir.isDirectory()); + assertTrue(historyDir.isFile()); + } finally { + manager.stop(); + } + } finally { + config.setDataRegionConsensusProtocolClass(originalProtocol); + } + } + @Test public void testSnapshotPreservesLegacyTerminalsWhenMigrationCannotPersist() throws Exception { final String originalProtocol = config.getDataRegionConsensusProtocolClass();