diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java index 4f8eca8f27551..461ee1daabf3a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.exception.BatchProcessException; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.WriteProcessRejectException; +import org.apache.iotdb.db.exception.load.LoadFileNotReadyException; import org.apache.iotdb.db.exception.query.OutOfTTLException; import org.apache.iotdb.db.exception.runtime.TableLostRuntimeException; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; @@ -90,6 +91,8 @@ public TSStatus visitLoadTsFileConsensus( final boolean prepared = dataRegion.writeLoadTsFilePrepare(node, deserializeProgressIndexes(node)); yield prepared ? StatusUtils.OK : RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR); + } catch (final LoadFileNotReadyException e) { + yield getLoadFileNotReadyStatus(node, e); } catch (final Exception e) { LOGGER.error(DataNodeMiscMessages.ERROR_EXECUTING_PLAN_NODE, node, e); yield RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, e.getMessage()); @@ -100,6 +103,8 @@ public TSStatus visitLoadTsFileConsensus( final boolean committed = dataRegion.writeLoadTsFileCommit(node, deserializeProgressIndexes(node)); yield committed ? StatusUtils.OK : RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR); + } catch (final LoadFileNotReadyException e) { + yield getLoadFileNotReadyStatus(node, e); } catch (final Exception e) { LOGGER.error(DataNodeMiscMessages.ERROR_EXECUTING_PLAN_NODE, node, e); yield RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, e.getMessage()); @@ -117,6 +122,15 @@ public TSStatus visitLoadTsFileConsensus( }; } + private TSStatus getLoadFileNotReadyStatus( + final LoadTsFileConsensusNode node, final LoadFileNotReadyException exception) { + if (node.isGeneratedByRemoteConsensusLeader()) { + return RpcUtils.getStatus(TSStatusCode.WRITE_PROCESS_REJECT, exception.getMessage()); + } + LOGGER.error(DataNodeMiscMessages.ERROR_EXECUTING_PLAN_NODE, node, exception); + return RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exception.getMessage()); + } + /** Restores the pipe progress a PREPARE or COMMIT carries, per time partition. */ private Map deserializeProgressIndexes( final LoadTsFileConsensusNode node) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java index 3d2fb3d2006b3..f3c833f4b9b92 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceManager; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.SearchNode; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache; @@ -343,6 +344,10 @@ public static boolean needRetryForSpecificCases(int statusCode, PlanNode planNod // consensus layer. if (statusCode == TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode() && !planNode.isGeneratedByPipe()) { + if (planNode instanceof LoadTsFileConsensusNode + && ((LoadTsFileConsensusNode) planNode).isGeneratedByRemoteConsensusLeader()) { + return false; + } return true; } return statusCode == TSStatusCode.METADATA_LEASE_FENCED_RETRY_REQUIRED.getStatusCode() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileNotReadyException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileNotReadyException.java new file mode 100644 index 0000000000000..213b039598a9c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileNotReadyException.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.exception.load; + +/** A LOAD command arrived before the staged data needed to apply it. */ +public class LoadFileNotReadyException extends LoadFileException { + + public LoadFileNotReadyException() { + super(""); + } + + public LoadFileNotReadyException(final String message) { + super(message); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 94f44022ceffe..84e1a2b847647 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -573,18 +573,21 @@ public LoadTsFileManager getLoadTsFileManager() { LoadTsFileManager manager = loadTsFileManager; if (manager == null) { synchronized (this) { - manager = loadTsFileManager; - if (manager == null) { - manager = new LoadTsFileManager(this); - loadTsFileManager = manager; + // SnapshotTaker takes this monitor before insertLock. Keep that order while still draining + // initialization before markDeleted stops the manager and clears its terminal history. + insertLock.readLock().lock(); + try { + manager = loadTsFileManager; + if (manager == null && !deleted) { + manager = new LoadTsFileManager(this); + loadTsFileManager = manager; + } + } finally { + insertLock.readLock().unlock(); } } } if (deleted) { - // The region went away: it takes no LOAD work, and the manager is stopped for good. A manager - // created in this window stops itself here, the one stopLoadTsFileManager saw was stopped by - // it. - manager.stop(); throw new IllegalStateException( String.format( StorageEngineMessages @@ -5664,6 +5667,7 @@ public void markDeleted() { try { deleted = true; stopLoadTsFileManager(); + LoadTsFileManager.clearTerminalHistory(databaseName, dataRegionIdString); shutdownUpgradeModFileThreadPool(); releaseDirectBufferMemory(); MetricService.getInstance().removeMetricSet(metrics); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java index d32615fec33b6..7a580450e9069 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.load.LoadTsFileManager; import org.apache.iotdb.db.storageengine.load.LoadTsFileSnapshot; import org.slf4j.Logger; @@ -40,6 +41,7 @@ import java.nio.file.Files; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -94,20 +96,38 @@ public boolean takeFullSnapshot( boolean success; snapshotLogger.logSnapshotId(finalSnapshotId); + final Map loadTerminalHistory; try { - readLockTheFile(); - if (flushBeforeSnapshot) { + synchronized (dataRegion) { + // Prevent lazy manager creation while selecting the LOAD and data snapshot boundary. + final LoadTsFileManager loadManager = + dataRegion.getLoadTsFileManagerIfPresent().orElse(null); + if (loadManager != null) { + loadManager.lockSnapshotBoundary(); + } try { - dataRegion.writeLock("snapshotTaker"); - dataRegion.syncCloseAllWorkingTsFileProcessors(); + loadTerminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion); + readLockTheFile(); + if (flushBeforeSnapshot) { + dataRegion.writeLock("snapshotTaker"); + try { + dataRegion.syncCloseAllWorkingTsFileProcessors(); + } finally { + dataRegion.writeUnlock(); + } + } + // A COMMIT after data-file selection must not remove staging before it is copied. + // Its pieces may predate the new replica's sync channel and never be replayed there. + success = LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir, loadTerminalHistory); } finally { - dataRegion.writeUnlock(); + if (loadManager != null) { + loadManager.unlockSnapshotBoundary(); + } } } - success = createSnapshot(seqFiles, tempSnapshotId); + success = success && createSnapshot(seqFiles, tempSnapshotId); success = success && createSnapshot(unseqFiles, tempSnapshotId); success = success && snapshotCompressionRatio(snapshotDirPath); - success = success && LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir); } finally { readUnlockTheFile(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java index 93a537c1e4aca..7fbfebe6a7d6d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleaner.java @@ -173,45 +173,66 @@ private void scanRegions() { new File(baseDir), dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + scanRegion(dataRegion, regionDir); + } + } + } + + private void scanRegion(final DataRegion dataRegion, final File regionDir) { + File[] taskDirs = regionDir.listFiles(); + if (taskDirs == null) { + return; + } + + for (final File taskDir : taskDirs) { + if (!taskDir.isDirectory() + || LoadTsFileManager.TERMINAL_HISTORY_DIR.equals(taskDir.getName()) + || loadId2RetainedTask.containsKey(taskDir.getName())) { + continue; + } - final File[] taskDirs = regionDir.listFiles(); - if (taskDirs == null) { + // A task the region still holds is in progress: the command path and the terminal path + // own it, and reading its records here would open every progress file of the directory on + // every scan to learn nothing. The idle timeout is still applied to it. + final long recordedCommandTimeMillis = + recordedCommandTimeMillis(dataRegion, taskDir.getName()); + if (recordedCommandTimeMillis >= 0L) { + reclaimIfIdle(dataRegion, taskDir.getName(), taskDir, recordedCommandTimeMillis); + continue; + } + + final LoadTsFileProgress.TerminalRecord terminal = + LoadTsFileProgress.readTerminalOfTaskDir(taskDir); + // A task that never reached COMMIT or ABORT belongs to a coordinator that died before it + // sent one: nothing else would ever revisit its staged data. + if (terminal == null) { + reclaimIfIdle( + dataRegion, taskDir.getName(), taskDir, lastCommandTimeMillis(dataRegion, taskDir)); + continue; + } + + // Do not release the only copy of a legacy terminal decision if its migration failed. This + // check also covers a manager that was initialized before this scan. + if (!LoadTsFileManager.hasTerminalHistory( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), taskDir.getName())) { + try { + dataRegion.getLoadTsFileManager(); + } catch (final IllegalStateException e) { + return; + } + if (!LoadTsFileManager.hasTerminalHistory( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), taskDir.getName()) + || loadId2RetainedTask.containsKey(taskDir.getName())) { continue; } + } - for (final File taskDir : taskDirs) { - if (!taskDir.isDirectory() || loadId2RetainedTask.containsKey(taskDir.getName())) { - continue; - } - - // A task the region still holds is in progress: the command path and the terminal path - // own it, and reading its records here would open every progress file of the directory on - // every scan to learn nothing. The idle timeout is still applied to it. - final long recordedCommandTimeMillis = - recordedCommandTimeMillis(dataRegion, taskDir.getName()); - if (recordedCommandTimeMillis >= 0L) { - reclaimIfIdle(dataRegion, taskDir.getName(), taskDir, recordedCommandTimeMillis); - continue; - } - - final LoadTsFileProgress.TerminalRecord terminal = - LoadTsFileProgress.readTerminalOfTaskDir(taskDir); - // A task that never reached COMMIT or ABORT belongs to a coordinator that died before it - // sent one: nothing else would ever revisit its staged data. - if (terminal == null) { - reclaimIfIdle( - dataRegion, taskDir.getName(), taskDir, lastCommandTimeMillis(dataRegion, taskDir)); - continue; - } - - final RetainedTask unmanagedTask = - new RetainedTask( - taskDir.getName(), dataRegion, taskDir, terminal.searchIndex(), terminal.op()); - - if (canDelete(unmanagedTask)) { - delete(unmanagedTask); - } - } + final RetainedTask unmanagedTask = + new RetainedTask( + taskDir.getName(), dataRegion, taskDir, terminal.searchIndex(), terminal.op()); + + if (canDelete(unmanagedTask)) { + delete(unmanagedTask); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 7e373d774668c..47e9b79c7787e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.load.LoadFileException; +import org.apache.iotdb.db.exception.load.LoadFileNotReadyException; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusOp; @@ -47,7 +48,9 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -56,6 +59,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; @@ -78,6 +82,8 @@ public class LoadTsFileManager { private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); + static final String TERMINAL_HISTORY_DIR = ".finished"; + public static final Cache MEASUREMENT_ID_CACHE = Caffeine.newBuilder() .maximumWeight(CONFIG.getLoadMeasurementIdCacheSizeInBytes()) @@ -99,25 +105,22 @@ public class LoadTsFileManager { private final LoadTaskRetention retention; /** - * The tasks this region finished committing, most recent last, to answer a repeated COMMIT. + * The tasks this region finished, most recent last, to answer repeated consensus commands. * - *

The coordinator cannot tell a lost answer from a lost command - an RPC that timed out may - * have applied - so it has to be able to re-send the COMMIT that ended a task. A task this region - * already imported is then a no-op and not a failure, otherwise a retry that follows a lost - * answer would report a load that committed everywhere as failed. A task whose staged directory - * is still there is recognized from the terminal record of its progress files as well; the record - * below covers the tasks that were released right away, and it is bounded because a restart may - * forget it: a repeated COMMIT of a task that committed before the restart is then reported as - * unknown again, which is what it is. + *

This is only a bounded cache. Terminal records also live outside the staged task directories + * for the lifetime of the region, and are included in its LOAD snapshot. A replica must still + * recognize a command whose successful response was lost after staging cleanup or a restart; + * treating that command as missing data would otherwise retry it forever. */ - private static final int COMMITTED_TASK_HISTORY = 256; + private static final int FINISHED_TASK_HISTORY = 256; - private final Map committedLoadIds = + private final Map finishedLoadIds = Collections.synchronizedMap( - new LinkedHashMap(16, 0.75f, true) { + new LinkedHashMap(16, 0.75f, true) { @Override - protected boolean removeEldestEntry(final Map.Entry eldest) { - return size() > COMMITTED_TASK_HISTORY; + protected boolean removeEldestEntry( + final Map.Entry eldest) { + return size() > FINISHED_TASK_HISTORY; } }); @@ -126,6 +129,9 @@ protected boolean removeEldestEntry(final Map.Entry eldest) { /** Pieces read, stop() and closing a writer write. Fair, so a stop is not starved. */ private final ReentrantReadWriteLock writersLock = new ReentrantReadWriteLock(true); + /** Fences COMMIT/ABORT against selection and copying of a data and LOAD-state snapshot. */ + private final ReentrantReadWriteLock snapshotBoundaryLock = new ReentrantReadWriteLock(true); + /** Set under the write side, read as the fast path of a piece. */ private volatile boolean stopped = false; @@ -238,14 +244,28 @@ private void recover() { continue; } for (final File uuidDir : uuidDirs) { - if (!uuidDir.isDirectory()) { + if (!uuidDir.isDirectory() || TERMINAL_HISTORY_DIR.equals(uuidDir.getName())) { continue; } final String uuid = uuidDir.getName(); - if (retention.recoverTerminalTask(uuid, uuidDir)) { - continue; - } try { + final LoadTsFileProgress.TerminalRecord terminal = readTerminalHistory(uuid); + if (terminal != null) { + // The process may have stopped after publishing the decision but before closing or + // marking staging. That directory belongs to a finished task, never an active writer. + finishedLoadIds.put(uuid, terminal.op()); + retention.retain(uuid, uuidDir, terminal.op(), terminal.searchIndex()); + continue; + } + final LoadTsFileProgress.TerminalRecord retainedTerminal = + LoadTsFileProgress.readTerminalOfTaskDir(uuidDir); + if (retainedTerminal != null) { + // Older versions kept the decision only in each staged file's progress tail. + // Migrate it before registering a listener that may release the only staged copy. + rememberTerminal(uuid, retainedTerminal.op(), retainedTerminal.searchIndex()); + retention.retain(uuid, uuidDir, retainedTerminal.op(), retainedTerminal.searchIndex()); + continue; + } final TsFileWriterManager recovered = new TsFileWriterManager(dataRegion, uuidDir).recoverFromDisk(); if (recovered.hasPendingTsFiles()) { @@ -341,6 +361,9 @@ public List writePiece(final LoadTsFileConsens if (stopped) { throw newStoppedIOException(); } + if (node.isGeneratedByRemoteConsensusLeader() && getTerminalOp(node.getLoadId()) != null) { + return Collections.emptyList(); + } final Lock readLock = writersLock.readLock(); readLock.lock(); try { @@ -384,12 +407,133 @@ public List getActiveTaskDirs() { return result; } - /** Whether this region already committed the task, so that a repeated COMMIT is a no-op. */ - private boolean isAlreadyCommitted(final String loadId) { - if (committedLoadIds.containsKey(loadId)) { - return true; + /** Holds the write side while a region snapshot selects its data and LOAD state. */ + public void lockSnapshotBoundary() { + snapshotBoundaryLock.writeLock().lock(); + } + + public void unlockSnapshotBoundary() { + snapshotBoundaryLock.writeLock().unlock(); + } + + /** Terminal metadata is snapshotted separately from active tasks and their staged bytes. */ + List getTerminalHistoryDirs() { + return getTerminalHistoryDirs(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + } + + static List getTerminalHistoryDirs(final String databaseName, final String dataRegionId) { + final List result = new ArrayList<>(); + for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) { + final File historyDir = + new File( + LoadStagingDirs.regionLoadDir(new File(baseDir), databaseName, dataRegionId), + TERMINAL_HISTORY_DIR); + if (historyDir.isDirectory()) { + result.add(historyDir); + } } - for (final String baseDir : LoadStagingDirs.baseDirs()) { + return result; + } + + static boolean hasTerminalHistory( + final String databaseName, final String dataRegionId, final String loadId) { + for (final File historyDir : getTerminalHistoryDirs(databaseName, dataRegionId)) { + try { + if (LoadTsFileProgress.readTerminal( + LoadTsFileProgress.progressFileFor(new File(historyDir, loadId))) + != null) { + return true; + } + } catch (final IOException e) { + LOGGER.warn( + StorageEngineMessages.LOG_LOAD_CONSENSUS_RECOVER_TASK_META_FAILED_C39E04BB, + historyDir.getPath(), + e); + return false; + } + } + return false; + } + + /** Region deletion ends consensus retries; ordinary shutdown must keep these records. */ + public static void clearTerminalHistory(final String databaseName, final String dataRegionId) { + for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) { + LoadStagingDirs.deleteTaskDir( + new File( + LoadStagingDirs.regionLoadDir(new File(baseDir), databaseName, dataRegionId), + TERMINAL_HISTORY_DIR)); + } + } + + private void rememberTerminal(final LoadTsFileConsensusNode node, final LoadTsFileConsensusOp op) + throws IOException { + rememberTerminal(node.getLoadId(), op, node.getSearchIndex()); + } + + private void rememberTerminal( + final String loadId, final LoadTsFileConsensusOp op, final long searchIndex) + throws IOException { + // Region deletion clears history after stop(), so no terminal publication may outlive it. + final Lock readLock = writersLock.readLock(); + readLock.lock(); + try { + if (stopped) { + throw newStoppedIOException(); + } + final File historyDir = + new File( + getDataRegionLoadDir(new File(LoadStagingDirs.configuredBaseDirs()[0])), + TERMINAL_HISTORY_DIR); + final LoadTsFileProgress terminal = + new LoadTsFileProgress(new File(historyDir, loadId + ".copying." + UUID.randomUUID())); + final File temporaryRecord = terminal.getProgressFile(); + final File record = LoadTsFileProgress.progressFileFor(new File(historyDir, loadId)); + try { + terminal.recordTerminal(op, searchIndex); + try { + Files.move( + temporaryRecord.toPath(), + record.toPath(), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (final AtomicMoveNotSupportedException e) { + Files.move( + temporaryRecord.toPath(), record.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } finally { + Files.deleteIfExists(temporaryRecord.toPath()); + } + finishedLoadIds.put(loadId, op); + } finally { + readLock.unlock(); + } + } + + private LoadTsFileProgress.TerminalRecord readTerminalHistory(final String loadId) + throws IOException { + for (final File historyDir : getTerminalHistoryDirs()) { + final LoadTsFileProgress.TerminalRecord terminal = + LoadTsFileProgress.readTerminal( + LoadTsFileProgress.progressFileFor(new File(historyDir, loadId))); + if (terminal != null) { + return terminal; + } + } + return null; + } + + /** The terminal command this region already applied to the task, or null if it is unknown. */ + private LoadTsFileConsensusOp getTerminalOp(final String loadId) throws IOException { + final LoadTsFileConsensusOp finishedOp = finishedLoadIds.get(loadId); + if (finishedOp != null) { + return finishedOp; + } + final LoadTsFileProgress.TerminalRecord history = readTerminalHistory(loadId); + if (history != null) { + finishedLoadIds.put(loadId, history.op()); + return history.op(); + } + for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) { final File taskDir = new File( LoadStagingDirs.regionLoadDir( @@ -402,34 +546,57 @@ private boolean isAlreadyCommitted(final String loadId) { continue; } for (final File file : files) { + final LoadTsFileProgress.TerminalRecord terminal; try { - final LoadTsFileProgress.TerminalRecord terminal = LoadTsFileProgress.readTerminal(file); - if (terminal != null && terminal.op() == LoadTsFileConsensusOp.COMMIT) { - return true; - } + terminal = LoadTsFileProgress.readTerminal(file); } catch (final IOException e) { LOGGER.warn( StorageEngineMessages.LOG_LOAD_CONSENSUS_RECOVER_TASK_META_FAILED_C39E04BB, file.getPath(), e); + continue; + } + if (terminal != null) { + try { + rememberTerminal(loadId, terminal.op(), terminal.searchIndex()); + } catch (final IOException e) { + // The legacy terminal is still authoritative when its migration cannot be published. + LOGGER.warn( + StorageEngineMessages.LOG_LOAD_CONSENSUS_RECOVER_TASK_META_FAILED_C39E04BB, + file.getPath(), + e); + } + return terminal.op(); } } } - return false; + return null; } public boolean prepare( final LoadTsFileConsensusNode node, final Map timePartitionProgressIndexMap) throws IOException, LoadFileException { + // A terminal decision also fences late pieces and retries from an older consensus batch. + if (node.isGeneratedByRemoteConsensusLeader() && getTerminalOp(node.getLoadId()) != null) { + return true; + } if (!uuid2WriterManager.containsKey(node.getLoadId())) { + // A lost PREPARE acknowledgement can be retried after the follower applied COMMIT or ABORT. + if (node.isGeneratedByRemoteConsensusLeader()) { + throw new LoadFileNotReadyException(); + } return false; } if (!uuid2WriterManager.get(node.getLoadId()).hasStagedData()) { - throw new LoadFileException( + final String message = String.format( StorageEngineMessages.MESSAGE_LOAD_CONSENSUS_PREPARE_WITHOUT_STAGED_DATA_FE8ADC37, - node.getLoadId())); + node.getLoadId()); + if (node.isGeneratedByRemoteConsensusLeader()) { + throw new LoadFileNotReadyException(message); + } + throw new LoadFileException(message); } LOGGER.info( StorageEngineMessages @@ -453,35 +620,70 @@ public boolean loadAll( final LoadTsFileConsensusNode node, final Map timePartitionProgressIndexMap) throws IOException, LoadFileException { - if (!uuid2WriterManager.containsKey(node.getLoadId())) { - return isAlreadyCommitted(node.getLoadId()); + final Lock snapshotReadLock = snapshotBoundaryLock.readLock(); + snapshotReadLock.lock(); + final boolean mustRetainStagedFiles; + try { + final LoadTsFileConsensusOp terminalOp = getTerminalOp(node.getLoadId()); + if (terminalOp != null) { + return terminalOp == LoadTsFileConsensusOp.COMMIT; + } + if (!uuid2WriterManager.containsKey(node.getLoadId())) { + if (node.isGeneratedByRemoteConsensusLeader()) { + throw new LoadFileNotReadyException(); + } + return false; + } + LOGGER.info( + StorageEngineMessages + .LOG_COMMITTING_LOAD_TSFILE_ARG_LOADING_PREPARED_RESOURCES_INTO_DATAREGION_EA1D6335, + node.getLoadId()); + // Whether the staged bytes are still needed has to be decided before they are imported: the + // WAL entries of the pieces of this task only reference those bytes, and they are expanded on + // this node for a replica that has not applied them yet. Importing them by moving the staged + // file away would leave those entries unexpandable, so a copy is imported while the watermark + // of the WAL has not got past this command yet. + mustRetainStagedFiles = retention.mustRetain(node.getSearchIndex()); + uuid2WriterManager + .get(node.getLoadId()) + .loadAll( + node.isGeneratedByPipe(), + !mustRetainStagedFiles && !node.isGeneratedByRemoteConsensusLeader()); + // Keep a follower's staged copy until the terminal answer is durable, even without a local + // WAL watermark to retain it. A restart before this record can then resume the prepared task. + rememberTerminal(node, LoadTsFileConsensusOp.COMMIT); + finishConsensusTask(node, mustRetainStagedFiles, LoadTsFileConsensusOp.COMMIT); + } finally { + snapshotReadLock.unlock(); } - LOGGER.info( - StorageEngineMessages - .LOG_COMMITTING_LOAD_TSFILE_ARG_LOADING_PREPARED_RESOURCES_INTO_DATAREGION_EA1D6335, - node.getLoadId()); - // Whether the staged bytes are still needed has to be decided before they are imported: the WAL - // entries of the pieces of this task only reference those bytes, and they are expanded on this - // node for a replica that has not applied them yet. Importing them by moving the staged file - // away would leave those entries unexpandable, so a copy is imported while the watermark of the - // WAL has not got past this command yet. - final boolean mustRetainStagedFiles = retention.mustRetain(node.getSearchIndex()); - uuid2WriterManager - .get(node.getLoadId()) - .loadAll(node.isGeneratedByPipe(), !mustRetainStagedFiles); - finishConsensusTask(node, mustRetainStagedFiles, LoadTsFileConsensusOp.COMMIT); - committedLoadIds.put(node.getLoadId(), Boolean.TRUE); logLoadNodeToWAL(node); dataRegion.insertSeparatorToWAL(node); return true; } public boolean deleteAll(final LoadTsFileConsensusNode node) throws IOException { - if (!uuid2WriterManager.containsKey(node.getLoadId())) { + final Lock snapshotReadLock = snapshotBoundaryLock.readLock(); + snapshotReadLock.lock(); + final boolean hasWriter; + try { + if (getTerminalOp(node.getLoadId()) != null) { + return false; + } + hasWriter = uuid2WriterManager.containsKey(node.getLoadId()); + // ABORT may overtake every PIECE. Its success must still settle pending PREPARE retries. + rememberTerminal(node, LoadTsFileConsensusOp.ABORT); + if (hasWriter) { + finishConsensusTask( + node, retention.mustRetain(node.getSearchIndex()), LoadTsFileConsensusOp.ABORT); + } + } finally { + snapshotReadLock.unlock(); + } + if (!hasWriter) { + logLoadNodeToWAL(node); + dataRegion.insertSeparatorToWAL(node); return false; } - finishConsensusTask( - node, retention.mustRetain(node.getSearchIndex()), LoadTsFileConsensusOp.ABORT); logLoadNodeToWAL(node); dataRegion.insertSeparatorToWAL(node); return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java index 64e3e44a524fc..f75b312c93bcb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java @@ -89,36 +89,107 @@ private LoadTsFileSnapshot() {} // Snapshot Entrypoint // ------------------------------------------------------------------------- - /** Captures an atomic snapshot of active staged tasks under the DataRegion. */ + /** Captures the LOAD state when no data-file snapshot boundary needs to be supplied. */ public static boolean snapshot(final DataRegion dataRegion, final File snapshotDir) { + return snapshot(dataRegion, snapshotDir, null); + } + + /** + * Selects terminal records before the data-file snapshot captures its resource lists. A COMMIT + * published afterwards may describe data absent from those lists, so replay must not be fenced by + * that decision. Records are immutable and can be copied after the data files. + */ + public static Map captureTerminalHistory(final DataRegion dataRegion) + throws IOException { + Objects.requireNonNull( + dataRegion, StorageEngineMessages.EXCEPTION_DATAREGION_CANNOT_BE_NULL_0B936879); + final Map result = new HashMap<>(); + if (isSupportedProtocol()) { + // A restarted region need not have initialized its lazy LOAD manager yet. + for (final File historyDir : + LoadTsFileManager.getTerminalHistoryDirs( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString())) { + for (final File file : listFiles(historyDir)) { + if (file.isFile() && isProgressLog(file) && !file.getName().contains(TEMP_FILE_SUFFIX)) { + result.put(file.getName(), file); + } + } + } + final Set activeLoadIds = new HashSet<>(); + dataRegion + .getLoadTsFileManagerIfPresent() + .ifPresent( + manager -> + manager.getActiveTaskDirs().forEach(dir -> activeLoadIds.add(dir.getName()))); + for (final String baseDir : LoadStagingDirs.configuredBaseDirs()) { + final File regionDir = + LoadStagingDirs.regionLoadDir( + new File(baseDir), + dataRegion.getDatabaseName(), + dataRegion.getDataRegionIdString()); + for (final File taskDir : listFiles(regionDir)) { + final String recordName = taskDir.getName() + PROGRESS_SUFFIX; + if (!taskDir.isDirectory() + || LoadTsFileManager.TERMINAL_HISTORY_DIR.equals(taskDir.getName()) + || activeLoadIds.contains(taskDir.getName()) + || result.containsKey(recordName)) { + continue; + } + // A failed migration deliberately leaves the legacy decision in staging. Snapshot that + // record under its task identity, even when no lazy manager has recovered the directory. + for (final File file : listFiles(taskDir)) { + if (file.isFile() + && isProgressLog(file) + && !file.getName().contains(TEMP_FILE_SUFFIX) + && LoadTsFileProgress.readTerminal(file) != null) { + result.put(recordName, file); + break; + } + } + } + } + } + return result; + } + + /** Captures active staging and only the terminal decisions covered by the data-file snapshot. */ + public static boolean snapshot( + final DataRegion dataRegion, + final File snapshotDir, + final Map capturedTerminalHistory) { Objects.requireNonNull( dataRegion, StorageEngineMessages.EXCEPTION_DATAREGION_CANNOT_BE_NULL_0B936879); Objects.requireNonNull( snapshotDir, StorageEngineMessages.EXCEPTION_SNAPSHOTDIR_CANNOT_BE_NULL_283211CB); - if (!SUPPORTED_PROTOCOLS.contains( - ConsensusFactory.normalizeConsensusProtocolClass( - IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass()))) { + if (!isSupportedProtocol()) { return true; } final LoadTsFileManager manager = dataRegion.getLoadTsFileManagerIfPresent().orElse(null); - if (manager == null) { - return true; - } - - final List taskDirs = manager.getActiveTaskDirs(); - if (taskDirs.isEmpty()) { - return true; - } + final List taskDirs = + manager == null ? Collections.emptyList() : manager.getActiveTaskDirs(); final File loadSnapshotDir = new File(snapshotDir, SNAPSHOT_SUBDIR_NAME); try { + final Map terminalHistory = + capturedTerminalHistory == null + ? captureTerminalHistory(dataRegion) + : capturedTerminalHistory; + if (taskDirs.isEmpty() && terminalHistory.isEmpty()) { + return true; + } writeRootsManifest(loadSnapshotDir, taskDirs); for (final File taskDir : taskDirs) { final File targetDir = new File(loadSnapshotDir, taskDir.getName()); copyTask(taskDir, targetDir); } + // Terminal records have no payload references and may share any restored staging root. + final File historyTargetDir = + new File(loadSnapshotDir, LoadTsFileManager.TERMINAL_HISTORY_DIR); + for (final Map.Entry terminal : terminalHistory.entrySet()) { + publishAtomicCopy(terminal.getValue(), new File(historyTargetDir, terminal.getKey())); + } LOGGER.info( String.format( @@ -136,6 +207,12 @@ public static boolean snapshot(final DataRegion dataRegion, final File snapshotD } } + private static boolean isSupportedProtocol() { + return SUPPORTED_PROTOCOLS.contains( + ConsensusFactory.normalizeConsensusProtocolClass( + IoTDBDescriptor.getInstance().getConfig().getDataRegionConsensusProtocolClass())); + } + // ------------------------------------------------------------------------- // Task Copy Pipeline & Live Catch-Up // ------------------------------------------------------------------------- @@ -146,14 +223,14 @@ public static boolean snapshot(final DataRegion dataRegion, final File snapshotD private static void copyTask(final File taskDir, final File targetDir) throws IOException { // 1. Snapshot raw staged and modification files first for (final File file : listFiles(taskDir)) { - if (file.isFile() && !isProgressLog(file)) { + if (file.isFile() && !file.getName().contains(TEMP_FILE_SUFFIX) && !isProgressLog(file)) { publishAtomicCopy(file, new File(targetDir, file.getName())); } } // 2. Snapshot progress logs afterwards so they describe an existing snapshot boundary for (final File file : listFiles(taskDir)) { - if (file.isFile() && isProgressLog(file)) { + if (file.isFile() && !file.getName().contains(TEMP_FILE_SUFFIX) && isProgressLog(file)) { publishAtomicCopy(file, new File(targetDir, file.getName())); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java index ca30a78b6df23..55e4539372098 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/TsFileWriterManager.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.load.LoadFileException; +import org.apache.iotdb.db.exception.load.LoadFileNotReadyException; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; @@ -497,7 +498,7 @@ private void validateStagedFiles( || progress == null || !progress.isReady(fileLength) || (expectedDataEnd >= 0L && fileLength < expectedDataEnd)) { - throw new LoadFileException( + throw new LoadFileNotReadyException( String.format( StorageEngineMessages.EXCEPTION_LOAD_CONSENSUS_STAGED_FILE_NOT_CONTINUOUS_F9408C19, stagedFile.getAbsolutePath(), @@ -570,7 +571,7 @@ void loadAll(final boolean isGeneratedByPipe, final boolean deleteStagedSource) final TsFilePrecalculatedChunkWriter writer = timePartition2Writer.get(entry.getKey()); if (writer != null && !writer.isSealed()) { - throw new LoadFileException( + throw new LoadFileNotReadyException( String.format( StorageEngineMessages.EXCEPTION_LOAD_CONSENSUS_STAGED_FILE_INCOMPLETE_1CDE954B, tsFileResource.getTsFilePath(), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java index b77765301ecc8..c0cc8416a7ca4 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachineTest.java @@ -25,14 +25,20 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.IPlanVisitor; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.utils.RetryUtils; +import org.apache.iotdb.db.exception.load.LoadFileException; +import org.apache.iotdb.db.exception.load.LoadFileNotReadyException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.io.DataOutputStream; import java.io.IOException; @@ -43,6 +49,74 @@ public class DataRegionStateMachineTest { + @Test + public void testGenericReplicatedLoadFailuresAreNotRetried() throws Exception { + final DataRegion dataRegion = Mockito.mock(DataRegion.class); + final LoadFileException failure = new LoadFileException("injected permanent LOAD failure"); + Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap())) + .thenThrow(failure); + Mockito.when(dataRegion.writeLoadTsFileCommit(Mockito.any(), Mockito.anyMap())) + .thenThrow(failure); + final IoTConsensusDataRegionStateMachine stateMachine = + new IoTConsensusDataRegionStateMachine(dataRegion); + final LoadTsFileConsensusNode prepare = + LoadTsFileConsensusNode.prepare( + new PlanNodeId("prepare"), "load", 1, 1L, false, Collections.emptyMap()); + final LoadTsFileConsensusNode commit = + LoadTsFileConsensusNode.commit( + new PlanNodeId("commit"), "load", false, Collections.emptyMap()); + for (final LoadTsFileConsensusNode node : new LoadTsFileConsensusNode[] {prepare, commit}) { + final TSStatus localStatus = stateMachine.write(node); + Assert.assertEquals(TSStatusCode.LOAD_FILE_ERROR.getStatusCode(), localStatus.getCode()); + Assert.assertFalse(RetryUtils.needRetryForWrite(localStatus.getCode())); + Assert.assertEquals(failure.getMessage(), localStatus.getMessage()); + + node.markAsGeneratedByRemoteConsensusLeader(); + final TSStatus remoteStatus = stateMachine.write(node); + Assert.assertEquals(TSStatusCode.LOAD_FILE_ERROR.getStatusCode(), remoteStatus.getCode()); + Assert.assertFalse(RetryUtils.needRetryForWrite(remoteStatus.getCode())); + Assert.assertEquals(localStatus.getMessage(), remoteStatus.getMessage()); + } + Mockito.verify(dataRegion, Mockito.times(2)) + .writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap()); + Mockito.verify(dataRegion, Mockito.times(2)) + .writeLoadTsFileCommit(Mockito.any(), Mockito.anyMap()); + } + + @Test + public void testRemoteLoadNotReadyIsRetriedByConsensus() throws Exception { + final DataRegion dataRegion = Mockito.mock(DataRegion.class); + Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap())) + .thenThrow(new LoadFileNotReadyException()); + final CountingIoTConsensusDataRegionStateMachine stateMachine = + new CountingIoTConsensusDataRegionStateMachine(dataRegion); + final LoadTsFileConsensusNode prepare = + LoadTsFileConsensusNode.prepare( + new PlanNodeId("prepare"), "load", 1, 1L, false, Collections.emptyMap()); + prepare.markAsGeneratedByRemoteConsensusLeader(); + + final TSStatus status = stateMachine.write(prepare); + + Assert.assertEquals(TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode(), status.getCode()); + Assert.assertTrue(RetryUtils.needRetryForWrite(status.getCode())); + Assert.assertEquals(0, stateMachine.localRetries); + } + + private static class CountingIoTConsensusDataRegionStateMachine + extends IoTConsensusDataRegionStateMachine { + + private int localRetries; + + private CountingIoTConsensusDataRegionStateMachine(final DataRegion region) { + super(region); + } + + @Override + protected void waitBeforeNextWriteRetry() { + localRetries++; + } + } + @Test public void testMergeInsertRowNodes() throws IllegalPathException { List list = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index e131ae6e468d7..7e7c2f2543c9a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -147,9 +147,12 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; import static org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils.genInsertRowNode; import static org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils.genInsertTabletNode; @@ -2523,6 +2526,61 @@ public void testDeletedRegionStopsItsLoadManagerAndRefusesANewOne() throws Excep } } + @Test + public void testDeletedRegionDoesNotInitializeItsLoadManager() throws Exception { + Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty()); + + StorageEngine.getInstance().deleteDataRegion(new DataRegionId(0)); + + Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty()); + try { + dataRegion.getLoadTsFileManager(); + Assert.fail("A deleted region must not initialize a LOAD manager"); + } catch (final IllegalStateException e) { + Assert.assertTrue(e.getMessage().contains("is deleted")); + } + Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty()); + } + + @Test + public void testSnapshotDoesNotDeadlockWithLazyLoadManagerInitialization() throws Exception { + Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty()); + final Field insertLockField = DataRegion.class.getDeclaredField("insertLock"); + insertLockField.setAccessible(true); + final Lock insertWriteLock = ((ReadWriteLock) insertLockField.get(dataRegion)).writeLock(); + final FutureTask initialization = + new FutureTask<>(dataRegion::getLoadTsFileManager); + final Thread initializingThread = new Thread(initialization, "load-manager-initialization"); + try { + synchronized (dataRegion) { + // SnapshotTaker holds this monitor before it flushes under the region's insert write lock. + initializingThread.start(); + await() + .atMost(10, TimeUnit.SECONDS) + .until(() -> initializingThread.getState() == Thread.State.BLOCKED); + // Bound the acquisition so the regression fails instead of leaving deadlocked threads. + assertTrue( + "Snapshot must acquire insertLock while lazy initialization waits for its monitor", + insertWriteLock.tryLock(1, TimeUnit.SECONDS)); + try { + final File snapshot = + new File(TestConstant.OUTPUT_DATA_DIR, "snapshot-during-load-manager-initialization"); + assertTrue( + new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true)); + Assert.assertTrue(dataRegion.getLoadTsFileManagerIfPresent().isEmpty()); + } finally { + insertWriteLock.unlock(); + } + } + Assert.assertSame( + initialization.get(10, TimeUnit.SECONDS), dataRegion.getLoadTsFileManager()); + } finally { + initializingThread.join(10_000L); + Assert.assertFalse(initializingThread.isAlive()); + dataRegion.stopLoadTsFileManager(); + } + } + public static class DummyDataRegion extends DataRegion { public DummyDataRegion(String systemInfoDir, String storageGroupName) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleanerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleanerTest.java index 01482f2dfe045..3d925c68d56ee 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleanerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileCleanerTest.java @@ -45,12 +45,14 @@ import org.mockito.Mockito; import java.io.File; +import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; /** How the staged directory of a LOAD that received no command is reclaimed. */ @@ -266,6 +268,91 @@ public void testAFinishedCommittedTaskIsReleasedOnceItsStagedFilesAreComplete() } } + @Test + public void testLegacyTerminalIsMigratedBeforeCleanerReclaimsTask() throws Exception { + final String loadId = "legacy-terminal-task"; + final File regionDir = LoadStagingDirs.regionLoadDir(tempDir, DATABASE, "0"); + final File taskDir = new File(regionDir, loadId); + Assert.assertTrue(taskDir.mkdirs()); + stageOneChunk(taskDir, 5L); + new LoadTsFileProgress(new File(taskDir, "root.sg-0-0.tsfile")) + .recordTerminal(LoadTsFileConsensusOp.ABORT, 7L); + walWatermark.set(10L); + + final AtomicReference manager = new AtomicReference<>(); + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()) + .thenAnswer(invocation -> Optional.ofNullable(manager.get())); + Mockito.when(dataRegion.getLoadTsFileManager()) + .thenAnswer( + invocation -> { + final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); + manager.set(recovered); + return recovered; + }); + + try { + final Method scanRegion = + LoadTsFileCleaner.class.getDeclaredMethod("scanRegion", DataRegion.class, File.class); + scanRegion.setAccessible(true); + scanRegion.invoke(LoadTsFileCleaner.getInstance(), dataRegion, regionDir); + + Assert.assertFalse("the replicated ABORT may release its old staging", taskDir.exists()); + final File historyRecord = + LoadTsFileProgress.progressFileFor( + new File(new File(regionDir, LoadTsFileManager.TERMINAL_HISTORY_DIR), loadId)); + final LoadTsFileProgress.TerminalRecord terminal = + LoadTsFileProgress.readTerminal(historyRecord); + Assert.assertNotNull(terminal); + Assert.assertEquals(LoadTsFileConsensusOp.ABORT, terminal.op()); + } finally { + if (manager.get() != null) { + manager.get().stop(); + } + } + } + + @Test + public void testLegacyTerminalIsNotReclaimedWhenMigrationCannotPersist() throws Exception { + final String loadId = "unmigratable-legacy-terminal-task"; + final File regionDir = LoadStagingDirs.regionLoadDir(tempDir, DATABASE, "0"); + final File taskDir = new File(regionDir, loadId); + Assert.assertTrue(taskDir.mkdirs()); + stageOneChunk(taskDir, 5L); + new LoadTsFileProgress(new File(taskDir, "root.sg-0-0.tsfile")) + .recordTerminal(LoadTsFileConsensusOp.ABORT, 7L); + final File historyDir = new File(regionDir, LoadTsFileManager.TERMINAL_HISTORY_DIR); + Files.write(historyDir.toPath(), new byte[0]); + walWatermark.set(10L); + + final AtomicReference manager = new AtomicReference<>(); + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()) + .thenAnswer(invocation -> Optional.ofNullable(manager.get())); + Mockito.when(dataRegion.getLoadTsFileManager()) + .thenAnswer( + invocation -> { + final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); + manager.set(recovered); + return recovered; + }); + + try { + final Method scanRegion = + LoadTsFileCleaner.class.getDeclaredMethod("scanRegion", DataRegion.class, File.class); + scanRegion.setAccessible(true); + scanRegion.invoke(LoadTsFileCleaner.getInstance(), dataRegion, regionDir); + + Assert.assertTrue( + "the staged legacy decision is the only retry record when migration fails", + taskDir.exists()); + Assert.assertTrue( + "the test must keep the history destination unavailable", historyDir.isFile()); + } finally { + if (manager.get() != null) { + manager.get().stop(); + } + } + } + @Test public void testALoadCommandReclaimsAnIdleTaskOfTheSameRegion() throws Exception { config.setLoadCleanupTaskIdleTimeoutSeconds(1L); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java index 9c0060c5fa9f2..98a5facbc11c8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManagerTest.java @@ -23,9 +23,13 @@ import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.RecoverProgressIndex; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.request.IConsensusRequest; +import org.apache.iotdb.commons.utils.RetryUtils; +import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.consensus.iot.log.ConsensusReqReader; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.consensus.statemachine.dataregion.IoTConsensusDataRegionStateMachine; import org.apache.iotdb.db.exception.load.LoadFileException; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; @@ -34,6 +38,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusOp; import org.apache.iotdb.db.queryengine.plan.scheduler.load.ChunkOffsetCalculator; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.apache.iotdb.db.storageengine.dataregion.snapshot.SnapshotTaker; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.wal.node.IWALNode; @@ -45,6 +51,8 @@ import org.apache.iotdb.db.storageengine.load.splitter.ChunkPayloadUnavailableException; import org.apache.iotdb.db.storageengine.load.splitter.NonAlignedChunkData; import org.apache.iotdb.db.storageengine.load.splitter.TsFileData; +import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.header.ChunkHeader; @@ -81,11 +89,13 @@ import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.LongConsumer; +import java.util.function.Predicate; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -118,6 +128,8 @@ public class LoadTsFileManagerTest { public void setUp() throws Exception { tempDir = Files.createTempDirectory("load-tsfile-manager-test").toFile(); config = IoTDBDescriptor.getInstance().getConfig(); + // Startup resolves LOAD roots once. Do that before installing this test's isolated roots. + TierManager.getInstance(); originalLoadBaseDirs = config.getLoadTsFileDirs(); config.setLoadTsFileDirs(new String[] {tempDir.getAbsolutePath()}); @@ -246,12 +258,6 @@ public void testOutOfOrderPiecesSurviveTheLossOfTheInMemoryWriters() throws Exce // progress records stays on disk. dropInMemoryWriters(firstRun); - // The first lookup of a staging directory makes the tier manager reformulate the configured - // LOAD directories from the data directories, which in a test is not the directory the pieces - // were just staged in. A DataNode does that once at startup, before any load, so pinning the - // directory here is what keeps the restarted manager looking at the very same staged file. - config.setLoadTsFileDirs(new String[] {tempDir.getAbsolutePath()}); - // A new manager resumes the very same staged directory from those records. final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); @@ -501,6 +507,531 @@ public void testPrepareFailsWhileAPieceOfTheTaskIsStillMissing() throws Exceptio assertTrue(manager.prepare(prepareNode(loadId), Collections.emptyMap())); } + @Test + public void testReplicatedLoadRetriesCommandsThatOvertakePieces() throws Exception { + final String loadId = "replicated-out-of-order-load"; + final LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + final IoTConsensusDataRegionStateMachine stateMachine = + new IoTConsensusDataRegionStateMachine(dataRegion); + Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap())) + .thenAnswer( + invocation -> manager.prepare(invocation.getArgument(0), invocation.getArgument(1))); + Mockito.when(dataRegion.writeLoadTsFileCommit(Mockito.any(), Mockito.anyMap())) + .thenAnswer( + invocation -> manager.loadAll(invocation.getArgument(0), invocation.getArgument(1))); + emulateStagedFileImport(); + + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + final LoadTsFileConsensusNode commit = commitNode(loadId); + prepare.markAsGeneratedByRemoteConsensusLeader(); + commit.markAsGeneratedByRemoteConsensusLeader(); + try { + // Neither command may be acknowledged while the piece is still retrying after reconnect. + assertTrue( + RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) prepare).getCode())); + assertTrue( + RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) commit).getCode())); + + final NonAlignedChunkData[] chunks = twoChunksOfTheSamePartition(); + final List refs = + manager.writePiece(loadId, toTsFileDataList(Collections.singletonList(chunks[1]))); + final File staged = new File(tempDir, refs.get(0).getRelativePath()); + // A partially received piece set must remain retryable as well. + assertTrue( + RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) prepare).getCode())); + manager.writePiece(loadId, toTsFileDataList(Collections.singletonList(chunks[0]))); + assertTrue( + RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) commit).getCode())); + + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + stateMachine.write((IConsensusRequest) prepare).getCode()); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + stateMachine.write((IConsensusRequest) commit).getCode()); + final File imported = new File(new File(tempDir, "data"), staged.getName()); + assertAllSeriesReadable( + imported, + java.util.Arrays.asList( + new SeriesExpectation( + new StringArrayDeviceID("root", "load_manager_test", "d0"), "s0", 10, 0, 9), + new SeriesExpectation( + new StringArrayDeviceID("root", "load_manager_test", "d1"), "s1", 10, 100, 109))); + + // Losing an acknowledgement must not leave PREPARE retrying forever after COMMIT succeeds. + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + stateMachine.write((IConsensusRequest) prepare).getCode()); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + stateMachine.write((IConsensusRequest) commit).getCode()); + } finally { + manager.stop(); + } + } + + @Test + public void testReplicatedPrepareAfterAbortIsANoOp() throws Exception { + final String loadId = "replicated-aborted-load"; + final LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + try { + manager.writePiece(loadId, Collections.singletonList(laidOutChunk())); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertTrue(manager.deleteAll(abortNode(loadId))); + + assertFalse(manager.prepare(prepare, Collections.emptyMap())); + prepare.markAsGeneratedByRemoteConsensusLeader(); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertFalse(manager.loadAll(commitNode(loadId), Collections.emptyMap())); + } finally { + manager.stop(); + } + } + + @Test + public void testReplicatedCommitRetrySurvivesRestartAndSnapshot() throws Exception { + final String loadId = "replicated-committed-retry"; + LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + emulateStagedFileImport(); + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + final LoadTsFileConsensusNode commit = commitNode(loadId); + prepare.markAsGeneratedByRemoteConsensusLeader(); + commit.markAsGeneratedByRemoteConsensusLeader(); + try { + manager.writePiece(loadId, Collections.singletonList(laidOutChunk())); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertTrue(manager.loadAll(commit, Collections.emptyMap())); + assertTrue(manager.getActiveTaskDirs().isEmpty()); + + // The COMMIT response was lost, and the follower restarts before it is retried. + manager.stop(); + manager = new LoadTsFileManager(dataRegion); + assertTrue(manager.loadAll(commit, Collections.emptyMap())); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + + final String originalProtocol = config.getDataRegionConsensusProtocolClass(); + try { + config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + // After another restart no LOAD has initialized this region's lazy manager yet. + manager.stop(); + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.empty()); + final File snapshot = new File(tempDir, "finished-load-snapshot"); + assertTrue(LoadTsFileSnapshot.snapshot(dataRegion, snapshot)); + LoadTsFileSnapshot.clear(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + LoadTsFileSnapshot.restore( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), snapshot); + manager = new LoadTsFileManager(dataRegion); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertTrue(manager.loadAll(commit, Collections.emptyMap())); + } finally { + config.setDataRegionConsensusProtocolClass(originalProtocol); + } + Mockito.verify(dataRegion, Mockito.times(1)) + .loadNewTsFile( + Mockito.any(), + Mockito.anyBoolean(), + Mockito.anyBoolean(), + Mockito.anyBoolean(), + Mockito.any()); + } finally { + manager.stop(); + } + } + + @Test + public void testSnapshotRetainsStagingForLoadCommittedAfterDataFileCapture() throws Exception { + final String loadId = "commit-during-snapshot"; + final String originalProtocol = config.getDataRegionConsensusProtocolClass(); + final LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + try { + config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager)); + emulateStagedFileImport(); + manager.writePiece(loadId, Collections.singletonList(laidOutChunk())); + assertTrue(manager.prepare(prepareNode(loadId), Collections.emptyMap())); + + final TsFileManager tsFileManager = Mockito.mock(TsFileManager.class); + Mockito.when(dataRegion.getTsFileManager()).thenReturn(tsFileManager); + final AtomicBoolean committed = new AtomicBoolean(); + Mockito.when(tsFileManager.getTsFileList(Mockito.anyBoolean())) + .thenAnswer( + invocation -> + new ArrayList() { + @Override + public boolean removeIf(final Predicate filter) { + // Data-file copying starts after the snapshot boundary lock is released. + if (committed.compareAndSet(false, true)) { + try { + assertTrue(manager.loadAll(commitNode(loadId), Collections.emptyMap())); + } catch (final Exception e) { + throw new AssertionError(e); + } + } + return super.removeIf(filter); + } + }); + + final File snapshot = new File(tempDir, "concurrent-commit-snapshot"); + assertTrue(new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true)); + assertTrue(committed.get()); + final File history = + new File( + new File(snapshot, LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME), + LoadTsFileManager.TERMINAL_HISTORY_DIR); + assertFalse(LoadTsFileProgress.progressFileFor(new File(history, loadId)).exists()); + + // PIECE may predate the new replica's sync channel, so it cannot be replayed after restore. + // The snapshot itself must retain the staging that its later COMMIT still needs. + manager.stop(); + LoadTsFileSnapshot.clear(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + LoadTsFileSnapshot.restore( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), snapshot); + final LoadTsFileManager restored = new LoadTsFileManager(dataRegion); + try { + assertEquals(1, restored.getActiveTaskDirs().size()); + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + prepare.markAsGeneratedByRemoteConsensusLeader(); + assertTrue(restored.prepare(prepare, Collections.emptyMap())); + final File[] stagedFiles = + restored.getActiveTaskDirs().get(0).listFiles((dir, name) -> name.endsWith(".tsfile")); + assertNotNull(stagedFiles); + assertEquals(1, stagedFiles.length); + assertAllSeriesReadable( + stagedFiles[0], + Collections.singletonList( + new SeriesExpectation( + new StringArrayDeviceID("root", "load_manager_test", "d0"), "s0", 10, 0, 9))); + } finally { + restored.stop(); + } + } finally { + manager.stop(); + config.setDataRegionConsensusProtocolClass(originalProtocol); + } + } + + @Test + public void testAbortCannotReplaceACommittedDecisionBeforeStagingCleanup() throws Exception { + final String loadId = "commit-before-staging-cleanup"; + final LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + try { + emulateStagedFileImport(); + manager.writePiece(loadId, Collections.singletonList(laidOutChunk())); + assertTrue(manager.prepare(prepareNode(loadId), Collections.emptyMap())); + final TsFileWriterManager writer = writerOf(manager, loadId); + writer.loadAll(false, false); + final File historyDir = + new File(writer.getTaskDir().getParentFile(), LoadTsFileManager.TERMINAL_HISTORY_DIR); + new LoadTsFileProgress(new File(historyDir, loadId)) + .recordTerminal(LoadTsFileConsensusOp.COMMIT, 10L); + // A crash after the durable decision leaves staging without its per-file terminal tails. + writer.close(true); + + final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); + try { + assertFalse(recovered.deleteAll(abortNode(loadId))); + assertTrue(recovered.loadAll(commitNode(loadId), Collections.emptyMap())); + assertTrue(recovered.getActiveTaskDirs().isEmpty()); + Mockito.verify(dataRegion, Mockito.times(1)) + .loadNewTsFile( + Mockito.any(), + Mockito.anyBoolean(), + Mockito.anyBoolean(), + Mockito.anyBoolean(), + Mockito.any()); + } finally { + recovered.stop(); + } + } finally { + manager.stop(); + } + } + + @Test + public void testRecoveryMigratesLegacyTerminalRecordIntoSnapshotHistory() throws Exception { + final String originalProtocol = config.getDataRegionConsensusProtocolClass(); + config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + final String loadId = "legacy-terminal-load"; + try { + useWatermarkWALNode(5L); + final LoadTsFileManager beforeUpgrade = new LoadTsFileManager(dataRegion); + try { + beforeUpgrade.writePiece(loadId, Collections.singletonList(laidOutChunk())); + assertTrue(beforeUpgrade.prepare(prepareNode(loadId), Collections.emptyMap())); + final TsFileWriterManager writer = writerOf(beforeUpgrade, loadId); + writer.recordTerminal(LoadTsFileConsensusOp.COMMIT, 10L, Collections.emptyMap()); + writer.close(true); + + final IWALNode walNode = dataRegion.getWALNode().get(); + Mockito.doAnswer( + invocation -> { + // A watermark can advance as soon as recovery registers its release listener. + final LongConsumer listener = invocation.getArgument(0); + walWatermark.set(10L); + listener.accept(10L); + return null; + }) + .when(walNode) + .setSafeDeletedSearchIndexListener(Mockito.any()); + final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); + try { + assertFalse(writer.getTaskDir().exists()); + final Map terminalHistory = + LoadTsFileSnapshot.captureTerminalHistory(dataRegion); + assertEquals(1, terminalHistory.size()); + assertTrue(terminalHistory.containsKey(loadId + LoadTsFileProgress.PROGRESS_SUFFIX)); + final LoadTsFileConsensusNode commit = commitNode(loadId); + commit.markAsGeneratedByRemoteConsensusLeader(); + assertTrue(recovered.loadAll(commit, Collections.emptyMap())); + } finally { + recovered.stop(); + } + } finally { + beforeUpgrade.stop(); + } + } finally { + config.setDataRegionConsensusProtocolClass(originalProtocol); + } + } + + @Test + public void testReplicatedRetriesUseLegacyTerminalWhenMigrationFails() throws Exception { + final String originalProtocol = config.getDataRegionConsensusProtocolClass(); + config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + final String loadId = "unmigratable-legacy-abort"; + final File regionDir = + LoadStagingDirs.regionLoadDir(tempDir, dataRegion.getDatabaseName(), "0"); + final File taskDir = new File(regionDir, loadId); + final File historyDir = new File(regionDir, LoadTsFileManager.TERMINAL_HISTORY_DIR); + try { + assertTrue(taskDir.mkdirs()); + new LoadTsFileProgress(new File(taskDir, "legacy.tsfile")) + .recordTerminal(LoadTsFileConsensusOp.ABORT, 10L); + Files.write(historyDir.toPath(), new byte[0]); + + final LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + try { + final LoadTsFileConsensusNode piece = stagedPiece(loadId, laidOutChunk()); + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + final LoadTsFileConsensusNode commit = commitNode(loadId); + piece.markAsGeneratedByRemoteConsensusLeader(); + prepare.markAsGeneratedByRemoteConsensusLeader(); + commit.markAsGeneratedByRemoteConsensusLeader(); + + assertTrue(manager.writePiece(piece).isEmpty()); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertFalse(manager.loadAll(commit, Collections.emptyMap())); + assertFalse(manager.deleteAll(abortNode(loadId))); + assertTrue(taskDir.isDirectory()); + assertTrue(historyDir.isFile()); + } finally { + manager.stop(); + } + } finally { + config.setDataRegionConsensusProtocolClass(originalProtocol); + } + } + + @Test + public void testSnapshotPreservesLegacyTerminalsWhenMigrationCannotPersist() throws Exception { + final String originalProtocol = config.getDataRegionConsensusProtocolClass(); + config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + LoadStagingDirs.folderManager(); + final File regionDir = + LoadStagingDirs.regionLoadDir( + tempDir, dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + assertTrue(regionDir.mkdirs()); + final File historyDir = new File(regionDir, LoadTsFileManager.TERMINAL_HISTORY_DIR); + Files.write(historyDir.toPath(), new byte[0]); + final LoadTsFileConsensusOp[] outcomes = { + LoadTsFileConsensusOp.COMMIT, LoadTsFileConsensusOp.ABORT + }; + for (final LoadTsFileConsensusOp outcome : outcomes) { + final File taskDir = new File(regionDir, "legacy-" + outcome); + new LoadTsFileProgress(new File(taskDir, "legacy.tsfile")).recordTerminal(outcome, 10L); + } + + final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion); + try { + assertTrue(recovered.getActiveTaskDirs().isEmpty()); + assertTrue(historyDir.isFile()); + final TsFileManager tsFileManager = Mockito.mock(TsFileManager.class); + Mockito.when(dataRegion.getTsFileManager()).thenReturn(tsFileManager); + Mockito.when(tsFileManager.getTsFileList(Mockito.anyBoolean())) + .thenAnswer(invocation -> new ArrayList<>()); + File snapshot = null; + for (final boolean hasManager : new boolean[] {false, true}) { + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()) + .thenReturn(hasManager ? Optional.of(recovered) : Optional.empty()); + snapshot = new File(tempDir, "legacy-terminal-snapshot-" + hasManager); + assertTrue( + new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true)); + for (final LoadTsFileConsensusOp outcome : outcomes) { + final File snapshotHistory = + new File( + new File(snapshot, LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME), + LoadTsFileManager.TERMINAL_HISTORY_DIR); + final LoadTsFileProgress.TerminalRecord terminal = + LoadTsFileProgress.readTerminal( + new File( + snapshotHistory, "legacy-" + outcome + LoadTsFileProgress.PROGRESS_SUFFIX)); + assertNotNull(terminal); + assertEquals(outcome, terminal.op()); + assertEquals(10L, terminal.searchIndex()); + } + } + + recovered.stop(); + LoadTsFileSnapshot.clear(dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + LoadTsFileSnapshot.restore( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), snapshot); + final LoadTsFileManager restored = new LoadTsFileManager(dataRegion); + try { + for (final LoadTsFileConsensusOp outcome : outcomes) { + final String loadId = "legacy-" + outcome; + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + prepare.markAsGeneratedByRemoteConsensusLeader(); + assertTrue(restored.prepare(prepare, Collections.emptyMap())); + final LoadTsFileConsensusNode commit = commitNode(loadId); + commit.markAsGeneratedByRemoteConsensusLeader(); + assertEquals( + outcome == LoadTsFileConsensusOp.COMMIT, + restored.loadAll(commit, Collections.emptyMap())); + } + } finally { + restored.stop(); + } + } finally { + recovered.stop(); + config.setDataRegionConsensusProtocolClass(originalProtocol); + } + } + + @Test + public void testCommitCannotSplitSnapshotDataAndTerminalBoundary() throws Exception { + final String originalProtocol = config.getDataRegionConsensusProtocolClass(); + config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + final String loadId = "commit-at-snapshot-boundary"; + try { + final LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + final AtomicBoolean commitStarted = new AtomicBoolean(); + final CountDownLatch commitFinished = new CountDownLatch(1); + final AtomicReference commitFailure = new AtomicReference<>(); + final AtomicReference commitThread = new AtomicReference<>(); + try { + final LoadTsFileConsensusNode commit = commitNode(loadId); + commit.markAsGeneratedByRemoteConsensusLeader(); + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager)); + emulateStagedFileImport(); + manager.writePiece(loadId, Collections.singletonList(laidOutChunk())); + assertTrue(manager.prepare(prepareNode(loadId), Collections.emptyMap())); + + final TsFileManager tsFileManager = Mockito.mock(TsFileManager.class); + Mockito.when(tsFileManager.getTsFileList(Mockito.anyBoolean())) + .thenAnswer(invocation -> new ArrayList<>()); + Mockito.when(dataRegion.getTsFileManager()) + .thenAnswer( + invocation -> { + if (commitStarted.compareAndSet(false, true)) { + final Thread thread = + new Thread( + () -> { + try { + assertTrue(manager.loadAll(commit, Collections.emptyMap())); + } catch (final Throwable t) { + commitFailure.set(t); + } finally { + commitFinished.countDown(); + } + }); + commitThread.set(thread); + thread.start(); + assertFalse( + "COMMIT must wait for snapshot boundary capture", + commitFinished.await(1, TimeUnit.SECONDS)); + } + return tsFileManager; + }); + + final File snapshot = new File(tempDir, "commit-boundary-snapshot"); + assertTrue( + new SnapshotTaker(dataRegion).takeFullSnapshot(snapshot.getAbsolutePath(), true)); + assertTrue( + "COMMIT must continue after snapshot boundary capture", + commitFinished.await(10, TimeUnit.SECONDS)); + commitThread.get().join(10_000L); + assertFalse(commitThread.get().isAlive()); + assertNull(commitFailure.get()); + + final File history = + new File( + new File(snapshot, LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME), + LoadTsFileManager.TERMINAL_HISTORY_DIR); + assertFalse(LoadTsFileProgress.progressFileFor(new File(history, loadId)).exists()); + } finally { + final Thread thread = commitThread.get(); + if (thread != null) { + thread.join(10_000L); + } + manager.stop(); + } + } finally { + config.setDataRegionConsensusProtocolClass(originalProtocol); + } + } + + @Test + public void testReplicatedAbortBeforePiecesStopsRetries() throws Exception { + final String loadId = "replicated-abort-before-pieces"; + LoadTsFileManager manager = new LoadTsFileManager(dataRegion); + final LoadTsFileConsensusNode prepare = prepareNode(loadId); + final LoadTsFileConsensusNode commit = commitNode(loadId); + final LoadTsFileConsensusNode abort = abortNode(loadId); + final LoadTsFileConsensusNode piece = stagedPiece(loadId, laidOutChunk()); + prepare.markAsGeneratedByRemoteConsensusLeader(); + commit.markAsGeneratedByRemoteConsensusLeader(); + abort.markAsGeneratedByRemoteConsensusLeader(); + piece.markAsGeneratedByRemoteConsensusLeader(); + try { + Mockito.when(dataRegion.writeLoadTsFilePrepare(Mockito.any(), Mockito.anyMap())) + .thenAnswer( + invocation -> + dataRegion + .getLoadTsFileManagerIfPresent() + .get() + .prepare(invocation.getArgument(0), invocation.getArgument(1))); + Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager)); + final IoTConsensusDataRegionStateMachine stateMachine = + new IoTConsensusDataRegionStateMachine(dataRegion); + assertTrue( + RetryUtils.needRetryForWrite(stateMachine.write((IConsensusRequest) prepare).getCode())); + + // ABORT can overtake the same missing PIECE as PREPARE did. + assertFalse(manager.deleteAll(abort)); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + stateMachine.write((IConsensusRequest) prepare).getCode()); + manager.writePiece(piece); + assertTrue(manager.getActiveTaskDirs().isEmpty()); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertFalse(manager.loadAll(commit, Collections.emptyMap())); + + manager.stop(); + manager = new LoadTsFileManager(dataRegion); + assertTrue(manager.prepare(prepare, Collections.emptyMap())); + assertFalse(manager.loadAll(commit, Collections.emptyMap())); + manager.writePiece(piece); + assertTrue(manager.getActiveTaskDirs().isEmpty()); + manager.stop(); + LoadTsFileManager.clearTerminalHistory( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + assertTrue(manager.getTerminalHistoryDirs().isEmpty()); + } finally { + manager.stop(); + } + } + /** * A progress file that ends in the middle of an entry is what a snapshot of the staging directory * holds when it was taken while a piece was being appended. The fragment describes nothing, while @@ -1112,6 +1643,16 @@ public void testStopIsTerminalForWrites() throws Exception { } catch (final IOException e) { assertTrue(e.getMessage().contains("has been stopped")); } + LoadTsFileManager.clearTerminalHistory( + dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString()); + try { + // A command that already obtained the manager must not recreate a deleted region's history. + manager.deleteAll(abortNode("abort-after-region-deletion")); + fail("A stopped LOAD manager has to refuse terminal publication"); + } catch (final IOException e) { + assertTrue(e.getMessage().contains("has been stopped")); + } + assertTrue(manager.getTerminalHistoryDirs().isEmpty()); } /** A stop waits for the piece in flight, and still closes the writer that piece created. */