Skip to content
Draft
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.iotdb.db.exception.BatchProcessException;
import org.apache.iotdb.db.exception.WriteProcessException;
import org.apache.iotdb.db.exception.WriteProcessRejectException;
import org.apache.iotdb.db.exception.load.LoadFileNotReadyException;
import org.apache.iotdb.db.exception.query.OutOfTTLException;
import org.apache.iotdb.db.exception.runtime.TableLostRuntimeException;
import org.apache.iotdb.db.i18n.DataNodeMiscMessages;
Expand Down Expand Up @@ -90,6 +91,8 @@ public TSStatus visitLoadTsFileConsensus(
final boolean prepared =
dataRegion.writeLoadTsFilePrepare(node, deserializeProgressIndexes(node));
yield prepared ? StatusUtils.OK : RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR);
} catch (final LoadFileNotReadyException e) {
yield getLoadFileNotReadyStatus(node, e);
} catch (final Exception e) {
LOGGER.error(DataNodeMiscMessages.ERROR_EXECUTING_PLAN_NODE, node, e);
yield RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, e.getMessage());
Expand All @@ -100,6 +103,8 @@ public TSStatus visitLoadTsFileConsensus(
final boolean committed =
dataRegion.writeLoadTsFileCommit(node, deserializeProgressIndexes(node));
yield committed ? StatusUtils.OK : RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR);
} catch (final LoadFileNotReadyException e) {
yield getLoadFileNotReadyStatus(node, e);
} catch (final Exception e) {
LOGGER.error(DataNodeMiscMessages.ERROR_EXECUTING_PLAN_NODE, node, e);
yield RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, e.getMessage());
Expand All @@ -117,6 +122,15 @@ public TSStatus visitLoadTsFileConsensus(
};
}

private TSStatus getLoadFileNotReadyStatus(
final LoadTsFileConsensusNode node, final LoadFileNotReadyException exception) {
if (node.isGeneratedByRemoteConsensusLeader()) {
return RpcUtils.getStatus(TSStatusCode.WRITE_PROCESS_REJECT, exception.getMessage());
}
LOGGER.error(DataNodeMiscMessages.ERROR_EXECUTING_PLAN_NODE, node, exception);
return RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exception.getMessage());
}

/** Restores the pipe progress a PREPARE or COMMIT carries, per time partition. */
private Map<TTimePartitionSlot, ProgressIndex> deserializeProgressIndexes(
final LoadTsFileConsensusNode node) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceManager;
import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.SearchNode;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
Expand Down Expand Up @@ -343,6 +344,10 @@ public static boolean needRetryForSpecificCases(int statusCode, PlanNode planNod
// consensus layer.
if (statusCode == TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode()
&& !planNode.isGeneratedByPipe()) {
if (planNode instanceof LoadTsFileConsensusNode
&& ((LoadTsFileConsensusNode) planNode).isGeneratedByRemoteConsensusLeader()) {
return false;
}
return true;
}
return statusCode == TSStatusCode.METADATA_LEASE_FENCED_RETRY_REQUIRED.getStatusCode()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.db.exception.load;

/** A LOAD command arrived before the staged data needed to apply it. */
public class LoadFileNotReadyException extends LoadFileException {

public LoadFileNotReadyException() {
super("");
}

public LoadFileNotReadyException(final String message) {
super(message);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -573,18 +573,21 @@ public LoadTsFileManager getLoadTsFileManager() {
LoadTsFileManager manager = loadTsFileManager;
if (manager == null) {
synchronized (this) {
manager = loadTsFileManager;
if (manager == null) {
manager = new LoadTsFileManager(this);
loadTsFileManager = manager;
// SnapshotTaker takes this monitor before insertLock. Keep that order while still draining
// initialization before markDeleted stops the manager and clears its terminal history.
insertLock.readLock().lock();
try {
manager = loadTsFileManager;
if (manager == null && !deleted) {
manager = new LoadTsFileManager(this);
loadTsFileManager = manager;
}
} finally {
insertLock.readLock().unlock();
}
}
}
if (deleted) {
// The region went away: it takes no LOAD work, and the manager is stopped for good. A manager
// created in this window stops itself here, the one stopLoadTsFileManager saw was stopped by
// it.
manager.stop();
throw new IllegalStateException(
String.format(
StorageEngineMessages
Expand Down Expand Up @@ -5664,6 +5667,7 @@ public void markDeleted() {
try {
deleted = true;
stopLoadTsFileManager();
LoadTsFileManager.clearTerminalHistory(databaseName, dataRegionIdString);
shutdownUpgradeModFileThreadPool();
releaseDirectBufferMemory();
MetricService.getInstance().removeMetricSet(metrics);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.load.LoadTsFileManager;
import org.apache.iotdb.db.storageengine.load.LoadTsFileSnapshot;

import org.slf4j.Logger;
Expand All @@ -40,6 +41,7 @@
import java.nio.file.Files;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;

Expand Down Expand Up @@ -94,20 +96,38 @@ public boolean takeFullSnapshot(
boolean success;
snapshotLogger.logSnapshotId(finalSnapshotId);

final Map<String, File> loadTerminalHistory;
try {
readLockTheFile();
if (flushBeforeSnapshot) {
synchronized (dataRegion) {
// Prevent lazy manager creation while selecting the LOAD and data snapshot boundary.
final LoadTsFileManager loadManager =
dataRegion.getLoadTsFileManagerIfPresent().orElse(null);
if (loadManager != null) {
loadManager.lockSnapshotBoundary();
}
try {
dataRegion.writeLock("snapshotTaker");
dataRegion.syncCloseAllWorkingTsFileProcessors();
loadTerminalHistory = LoadTsFileSnapshot.captureTerminalHistory(dataRegion);
readLockTheFile();
if (flushBeforeSnapshot) {
dataRegion.writeLock("snapshotTaker");
try {
dataRegion.syncCloseAllWorkingTsFileProcessors();
} finally {
dataRegion.writeUnlock();
}
}
// A COMMIT after data-file selection must not remove staging before it is copied.
// Its pieces may predate the new replica's sync channel and never be replayed there.
success = LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir, loadTerminalHistory);
} finally {
dataRegion.writeUnlock();
if (loadManager != null) {
loadManager.unlockSnapshotBoundary();
}
}
}
success = createSnapshot(seqFiles, tempSnapshotId);
success = success && createSnapshot(seqFiles, tempSnapshotId);
success = success && createSnapshot(unseqFiles, tempSnapshotId);
success = success && snapshotCompressionRatio(snapshotDirPath);
success = success && LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir);
} finally {
readUnlockTheFile();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -173,45 +173,66 @@ private void scanRegions() {
new File(baseDir),
dataRegion.getDatabaseName(),
dataRegion.getDataRegionIdString());
scanRegion(dataRegion, regionDir);
}
}
}

private void scanRegion(final DataRegion dataRegion, final File regionDir) {
File[] taskDirs = regionDir.listFiles();
if (taskDirs == null) {
return;
}

for (final File taskDir : taskDirs) {
if (!taskDir.isDirectory()
|| LoadTsFileManager.TERMINAL_HISTORY_DIR.equals(taskDir.getName())
|| loadId2RetainedTask.containsKey(taskDir.getName())) {
continue;
}

final File[] taskDirs = regionDir.listFiles();
if (taskDirs == null) {
// A task the region still holds is in progress: the command path and the terminal path
// own it, and reading its records here would open every progress file of the directory on
// every scan to learn nothing. The idle timeout is still applied to it.
final long recordedCommandTimeMillis =
recordedCommandTimeMillis(dataRegion, taskDir.getName());
if (recordedCommandTimeMillis >= 0L) {
reclaimIfIdle(dataRegion, taskDir.getName(), taskDir, recordedCommandTimeMillis);
continue;
}

final LoadTsFileProgress.TerminalRecord terminal =
LoadTsFileProgress.readTerminalOfTaskDir(taskDir);
// A task that never reached COMMIT or ABORT belongs to a coordinator that died before it
// sent one: nothing else would ever revisit its staged data.
if (terminal == null) {
reclaimIfIdle(
dataRegion, taskDir.getName(), taskDir, lastCommandTimeMillis(dataRegion, taskDir));
continue;
}

// Do not release the only copy of a legacy terminal decision if its migration failed. This
// check also covers a manager that was initialized before this scan.
if (!LoadTsFileManager.hasTerminalHistory(
dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), taskDir.getName())) {
try {
dataRegion.getLoadTsFileManager();
} catch (final IllegalStateException e) {
return;
}
if (!LoadTsFileManager.hasTerminalHistory(
dataRegion.getDatabaseName(), dataRegion.getDataRegionIdString(), taskDir.getName())
|| loadId2RetainedTask.containsKey(taskDir.getName())) {
continue;
}
}

for (final File taskDir : taskDirs) {
if (!taskDir.isDirectory() || loadId2RetainedTask.containsKey(taskDir.getName())) {
continue;
}

// A task the region still holds is in progress: the command path and the terminal path
// own it, and reading its records here would open every progress file of the directory on
// every scan to learn nothing. The idle timeout is still applied to it.
final long recordedCommandTimeMillis =
recordedCommandTimeMillis(dataRegion, taskDir.getName());
if (recordedCommandTimeMillis >= 0L) {
reclaimIfIdle(dataRegion, taskDir.getName(), taskDir, recordedCommandTimeMillis);
continue;
}

final LoadTsFileProgress.TerminalRecord terminal =
LoadTsFileProgress.readTerminalOfTaskDir(taskDir);
// A task that never reached COMMIT or ABORT belongs to a coordinator that died before it
// sent one: nothing else would ever revisit its staged data.
if (terminal == null) {
reclaimIfIdle(
dataRegion, taskDir.getName(), taskDir, lastCommandTimeMillis(dataRegion, taskDir));
continue;
}

final RetainedTask unmanagedTask =
new RetainedTask(
taskDir.getName(), dataRegion, taskDir, terminal.searchIndex(), terminal.op());

if (canDelete(unmanagedTask)) {
delete(unmanagedTask);
}
}
final RetainedTask unmanagedTask =
new RetainedTask(
taskDir.getName(), dataRegion, taskDir, terminal.searchIndex(), terminal.op());

if (canDelete(unmanagedTask)) {
delete(unmanagedTask);
}
}
}
Expand Down
Loading
Loading