From 00f1e0429c33733a8a2e466483dc21e0122c7e0c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:37:50 +0800 Subject: [PATCH] Preserve failed active LOAD file groups (#19253) * Fix active LOAD failed-file recovery * Fix active LOAD recovery IT time-column queries * Fix active LOAD recovery test database level (cherry picked from commit ed9504f3aeb8cef403b736dd773f95091b2acad9) --- .../db/it/IoTDBLoadTsFileActiveRetryIT.java | 439 ++++++++++++++++++ .../iotdb/db/i18n/StorageEngineMessages.java | 5 + .../iotdb/db/i18n/StorageEngineMessages.java | 5 + .../load/active/ActiveLoadPendingQueue.java | 18 +- .../load/active/ActiveLoadTsFileLoader.java | 293 +++++++++++- .../active/ActiveLoadTsFileLoaderTest.java | 135 +++++- 6 files changed, 870 insertions(+), 25 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java index 3827615fe24d8..f0a39e0da1a4e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java @@ -19,6 +19,13 @@ package org.apache.iotdb.db.it; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.wal.recover.file.SealedTsFileRecoverPerformer; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -37,10 +44,20 @@ import org.junit.runner.RunWith; import java.io.File; +import java.io.InputStream; import java.nio.file.Files; import java.sql.Connection; +import java.sql.ResultSet; import java.sql.Statement; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.Set; import java.util.concurrent.TimeUnit; @RunWith(IoTDBTestRunner.class) @@ -50,6 +67,8 @@ public class IoTDBLoadTsFileActiveRetryIT { private static final String DATABASE = "root.sg.test_0"; private static final String DEVICE = DATABASE + ".d_0"; private static final String MEASUREMENT = "sensor_00"; + private static final String COLLISION_DATABASE = "root.load_p118"; + private static final String COLLISION_DEVICE = COLLISION_DATABASE + ".d0"; private static final long UNALLOCATABLE_TABLET_CONVERSION_BATCH_MEMORY_SIZE_IN_BYTES = Long.MAX_VALUE / 4; private static final MeasurementSchema TSFILE_SCHEMA = @@ -129,6 +148,426 @@ public void testActiveLoadTemporaryUnavailableShouldKeepFileForRetry() throws Ex } } + @Test + public void testPermanentFailureKeepsSameNamedTsFileGroupsRecoverable() throws Exception { + final DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); + final File sourceRoot = new File(tmpDir, "collision-sources"); + Assert.assertTrue(sourceRoot.mkdir()); + + final List sourceTsFiles = new ArrayList<>(); + final File identicalFirst = + createFailureGroup(new File(sourceRoot, "identical-first"), 1, 10, false, 1, 1); + sourceTsFiles.add(identicalFirst); + + final File identicalSecondDir = new File(sourceRoot, "identical-second"); + Assert.assertTrue(identicalSecondDir.mkdir()); + final File identicalSecond = new File(identicalSecondDir, identicalFirst.getName()); + Files.copy(identicalFirst.toPath(), identicalSecond.toPath()); + writeResource(identicalSecond); + writeModification(identicalSecond, 2); + sourceTsFiles.add(identicalSecond); + + final File sameSizeDifferentContent = + createFailureGroup( + new File(sourceRoot, "same-size-different-content"), 2, 10, false, 101, 103); + sourceTsFiles.add(sameSizeDifferentContent); + + final File differentSize = + createFailureGroup(new File(sourceRoot, "different-size"), 3, 50, true, 201, 205); + sourceTsFiles.add(differentSize); + + final List expectedTsFiles = + snapshotTsFileGroups(sourceTsFiles, new File(tmpDir, "collision-expected")); + + Assert.assertEquals(identicalFirst.length(), sameSizeDifferentContent.length()); + Assert.assertNotEquals( + -1L, Files.mismatch(identicalFirst.toPath(), sameSizeDifferentContent.toPath())); + Assert.assertNotEquals(identicalFirst.length(), differentSize.length()); + + try (Connection connection = + EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(dataNodeWrapper); + Statement statement = connection.createStatement()) { + statement.execute("create database " + COLLISION_DATABASE); + statement.execute( + String.format( + "create timeseries %s.%s %s", + COLLISION_DEVICE, MEASUREMENT, TSDataType.INT64.name())); + for (final File tsFile : sourceTsFiles) { + statement.execute( + String.format( + "load \"%s\" with ('database-level'='1','async'='true','on-success'='none'," + + "'auto-create-schema'='false','convert-on-type-mismatch'='false')", + tsFile.getAbsolutePath())); + } + } + + final File failDir = getActiveLoadFailDir(dataNodeWrapper); + final List failedGroups = + waitForCompleteFailedGroups(failDir, sourceTsFiles.size(), 60_000L); + Assert.assertEquals(sourceTsFiles.size(), failedGroups.size()); + final Map failedGroupsBySource = + assertFailedGroupsMatchSources( + expectedTsFiles, sourceTsFiles, getActiveLoadDir(dataNodeWrapper), failedGroups); + final File recoveryInputRoot = new File(tmpDir, "recovery-inputs"); + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet resultSet = + statement.executeQuery("select count(" + MEASUREMENT + ") from " + COLLISION_DEVICE)) { + Assert.assertTrue(resultSet.next()); + Assert.assertEquals("Schema mismatch failures must not load data", 0, resultSet.getLong(1)); + } + final long[] startTimes = {1, 1, 101, 201}; + final long[] endTimes = {10, 10, 110, 250}; + final long[] deletedTimes = {1, 2, 103, 205}; + final long[] expectedCounts = {9, 9, 9, 49}; + final long[] seeds = {1, 1, 2, 3}; + for (int i = 0; i < expectedTsFiles.size(); i++) { + statement.execute("delete database " + COLLISION_DATABASE); + statement.execute("create database " + COLLISION_DATABASE); + statement.execute( + String.format( + "create timeseries %s.%s %s", + COLLISION_DEVICE, MEASUREMENT, TSDataType.INT32.name())); + if (i == 3) { + statement.execute("create timeseries " + COLLISION_DEVICE + ".extra INT32"); + } + + final File failedGroup = failedGroupsBySource.get(expectedTsFiles.get(i)); + Assert.assertNotNull("Failed source group could not be located", failedGroup); + final File failedTsFile = copyFailedGroup(failedGroup, recoveryInputRoot, i); + statement.execute( + String.format( + "load \"%s\" with ('database-level'='1','on-success'='none'," + + "'auto-create-schema'='false','convert-on-type-mismatch'='false')", + failedTsFile.getAbsolutePath())); + assertCount( + statement, + COLLISION_DEVICE, + String.format("time >= %d and time <= %d", startTimes[i], endTimes[i]), + expectedCounts[i]); + assertCount(statement, COLLISION_DEVICE, "time = " + deletedTimes[i], 0); + assertRecoveredRows( + statement, startTimes[i], endTimes[i], deletedTimes[i], seeds[i], i == 3); + } + + final File normalTsFile = new File(tmpDir, "normal-after-failure.tsfile"); + try (final TsFileGenerator generator = new TsFileGenerator(normalTsFile)) { + generator.registerTimeseries(COLLISION_DEVICE, Collections.singletonList(TSFILE_SCHEMA)); + generator.generateData(COLLISION_DEVICE, 10, 1, false, 300); + } + try (Connection activeConnection = + EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(dataNodeWrapper); + Statement activeStatement = activeConnection.createStatement()) { + activeStatement.execute( + String.format( + "load \"%s\" with ('database-level'='1','async'='true','on-success'='none'," + + "'auto-create-schema'='false','convert-on-type-mismatch'='false')", + normalTsFile.getAbsolutePath())); + } + TestUtils.assertDataEventuallyOnEnv( + EnvFactory.getEnv(), + String.format( + "select count(%s) from %s where time >= 301 and time <= 310", + MEASUREMENT, COLLISION_DEVICE), + "count(" + COLLISION_DEVICE + "." + MEASUREMENT + "),", + Collections.singleton("10,")); + } + } + + private File createFailureGroup( + final File groupDir, + final long randomSeed, + final int rowCount, + final boolean withExtraMeasurement, + final long startTimestamp, + final long deletionTimestamp) + throws Exception { + Assert.assertTrue(groupDir.mkdir()); + final File tsFile = new File(groupDir, "1-0-0-0.tsfile"); + final List schemas = new ArrayList<>(); + schemas.add(TSFILE_SCHEMA); + if (withExtraMeasurement) { + schemas.add(new MeasurementSchema("extra", TSDataType.INT32, TSEncoding.RLE)); + } + try (final TsFileGenerator generator = new TsFileGenerator(tsFile)) { + generator.resetRandom(randomSeed); + generator.registerTimeseries(COLLISION_DEVICE, schemas); + generator.generateData(COLLISION_DEVICE, rowCount, 1, false, startTimestamp - 1); + } + writeResource(tsFile); + writeModification(tsFile, deletionTimestamp); + return tsFile; + } + + private void writeResource(final File tsFile) throws Exception { + final TsFileResource resource = new TsFileResource(tsFile); + try (final SealedTsFileRecoverPerformer performer = + new SealedTsFileRecoverPerformer(resource)) { + performer.recover(); + } + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + resource.deserialize(); + resource.serialize(); + } + + private void writeModification(final File tsFile, final long timestamp) throws Exception { + try (final ModificationFile modificationFile = + new ModificationFile(tsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX)) { + modificationFile.write( + new Deletion( + new PartialPath(COLLISION_DEVICE + "." + MEASUREMENT), + Long.MAX_VALUE, + timestamp, + timestamp)); + } + } + + private List waitForCompleteFailedGroups( + final File failDir, final int expectedCount, final long timeoutMs) + throws InterruptedException { + final long deadline = System.currentTimeMillis() + timeoutMs; + List groups; + do { + groups = findFailedGroups(failDir); + if (groups.size() >= expectedCount + && groups.stream().allMatch(this::hasCompleteFailedGroup)) { + return groups; + } + if (System.currentTimeMillis() >= deadline) { + return groups; + } + Thread.sleep(250L); + } while (true); + } + + private boolean hasCompleteFailedGroup(final File groupDir) { + final File tsFile = findTsFile(groupDir); + return tsFile != null + && new File(tsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).isFile() + && new File(tsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX).isFile(); + } + + private List findFailedGroups(final File failDir) { + final List groups = new ArrayList<>(); + if (findTsFile(failDir) != null) { + groups.add(failDir); + } + if (failDir == null || !failDir.isDirectory()) { + return groups; + } + final File[] children = failDir.listFiles(); + if (children == null) { + return groups; + } + for (final File child : children) { + if (child.isDirectory() + && !child.getName().startsWith(".active-load-staging-") + && findTsFile(child) != null) { + groups.add(child); + } + } + return groups; + } + + private File findTsFile(final File directory) { + if (directory == null || !directory.isDirectory()) { + return null; + } + final File[] children = directory.listFiles(); + if (children == null) { + return null; + } + for (final File child : children) { + if (child.isFile() && child.getName().endsWith(".tsfile")) { + return child; + } + } + return null; + } + + private Map assertFailedGroupsMatchSources( + final List expectedTsFiles, + final List submittedTsFiles, + final File activeLoadDir, + final List failedGroups) + throws Exception { + Assert.assertEquals(expectedTsFiles.size(), submittedTsFiles.size()); + final Set unmatchedGroups = new HashSet<>(failedGroups); + final Map groupsBySource = new HashMap<>(); + final Set activeLoadFiles = new HashSet<>(); + for (int sourceIndex = 0; sourceIndex < expectedTsFiles.size(); sourceIndex++) { + final File expectedTsFile = expectedTsFiles.get(sourceIndex); + final File submittedTsFile = submittedTsFiles.get(sourceIndex); + Assert.assertTrue( + "Expected TsFile snapshot differs from submitted source", + sameBytes(expectedTsFile, submittedTsFile)); + Assert.assertTrue( + "Expected resource snapshot differs from submitted source", + sameBytes( + new File(expectedTsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX), + new File(submittedTsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX))); + Assert.assertTrue( + "Expected mods snapshot differs from submitted source", + sameBytes( + new File(expectedTsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX), + new File(submittedTsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX))); + File matchedGroup = null; + for (final File groupDir : unmatchedGroups) { + final File failedTsFile = findTsFile(groupDir); + if (failedTsFile != null && sameBytes(expectedTsFile, failedTsFile)) { + final File sourceResource = + new File(expectedTsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX); + final File failedResource = + new File(failedTsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX); + final File sourceMods = + new File(expectedTsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX); + final File failedMods = + new File(failedTsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX); + if (sourceResource.exists() + && failedResource.exists() + && sourceMods.exists() + && failedMods.exists() + && sameBytes(sourceResource, failedResource) + && sameBytes(sourceMods, failedMods)) { + final File failureInfoFile = + new File(groupDir, failedTsFile.getName() + ".load-failure"); + if (!failureInfoFile.isFile()) { + continue; + } + final Properties failureInfo = new Properties(); + try (InputStream inputStream = Files.newInputStream(failureInfoFile.toPath())) { + failureInfo.load(inputStream); + } + final String activeLoadFile = failureInfo.getProperty("activeLoadFile"); + final String pendingDir = failureInfo.getProperty("pendingDir"); + if (activeLoadFile == null + || !new File(activeLoadFile).getName().equals(failedTsFile.getName()) + || !new File(activeLoadFile) + .toPath() + .toAbsolutePath() + .normalize() + .startsWith(activeLoadDir.toPath().toAbsolutePath().normalize()) + || pendingDir == null + || !new File(pendingDir) + .toPath() + .toAbsolutePath() + .normalize() + .equals(activeLoadDir.toPath().toAbsolutePath().normalize()) + || Boolean.parseBoolean(failureInfo.getProperty("generatedByPipe", "true")) + || Boolean.parseBoolean(failureInfo.getProperty("tableModel", "true")) + || !activeLoadFiles.add(activeLoadFile)) { + continue; + } + matchedGroup = groupDir; + break; + } + } + } + Assert.assertNotNull("Failed directory lost or cross-paired a TsFile group", matchedGroup); + unmatchedGroups.remove(matchedGroup); + groupsBySource.put(expectedTsFile, matchedGroup); + } + Assert.assertTrue("Unexpected failed file groups remain", unmatchedGroups.isEmpty()); + return groupsBySource; + } + + private boolean sameBytes(final File left, final File right) throws Exception { + return Files.mismatch(left.toPath(), right.toPath()) == -1; + } + + private List snapshotTsFileGroups(final List sourceTsFiles, final File snapshotRoot) + throws Exception { + final List snapshots = new ArrayList<>(); + for (int i = 0; i < sourceTsFiles.size(); i++) { + final File sourceTsFile = sourceTsFiles.get(i); + final File groupDir = new File(snapshotRoot, Integer.toString(i)); + Assert.assertTrue(groupDir.mkdirs()); + final File snapshotTsFile = new File(groupDir, sourceTsFile.getName()); + Files.copy(sourceTsFile.toPath(), snapshotTsFile.toPath()); + final File sourceResource = + new File(sourceTsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX); + final File sourceMods = + new File(sourceTsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX); + Files.copy( + sourceResource.toPath(), + new File(snapshotTsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).toPath()); + Files.copy( + sourceMods.toPath(), + new File(snapshotTsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX).toPath()); + snapshots.add(snapshotTsFile); + } + return snapshots; + } + + private File copyFailedGroup(final File failedGroup, final File recoveryRoot, final int index) + throws Exception { + final File recoveryGroup = new File(recoveryRoot, Integer.toString(index)); + Assert.assertTrue(recoveryGroup.mkdirs()); + final File[] files = failedGroup.listFiles(File::isFile); + Assert.assertNotNull(files); + for (final File file : files) { + Files.copy(file.toPath(), new File(recoveryGroup, file.getName()).toPath()); + } + final File recoveryTsFile = findTsFile(recoveryGroup); + Assert.assertNotNull("Failed group copy must contain its TsFile", recoveryTsFile); + return recoveryTsFile; + } + + private void assertCount( + final Statement statement, final String device, final String filter, final long expected) + throws Exception { + try (ResultSet resultSet = + statement.executeQuery( + String.format("select count(%s) from %s where %s", MEASUREMENT, device, filter))) { + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(expected, resultSet.getLong(1)); + } + } + + private void assertRecoveredRows( + final Statement statement, + final long startTime, + final long endTime, + final long deletedTime, + final long seed, + final boolean withExtraMeasurement) + throws Exception { + final Random expectedValues = new Random(seed); + try (ResultSet resultSet = + statement.executeQuery( + String.format("select %s from %s order by time", MEASUREMENT, COLLISION_DEVICE))) { + for (long time = startTime; time <= endTime; time++) { + final int value = expectedValues.nextInt(); + if (withExtraMeasurement) { + expectedValues.nextInt(); + } + if (time == deletedTime) { + continue; + } + Assert.assertTrue("Recovered group lost a surviving key", resultSet.next()); + Assert.assertEquals(time, resultSet.getLong(1)); + Assert.assertEquals("Recovered value differs from its source", value, resultSet.getInt(2)); + Assert.assertFalse(resultSet.wasNull()); + } + Assert.assertFalse(resultSet.next()); + } + if (withExtraMeasurement) { + final Random expectedExtraValues = new Random(seed); + try (ResultSet resultSet = + statement.executeQuery("select extra from " + COLLISION_DEVICE + " order by time")) { + for (long time = startTime; time <= endTime; time++) { + expectedExtraValues.nextInt(); + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(time, resultSet.getLong(1)); + Assert.assertEquals(expectedExtraValues.nextInt(), resultSet.getInt(2)); + Assert.assertFalse(resultSet.wasNull()); + } + Assert.assertFalse(resultSet.next()); + } + } + } + private void generateTsFile(final File tsFile) throws Exception { try (final TsFileGenerator generator = new TsFileGenerator(tsFile)) { generator.registerTimeseries(DEVICE, Collections.singletonList(TSFILE_SCHEMA)); diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java index d799152d9b5fc..fda7df243c5b1 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -23,6 +23,11 @@ public final class StorageEngineMessages { private StorageEngineMessages() {} + public static final String LOG_ERROR_OCCURRED_DURING_MOVING_FILE_ARG_TO_FAIL_DIRECTORY_F40610CE = + "Error occurred during moving file {} to fail directory."; + public static final String LOG_FAILED_TO_READ_ACTIVE_LOAD_FAILURE_METADATA_ARG_E152544E = + "Failed to read active LOAD failure metadata {}"; + public static final String LOG_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_ARG_TARGET_ARG_13205CEC = "File renaming failed when loading tsfile. Origin: {}, Target: {}"; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java index 6f43d0b34c8ce..6e20a9ede2c5b 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -23,6 +23,11 @@ public final class StorageEngineMessages { private StorageEngineMessages() {} + public static final String LOG_ERROR_OCCURRED_DURING_MOVING_FILE_ARG_TO_FAIL_DIRECTORY_F40610CE = + "将文件 {} 移动到失败目录时发生错误。"; + public static final String LOG_FAILED_TO_READ_ACTIVE_LOAD_FAILURE_METADATA_ARG_E152544E = + "读取主动 LOAD 失败元数据 {} 失败"; + public static final String LOG_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_ARG_TARGET_ARG_13205CEC = "加载 TsFile 时重命名文件失败。源文件:{},目标文件:{}"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPendingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPendingQueue.java index 3275ba8368a67..718d99d9e751b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPendingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPendingQueue.java @@ -32,6 +32,7 @@ public class ActiveLoadPendingQueue { private final Queue pendingFileQueue = new ConcurrentLinkedQueue<>(); private final Set loadingFileSet = new HashSet<>(); + private final Set failedFileSet = new HashSet<>(); public synchronized boolean enqueue( final String file, final String pendingDir, final boolean isGeneratedByPipe) { @@ -43,7 +44,9 @@ public synchronized boolean enqueue( final String pendingDir, final boolean isGeneratedByPipe, final String conversionTaskId) { - if (!loadingFileSet.contains(file) && pendingFileSet.add(file)) { + if (!loadingFileSet.contains(file) + && !failedFileSet.contains(file) + && pendingFileSet.add(file)) { pendingFileQueue.offer( new ActiveLoadEntry(file, pendingDir, isGeneratedByPipe, conversionTaskId)); @@ -72,7 +75,17 @@ public synchronized void removeFromLoading(final String file) { } public synchronized boolean isFilePendingOrLoading(final String file) { - return loadingFileSet.contains(file) || pendingFileSet.contains(file); + return loadingFileSet.contains(file) + || pendingFileSet.contains(file) + || failedFileSet.contains(file); + } + + public synchronized void markFileFailed(final String file) { + failedFileSet.add(file); + } + + public synchronized void clearFailedFile(final String file) { + failedFileSet.remove(file); } public synchronized int size() { @@ -94,6 +107,7 @@ public synchronized void clearPending() { final int pendingFileCount = pendingFileSet.size(); pendingFileSet.clear(); pendingFileQueue.clear(); + failedFileSet.clear(); ActiveLoadingFilesNumberMetricsSet.getInstance().increaseQueuingFileCounter(-pendingFileCount); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java index 2c98f51b64e49..3ec85f87c2e15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.protocol.session.InternalClientSession; import org.apache.iotdb.db.protocol.session.SessionManager; @@ -38,6 +39,8 @@ import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.load.converter.PipeTsFileConversionTaskManager; import org.apache.iotdb.db.storageengine.load.metrics.ActiveLoadingFilesNumberMetricsSet; import org.apache.iotdb.db.storageengine.load.metrics.ActiveLoadingFilesSizeMetricsSet; @@ -50,15 +53,24 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; import java.nio.file.attribute.BasicFileAttributes; import java.time.ZoneId; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -80,6 +92,8 @@ public class ActiveLoadTsFileLoader { // Guarded by activeLoadExecutor, including tasks submitted but not yet started. private int activeLoadTasks; private final AtomicReference failDir = new AtomicReference<>(); + private final AtomicReference indexedFailDir = new AtomicReference<>(); + private final Map failedFileSnapshots = new ConcurrentHashMap<>(); private final boolean isVerify = IOTDB_CONFIG.isLoadActiveListeningVerifyEnable(); public int getCurrentAllowedPendingSize() { @@ -173,6 +187,7 @@ public void stop() { } if (executor == null) { pendingQueue.clearPending(); + clearFailedFileIndex(); return; } @@ -194,6 +209,7 @@ public void stop() { } else { pendingQueue.clearPending(); } + clearFailedFileIndex(); } } @@ -211,6 +227,11 @@ private void tryLoadPendingTsFiles(final WrappedThreadPoolExecutor executor) { } } + private void clearFailedFileIndex() { + failedFileSnapshots.clear(); + indexedFailDir.set(null); + } + private void tryLoadPendingTsFiles() { final IClientSession session = new InternalClientSession( @@ -350,7 +371,7 @@ private void handleLoadFailure( entry.getFile(), entry.isGeneratedByPipe(), status); - removeFileAndResourceAndModsToFailDir(entry.getFile()); + removeFileAndResourceAndModsToFailDir(entry); } } @@ -361,7 +382,7 @@ private void handleFileNotFoundException(final ActiveLoadPendingQueue.ActiveLoad "Failed to auto load tsfile {} (isGeneratedByPipe = {}) due to file not found, will skip this file.", entry.getFile(), entry.isGeneratedByPipe()); - removeFileAndResourceAndModsToFailDir(entry.getFile()); + removeFileAndResourceAndModsToFailDir(entry); } private void handleOtherException( @@ -380,32 +401,146 @@ private void handleOtherException( entry.getFile(), entry.isGeneratedByPipe(), e); - removeFileAndResourceAndModsToFailDir(entry.getFile()); + removeFileAndResourceAndModsToFailDir(entry); } } - private void removeFileAndResourceAndModsToFailDir(final String filePath) { - removeToFailDir(filePath); - removeToFailDir(filePath + ".resource"); - removeToFailDir(filePath + ".mods"); - } + private synchronized void removeFileAndResourceAndModsToFailDir( + final ActiveLoadPendingQueue.ActiveLoadEntry entry) { + final String filePath = entry.getFile(); + final File tsFile = new File(filePath); + final List sourceFiles = + List.of( + tsFile, + new File(filePath + TsFileResource.RESOURCE_SUFFIX), + new File(filePath + ModificationFile.FILE_SUFFIX)); + final boolean hasSourceFiles = sourceFiles.stream().anyMatch(File::exists); + if (!hasSourceFiles) { + return; + } + + final File failDirectory = new File(failDir.get()); + final String failureInfoSuffix = ".load-failure"; + final String failureInfoFileName = + tsFile.getName().getBytes(StandardCharsets.UTF_8).length + failureInfoSuffix.length() > 180 + ? "active-load-" + UUID.randomUUID() + failureInfoSuffix + : tsFile.getName() + failureInfoSuffix; + File stagingDirectory = null; + final File targetDirectory = new File(failDirectory, "active-load-" + UUID.randomUUID()); + try { + FileUtils.forceMkdir(failDirectory); + stagingDirectory = + Files.createTempDirectory(failDirectory.toPath(), ".active-load-staging-").toFile(); + + for (final File sourceFile : sourceFiles) { + if (!sourceFile.exists()) { + continue; + } + Files.copy( + sourceFile.toPath(), + new File(stagingDirectory, sourceFile.getName()).toPath(), + StandardCopyOption.COPY_ATTRIBUTES); + } + writeFailureInfo(new File(stagingDirectory, failureInfoFileName), entry, true); - private void removeToFailDir(final String filePath) { - final File sourceFile = new File(filePath); - // prevent the resource or mods not exist - if (!sourceFile.exists()) { + try { + Files.move( + stagingDirectory.toPath(), targetDirectory.toPath(), StandardCopyOption.ATOMIC_MOVE); + } catch (final AtomicMoveNotSupportedException e) { + Files.move(stagingDirectory.toPath(), targetDirectory.toPath()); + } + stagingDirectory = null; + } catch (final IOException e) { + if (targetDirectory.exists()) { + try { + FileUtils.deleteDirectory(targetDirectory); + } catch (final IOException cleanupException) { + e.addSuppressed(cleanupException); + } + } + if (stagingDirectory != null && stagingDirectory.exists()) { + try { + FileUtils.deleteDirectory(stagingDirectory); + } catch (final IOException cleanupException) { + e.addSuppressed(cleanupException); + } + } + LOGGER.warn( + StorageEngineMessages + .LOG_ERROR_OCCURRED_DURING_MOVING_FILE_ARG_TO_FAIL_DIRECTORY_F40610CE, + filePath, + e); return; } - final File targetDir = new File(failDir.get()); + deletePublishedSourceFiles(filePath, tsFile, sourceFiles, targetDirectory); + } + + private void deletePublishedSourceFiles( + final String filePath, + final File tsFile, + final List sourceFiles, + final File publishedGroup) { try { - RetryUtils.retryOnException( - () -> { - org.apache.iotdb.commons.utils.FileUtils.moveFileWithMD5Check(sourceFile, targetDir); - return null; - }); + Files.deleteIfExists(tsFile.toPath()); } catch (final IOException e) { - LOGGER.warn("Error occurred during moving file {} to fail directory.", filePath, e); + if (tsFile.exists()) { + pendingQueue.markFileFailed(filePath); + failedFileSnapshots.put(filePath, new FailedFileSnapshot(publishedGroup)); + } + LOGGER.warn( + StorageEngineMessages + .LOG_ERROR_OCCURRED_DURING_MOVING_FILE_ARG_TO_FAIL_DIRECTORY_F40610CE, + filePath, + e); + return; + } + + IOException cleanupException = null; + for (final File sourceFile : sourceFiles) { + if (sourceFile.equals(tsFile)) { + continue; + } + try { + Files.deleteIfExists(sourceFile.toPath()); + } catch (final IOException e) { + if (cleanupException == null) { + cleanupException = e; + } else { + cleanupException.addSuppressed(e); + } + } + } + if (cleanupException != null) { + LOGGER.warn( + StorageEngineMessages + .LOG_ERROR_OCCURRED_DURING_MOVING_FILE_ARG_TO_FAIL_DIRECTORY_F40610CE, + filePath, + cleanupException); + } + } + + private void writeFailureInfo( + final File failureInfoFile, + final ActiveLoadPendingQueue.ActiveLoadEntry entry, + final boolean hasFiles) + throws IOException { + if (!hasFiles) { + return; + } + + final Properties properties = new Properties(); + properties.setProperty("activeLoadFile", entry.getFile()); + properties.setProperty("generatedByPipe", Boolean.toString(entry.isGeneratedByPipe())); + properties.setProperty("tableModel", "false"); + if (entry.getConversionTaskId() != null) { + properties.setProperty("conversionTaskId", entry.getConversionTaskId()); + } + if (entry.getPendingDir() != null) { + properties.setProperty("pendingDir", entry.getPendingDir()); + } + try (final OutputStream outputStream = Files.newOutputStream(failureInfoFile.toPath())) { + properties.store(outputStream, "Active LOAD failure source"); } } @@ -436,7 +571,125 @@ private void cleanupEmptyDirectories(final ActiveLoadPendingQueue.ActiveLoadEntr } public boolean isFilePendingOrLoading(final File file) { - return pendingQueue.isFilePendingOrLoading(file.getAbsolutePath()); + final String filePath = file.getAbsolutePath(); + if (!file.exists()) { + pendingQueue.clearFailedFile(filePath); + failedFileSnapshots.remove(filePath); + } else { + indexPublishedFailedFilesIfNecessary(); + final FailedFileSnapshot failedFileSnapshot = failedFileSnapshots.get(filePath); + if (failedFileSnapshot != null && !failedFileSnapshot.matches(file)) { + pendingQueue.clearFailedFile(filePath); + failedFileSnapshots.remove(filePath, failedFileSnapshot); + } + } + return pendingQueue.isFilePendingOrLoading(filePath); + } + + private void indexPublishedFailedFilesIfNecessary() { + final String configuredFailDir = IOTDB_CONFIG.getLoadActiveListeningFailDir(); + if (configuredFailDir == null || configuredFailDir.isEmpty()) { + return; + } + if (Objects.equals(indexedFailDir.get(), configuredFailDir)) { + return; + } + + synchronized (indexedFailDir) { + if (Objects.equals(indexedFailDir.get(), configuredFailDir)) { + return; + } + + final File[] failedGroups = new File(configuredFailDir).listFiles(File::isDirectory); + if (failedGroups == null) { + return; + } + + boolean scanCompleted = true; + for (final File failedGroup : failedGroups) { + if (!failedGroup.getName().startsWith("active-load-")) { + continue; + } + final File[] failureInfoFiles = + failedGroup.listFiles((dir, name) -> name.endsWith(".load-failure")); + if (failureInfoFiles == null) { + scanCompleted = false; + continue; + } + for (final File failureInfoFile : failureInfoFiles) { + final Properties failureInfo = new Properties(); + try (final InputStream inputStream = Files.newInputStream(failureInfoFile.toPath())) { + failureInfo.load(inputStream); + final String activeLoadFile = failureInfo.getProperty("activeLoadFile"); + if (activeLoadFile == null) { + continue; + } + final String absoluteSourceFile = new File(activeLoadFile).getAbsolutePath(); + final File sourceTsFile = new File(absoluteSourceFile); + if (sourceTsFile.isFile() && matchesPublishedFileGroup(sourceTsFile, failedGroup)) { + pendingQueue.markFileFailed(absoluteSourceFile); + failedFileSnapshots.put(absoluteSourceFile, new FailedFileSnapshot(failedGroup)); + } + } catch (final IOException e) { + scanCompleted = false; + LOGGER.debug( + StorageEngineMessages.LOG_FAILED_TO_READ_ACTIVE_LOAD_FAILURE_METADATA_ARG_E152544E, + failureInfoFile, + e); + } catch (final RuntimeException e) { + LOGGER.debug( + StorageEngineMessages.LOG_FAILED_TO_READ_ACTIVE_LOAD_FAILURE_METADATA_ARG_E152544E, + failureInfoFile, + e); + } + } + } + if (scanCompleted) { + indexedFailDir.set(configuredFailDir); + } + } + } + + private static boolean matchesPublishedFileGroup( + final File sourceTsFile, final File failedGroup) { + for (final File sourceFile : getTsFileGroupFiles(sourceTsFile)) { + final File failedFile = new File(failedGroup, sourceFile.getName()); + if (sourceFile.exists() != failedFile.exists()) { + return false; + } + if (sourceFile.exists()) { + try { + if (Files.mismatch(sourceFile.toPath(), failedFile.toPath()) != -1) { + return false; + } + } catch (final IOException e) { + return false; + } + } + } + return true; + } + + private static List getTsFileGroupFiles(final File tsFile) { + final String filePath = tsFile.getAbsolutePath(); + return List.of( + tsFile, + new File(filePath + TsFileResource.RESOURCE_SUFFIX), + new File(filePath + ModificationFile.FILE_SUFFIX)); + } + + private static class FailedFileSnapshot { + private final File publishedGroup; + + private FailedFileSnapshot(final File publishedGroup) { + this.publishedGroup = publishedGroup; + } + + private boolean matches(final File tsFile) { + // A replacement can preserve both size and mtime (for example, a restored sidecar). + // Only unchanged contents should remain suppressed. + return matchesPublishedFileGroup(tsFile, publishedGroup); + } } // Metrics diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoaderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoaderTest.java index 34986ebbc360b..8f8671071ec5b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoaderTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoaderTest.java @@ -40,8 +40,10 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; import java.nio.file.Files; +import java.nio.file.attribute.FileTime; import java.util.ArrayList; import java.util.List; +import java.util.Properties; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -121,10 +123,61 @@ public void testPermanentFailureStatusMovesFileToFailDir() throws Exception { final File failDir = new File(config.getLoadActiveListeningFailDir()); Assert.assertFalse(tsFile.exists()); - Assert.assertTrue(new File(failDir, tsFile.getName()).exists()); + final File[] failedGroups = failDir.listFiles(File::isDirectory); + Assert.assertNotNull(failedGroups); + Assert.assertEquals(1, failedGroups.length); + final File failedGroup = failedGroups[0]; + Assert.assertTrue(new File(failedGroup, tsFile.getName()).exists()); Assert.assertTrue( - new File(failDir, tsFile.getName() + TsFileResource.RESOURCE_SUFFIX).exists()); - Assert.assertTrue(new File(failDir, tsFile.getName() + ModificationFile.FILE_SUFFIX).exists()); + new File(failedGroup, tsFile.getName() + TsFileResource.RESOURCE_SUFFIX).exists()); + Assert.assertTrue( + new File(failedGroup, tsFile.getName() + ModificationFile.FILE_SUFFIX).exists()); + Assert.assertTrue(new File(failedGroup, tsFile.getName() + ".load-failure").exists()); + final Properties failureInfo = new Properties(); + try (java.io.InputStream inputStream = + Files.newInputStream(new File(failedGroup, tsFile.getName() + ".load-failure").toPath())) { + failureInfo.load(inputStream); + } + Assert.assertEquals(tsFile.getAbsolutePath(), failureInfo.getProperty("activeLoadFile")); + Assert.assertEquals(tempDir.getAbsolutePath(), failureInfo.getProperty("pendingDir")); + Assert.assertEquals("false", failureInfo.getProperty("generatedByPipe")); + Assert.assertEquals("false", failureInfo.getProperty("tableModel")); + } + + @Test + public void testFailurePublicationPreservesSourcesWhenFailDirectoryIsUnavailable() + throws Exception { + final ActiveLoadTsFileLoader loader = new ActiveLoadTsFileLoader(); + final File tsFile = createTsFileWithCompanionFiles("unpublished.tsfile"); + final ActiveLoadPendingQueue.ActiveLoadEntry entry = + new ActiveLoadPendingQueue.ActiveLoadEntry( + tsFile.getAbsolutePath(), tempDir.getAbsolutePath(), false); + invokeInitFailDirIfNecessary(loader); + final File failDir = new File(config.getLoadActiveListeningFailDir()); + Files.delete(failDir.toPath()); + Assert.assertTrue(failDir.createNewFile()); + + invokeHandleLoadFailure( + loader, + entry, + new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage("permanent error")); + for (final String suffix : + new String[] {"", TsFileResource.RESOURCE_SUFFIX, ModificationFile.FILE_SUFFIX}) { + Assert.assertTrue( + "An unpublished group must retain every source", + new File(tsFile.getAbsolutePath() + suffix).exists()); + } + Assert.assertFalse( + "Publication failure must remain retryable", loader.isFilePendingOrLoading(tsFile)); + + Files.delete(failDir.toPath()); + Assert.assertTrue(failDir.mkdir()); + invokeHandleLoadFailure( + loader, + entry, + new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage("permanent error")); + Assert.assertFalse(tsFile.exists()); + Assert.assertEquals(1, failDir.listFiles(File::isDirectory).length); } @Test @@ -146,6 +199,82 @@ public void testStopClearsPendingFilesForRestart() throws Exception { Assert.assertTrue(pendingQueue.isEmpty()); } + @Test + public void testFailedFileIsNotRequeuedUntilPendingStateIsCleared() { + final ActiveLoadPendingQueue pendingQueue = new ActiveLoadPendingQueue(); + final String tsFilePath = new File(tempDir, "failed-pending.tsfile").getAbsolutePath(); + Assert.assertTrue(pendingQueue.enqueue(tsFilePath, tempDir.getAbsolutePath(), false)); + Assert.assertNotNull(pendingQueue.dequeueFromPending()); + + pendingQueue.markFileFailed(tsFilePath); + pendingQueue.removeFromLoading(tsFilePath); + + Assert.assertTrue(pendingQueue.isFilePendingOrLoading(tsFilePath)); + Assert.assertFalse(pendingQueue.enqueue(tsFilePath, tempDir.getAbsolutePath(), false)); + pendingQueue.clearPending(); + Assert.assertFalse(pendingQueue.isFilePendingOrLoading(tsFilePath)); + } + + @Test + public void testPublishedFailureGroupSuppressesUnchangedSourceAfterLoaderRestart() + throws Exception { + final File tsFile = createTsFileWithCompanionFiles("restart-failed.tsfile"); + final File resourceFile = new File(tsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX); + Files.writeString(resourceFile.toPath(), "original resource"); + final File failedGroup = writePublishedFailureGroup(tsFile, "active-load-recovered"); + + final ActiveLoadTsFileLoader restartedLoader = new ActiveLoadTsFileLoader(); + Assert.assertTrue( + "A published failure group must prevent an unchanged active source from being loaded again", + restartedLoader.isFilePendingOrLoading(tsFile)); + + final FileTime originalModifiedTime = Files.getLastModifiedTime(resourceFile.toPath()); + Files.writeString(resourceFile.toPath(), "replaced resource"); + Files.setLastModifiedTime(resourceFile.toPath(), originalModifiedTime); + final File publishedResource = new File(failedGroup, resourceFile.getName()); + Assert.assertNotEquals(-1L, Files.mismatch(resourceFile.toPath(), publishedResource.toPath())); + Assert.assertEquals(publishedResource.length(), resourceFile.length()); + Assert.assertEquals(originalModifiedTime, Files.getLastModifiedTime(resourceFile.toPath())); + Assert.assertFalse( + "A same-sized replacement preserving mtime must be eligible for loading", + restartedLoader.isFilePendingOrLoading(tsFile)); + } + + @Test + public void testFailureIndexRetriesWhenFailDirectoryBecomesAvailable() throws Exception { + final File tsFile = createTsFileWithCompanionFiles("late-fail-dir.tsfile"); + final File failDir = new File(config.getLoadActiveListeningFailDir()); + final ActiveLoadTsFileLoader restartedLoader = new ActiveLoadTsFileLoader(); + + Assert.assertFalse("The fail directory is absent at the first scan", failDir.exists()); + Assert.assertFalse(restartedLoader.isFilePendingOrLoading(tsFile)); + + Assert.assertTrue(failDir.mkdirs()); + writePublishedFailureGroup(tsFile, "active-load-late"); + + Assert.assertTrue( + "A failed index scan must retry after the fail directory becomes available", + restartedLoader.isFilePendingOrLoading(tsFile)); + } + + private File writePublishedFailureGroup(final File tsFile, final String groupName) + throws Exception { + final File failedGroup = new File(config.getLoadActiveListeningFailDir(), groupName); + Assert.assertTrue(failedGroup.mkdirs()); + for (final String suffix : + new String[] {"", TsFileResource.RESOURCE_SUFFIX, ModificationFile.FILE_SUFFIX}) { + final File sourceFile = new File(tsFile.getAbsolutePath() + suffix); + Files.copy(sourceFile.toPath(), new File(failedGroup, sourceFile.getName()).toPath()); + } + final Properties failureInfo = new Properties(); + failureInfo.setProperty("activeLoadFile", tsFile.getAbsolutePath()); + try (java.io.OutputStream outputStream = + Files.newOutputStream(new File(failedGroup, tsFile.getName() + ".load-failure").toPath())) { + failureInfo.store(outputStream, "test failure metadata"); + } + return failedGroup; + } + @Test public void testPendingEntryRetainsConversionTaskId() { final ActiveLoadPendingQueue pendingQueue = new ActiveLoadPendingQueue();