From 444f5b544e199202c5abb54f0cc5c524da1469e6 Mon Sep 17 00:00:00 2001 From: HTHou Date: Mon, 16 Nov 2026 11:00:00 +0800 Subject: [PATCH 1/4] [WAL] Clean up failed WAL creation and make checkpoint rolls atomic --- .../wal/checkpoint/CheckpointManager.java | 135 ++++---- .../dataregion/wal/io/CheckpointWriter.java | 24 ++ .../dataregion/wal/io/LogWriter.java | 61 +++- .../wal/recover/CheckpointRecoverUtils.java | 20 +- .../CheckpointRollRecoveryTest.java | 304 ++++++++++++++++++ .../wal/io/WALFormatExtensionTest.java | 101 +++++- 6 files changed, 559 insertions(+), 86 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointRollRecoveryTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java index ec96f43c7d401..5f734cb2ab2a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java @@ -38,6 +38,8 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -75,6 +77,8 @@ public class CheckpointManager implements AutoCloseable { private long currentCheckPointFileVersion = 0; // current checkpoint file log writer, only updated by fsyncAndDeleteThread private ILogWriter currentLogWriter; + // size of the file before any incremental checkpoints are appended + private long initialCheckpointSize; // endregion @@ -85,11 +89,13 @@ public CheckpointManager(String identifier, String logDirectory) throws IOExcept if (!logDirFile.exists() && logDirFile.mkdirs()) { logger.info(StorageEngineMessages.CREATE_FOLDER_FOR_WAL_BUFFER, logDirectory, identifier); } + File initialFile = + SystemFileFactory.INSTANCE.getFile( + logDirectory, CheckpointFileUtils.getLogFileName(currentCheckPointFileVersion)); currentLogWriter = - new CheckpointWriter( - SystemFileFactory.INSTANCE.getFile( - logDirectory, CheckpointFileUtils.getLogFileName(currentCheckPointFileVersion))); - logHeader(); + initializeLogWriter( + initialFile, Files.exists(initialFile.toPath(), LinkOption.NOFOLLOW_LINKS)); + initialCheckpointSize = currentLogWriter.size(); } public List activeOrPinnedMemTables() { @@ -101,36 +107,15 @@ public List activeOrPinnedMemTables() { } } - private void logHeader() { - infoLock.lock(); - try { - // log max memTable id - ByteBuffer tmpBuffer = ByteBuffer.allocate(Long.BYTES); - tmpBuffer.putLong(maxMemTableId); - try { - currentLogWriter.write(tmpBuffer); - } catch (IOException e) { - logger.error(StorageEngineMessages.FAIL_TO_LOG_MAX_MEMTABLE_ID, maxMemTableId, e); - DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); - } - // log global memTables' info - makeGlobalInfoCP(); - } finally { - infoLock.unlock(); - } - } - - /** - * Make checkpoint for global memTables info, this checkpoint only exists in the beginning of each - * checkpoint file. - */ - private void makeGlobalInfoCP() { + /** Writes the recovery header and global snapshot while the checkpoint state is stable. */ + private void writeInitialCheckpoint(ILogWriter writer) throws IOException { + writer.write(ByteBuffer.allocate(Long.BYTES).putLong(maxMemTableId)); long start = System.nanoTime(); List memTableInfos = activeOrPinnedMemTables(); memTableInfos.removeIf(MemTableInfo::isFlushed); Checkpoint checkpoint = new Checkpoint(CheckpointType.GLOBAL_MEMORY_TABLE_INFO, memTableInfos); - logByCachedByteBuffer(checkpoint); - fsyncCheckpointFile(); + writeCheckpoint(writer, checkpoint); + writer.force(); WRITING_METRICS.recordMakeCheckpointCost(checkpoint.getType(), System.nanoTime() - start); } @@ -191,18 +176,23 @@ public void makeFlushMemTableCP(long memTableId) { } private void logByCachedByteBuffer(Checkpoint checkpoint) { + try { + writeCheckpoint(currentLogWriter, checkpoint); + } catch (IOException e) { + logger.error(StorageEngineMessages.FAIL_TO_MAKE_CHECKPOINT, checkpoint, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); + } + } + + private void writeCheckpoint(ILogWriter writer, Checkpoint checkpoint) throws IOException { // make sure cached ByteBuffer has enough capacity int estimateSize = checkpoint.serializedSize(); if (cachedByteBuffer == null || estimateSize > cachedByteBuffer.capacity()) { cachedByteBuffer = ByteBuffer.allocate(estimateSize); } - checkpoint.serialize(cachedByteBuffer); - try { - currentLogWriter.write(cachedByteBuffer); - } catch (IOException e) { - logger.error(StorageEngineMessages.FAIL_TO_MAKE_CHECKPOINT, checkpoint, e); - DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); + checkpoint.serialize(cachedByteBuffer); + writer.write(cachedByteBuffer); } finally { cachedByteBuffer.clear(); } @@ -223,19 +213,11 @@ public void fsyncCheckpointFile() { e); DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e); + return; } try { - if (tryRollingLogWriter()) { - // first log max memTable id and global memTables' info, then delete old checkpoint file - logHeader(); - currentLogWriter.force(); - File oldFile = - SystemFileFactory.INSTANCE.getFile( - logDirectory, - CheckpointFileUtils.getLogFileName(currentCheckPointFileVersion - 1)); - Files.delete(oldFile.toPath()); - } + rollLogWriterIfNeeded(); } catch (IOException e) { logger.error( StorageEngineMessages @@ -250,17 +232,62 @@ public void fsyncCheckpointFile() { } } - private boolean tryRollingLogWriter() throws IOException { - if (currentLogWriter.size() < config.getCheckpointFileSizeThresholdInByte()) { - return false; + private void rollLogWriterIfNeeded() throws IOException { + long size = currentLogWriter.size(); + // A global snapshot can exceed the threshold by itself. Wait for an incremental checkpoint. + if (size <= initialCheckpointSize || size < config.getCheckpointFileSizeThresholdInByte()) { + return; } - currentLogWriter.close(); - currentCheckPointFileVersion++; + long nextVersion = currentCheckPointFileVersion + 1; File nextLogFile = SystemFileFactory.INSTANCE.getFile( - logDirectory, CheckpointFileUtils.getLogFileName(currentCheckPointFileVersion)); - currentLogWriter = new CheckpointWriter(nextLogFile); - return true; + logDirectory, CheckpointFileUtils.getLogFileName(nextVersion)); + ILogWriter oldWriter = currentLogWriter; + currentLogWriter = initializeLogWriter(nextLogFile, false); + currentCheckPointFileVersion = nextVersion; + initialCheckpointSize = currentLogWriter.size(); + oldWriter.close(); + Files.deleteIfExists(oldWriter.getLogFile().toPath()); + } + + private CheckpointWriter initializeLogWriter(File file, boolean append) throws IOException { + // The .tmp suffix keeps a candidate out of checkpoint recovery, even after a crash. + Path candidate = + append + ? null + : Files.createTempFile( + file.toPath().toAbsolutePath().getParent(), file.getName(), ".tmp"); + CheckpointWriter writer = null; + try { + writer = openCheckpointWriter(candidate == null ? file : candidate.toFile()); + writeInitialCheckpoint(writer); + if (candidate != null) { + writer.publish(file); + } + return writer; + } catch (IOException | RuntimeException | Error e) { + if (writer != null) { + try { + writer.close(); + } catch (IOException | RuntimeException | Error closeException) { + if (e != closeException) { + e.addSuppressed(closeException); + } + } + } + if (candidate != null) { + try { + Files.deleteIfExists(candidate); + } catch (IOException | RuntimeException | Error deleteException) { + e.addSuppressed(deleteException); + } + } + throw e; + } + } + + CheckpointWriter openCheckpointWriter(File file) throws IOException { + return new CheckpointWriter(file); } // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java index 1f8cd94336062..4d27b29fd3b4d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java @@ -23,10 +23,34 @@ import java.io.File; import java.io.IOException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; /** CheckpointWriter writes the binary {@link Checkpoint} into .checkpoint file. */ public class CheckpointWriter extends LogWriter { + private File currentFile; + public CheckpointWriter(File logFile) throws IOException { super(logFile, WALFormats.writeVersion(WALFileVersion.V2)); + currentFile = logFile; + } + + /** Publishes a forced checkpoint in the same directory, keeping its writer open for appends. */ + public void publish(File targetFile) throws IOException { + Path target = targetFile.toPath(); + // The manager owns the directory. Check for a stale target since ATOMIC_MOVE may replace it. + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + throw new FileAlreadyExistsException(target.toString()); + } + Files.move(currentFile.toPath(), target, StandardCopyOption.ATOMIC_MOVE); + currentFile = targetFile; + } + + @Override + public File getLogFile() { + return currentFile; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index abf96cb19c3c4..53897341213de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -31,6 +31,10 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.nio.file.StandardOpenOption; /** @@ -64,12 +68,35 @@ public abstract class LogWriter implements ILogWriter { protected LogWriter(File logFile, WALFileVersion version) throws IOException { this.logFile = logFile; - this.logChannel = - FileChannel.open( - logFile.toPath(), - StandardOpenOption.CREATE, - StandardOpenOption.WRITE, - StandardOpenOption.APPEND); + Path path = logFile.toPath(); + FileChannel channel; + boolean created; + while (true) { + try { + channel = + FileChannel.open( + path, + StandardOpenOption.CREATE_NEW, + StandardOpenOption.WRITE, + StandardOpenOption.APPEND); + created = true; + break; + } catch (FileAlreadyExistsException e) { + try { + channel = FileChannel.open(path, StandardOpenOption.WRITE, StandardOpenOption.APPEND); + created = false; + break; + } catch (NoSuchFileException missing) { + // Preserve CREATE's behavior when a symlink points to a missing file. Resolve its + // target so a failed initialization can remove that file and leave the link intact. + if (Files.isSymbolicLink(path)) { + path = path.resolveSibling(Files.readSymbolicLink(path)); + } + // The existing file may also have been removed after the exclusive open failed. + } + } + } + this.logChannel = channel; try { if (version == WALFileVersion.EXTENDED) { fileContext = WALFormats.create(logFile, logChannel); @@ -82,18 +109,28 @@ protected LogWriter(File logFile, WALFileVersion version) throws IOException { } dataStartOffset = logChannel.position(); } catch (IOException | RuntimeException | Error e) { - // A full disk can fail initialization after open() succeeds. Release the orphan channel - // before the owner retries creating the successor. + // Release the channel and format resources before removing a failed successor. try { logChannel.close(); - } catch (IOException closeException) { - e.addSuppressed(closeException); + } catch (IOException | RuntimeException | Error closeException) { + if (e != closeException) { + e.addSuppressed(closeException); + } } if (fileContext != null) { try { fileContext.close(); } catch (RuntimeException | Error closeException) { - e.addSuppressed(closeException); + if (e != closeException) { + e.addSuppressed(closeException); + } + } + } + if (created) { + try { + Files.deleteIfExists(path); + } catch (IOException | RuntimeException | Error deleteException) { + e.addSuppressed(deleteException); } } throw e; @@ -189,7 +226,7 @@ public void force(boolean metaData) throws IOException { @Override public long size() { - return logFile.length(); + return getLogFile().length(); } public long originalSize() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/CheckpointRecoverUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/CheckpointRecoverUtils.java index cad24b38098e4..a8b780a3a9aa2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/CheckpointRecoverUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/CheckpointRecoverUtils.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.storageengine.dataregion.wal.recover; import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint; +import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.CheckpointType; import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.MemTableInfo; import org.apache.iotdb.db.storageengine.dataregion.wal.io.CheckpointReader; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.CheckpointFileUtils; @@ -43,19 +44,20 @@ public static CheckpointInfo recoverMemTableInfo(File logDirectory) { // desc sort by version id CheckpointFileUtils.descSortByVersionId(checkpointFiles); // find last valid .checkpoint file and load checkpoints from it - long maxMemTableId = 0; - List checkpoints = null; for (File checkpointFile : checkpointFiles) { CheckpointReader reader = new CheckpointReader(checkpointFile); - maxMemTableId = reader.getMaxMemTableId(); - checkpoints = reader.getCheckpoints(); - if (!checkpoints.isEmpty()) { - break; + List checkpoints = reader.getCheckpoints(); + // A file left by an older failed roll may have incremental records without a snapshot. + if (!checkpoints.isEmpty() + && checkpoints.get(0).getType() == CheckpointType.GLOBAL_MEMORY_TABLE_INFO) { + return recoverMemTableInfo(reader.getMaxMemTableId(), checkpoints); } } - if (checkpoints == null || checkpoints.isEmpty()) { - return new CheckpointInfo(0, Collections.emptyMap()); - } + return new CheckpointInfo(0, Collections.emptyMap()); + } + + private static CheckpointInfo recoverMemTableInfo( + long maxMemTableId, List checkpoints) { // recover memTables information by checkpoints Map memTableId2Info = new HashMap<>(); for (Checkpoint checkpoint : checkpoints) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointRollRecoveryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointRollRecoveryTest.java new file mode 100644 index 0000000000000..dd1852118f37f --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointRollRecoveryTest.java @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint; + +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; +import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALInfoEntry; +import org.apache.iotdb.db.storageengine.dataregion.wal.io.CheckpointWriter; +import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALMetaData; +import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter; +import org.apache.iotdb.db.storageengine.dataregion.wal.recover.CheckpointRecoverUtils; +import org.apache.iotdb.db.storageengine.dataregion.wal.recover.WALNodeRecoverTask; +import org.apache.iotdb.db.storageengine.dataregion.wal.recover.WALRecoverManager; +import org.apache.iotdb.db.storageengine.dataregion.wal.recover.file.UnsealedTsFileRecoverPerformer; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.CheckpointFileUtils; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.WALRecoverListener; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.ArgumentCaptor; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.util.Collections; +import java.util.Set; +import java.util.concurrent.CountDownLatch; + +import static org.apache.iotdb.db.storageengine.dataregion.wal.WALTestUtils.getInsertRowNode; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class CheckpointRollRecoveryTest { + private static final String DEVICE = "root.recovery.d1"; + + @Rule public final TemporaryFolder folder = new TemporaryFolder(); + + private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + private final WALRecoverManager recovery = WALRecoverManager.getInstance(); + private long previousThreshold; + private String previousConsensus; + private NodeStatus previousStatus; + private String previousReason; + private File directory; + + @Before + public void setUp() throws IOException { + previousThreshold = config.getCheckpointFileSizeThresholdInByte(); + previousConsensus = config.getDataRegionConsensusProtocolClass(); + previousStatus = commonConfig.getNodeStatus(); + previousReason = commonConfig.getStatusReason(); + config.setCheckpointFileSizeThresholdInByte(Long.MAX_VALUE); + config.setDataRegionConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS); + commonConfig.setNodeStatus(NodeStatus.Running); + recovery.clear(); + directory = folder.newFolder("wal"); + } + + @After + public void tearDown() { + recovery.clear(); + config.setCheckpointFileSizeThresholdInByte(previousThreshold); + config.setDataRegionConsensusProtocolClass(previousConsensus); + commonConfig.setNodeStatus(NodeStatus.Running); + commonConfig.setNodeStatusWithReason(previousStatus, previousReason); + } + + @Test + public void testCandidateOpenFailureKeepsCheckpointForReplay() throws Exception { + assertFailedRollKeepsCheckpoint(FailedOperation.OPEN); + } + + @Test + public void testMaxMemTableIdWriteFailureKeepsCheckpointForReplay() throws Exception { + assertFailedRollKeepsCheckpoint(FailedOperation.WRITE_MAX_ID); + } + + @Test + public void testGlobalSnapshotWriteFailureKeepsCheckpointForReplay() throws Exception { + assertFailedRollKeepsCheckpoint(FailedOperation.WRITE_SNAPSHOT); + } + + @Test + public void testCandidateForceFailureKeepsCheckpointForReplay() throws Exception { + assertFailedRollKeepsCheckpoint(FailedOperation.FORCE); + } + + @Test + public void testCandidatePublishFailureKeepsCheckpointForReplay() throws Exception { + assertFailedRollKeepsCheckpoint(FailedOperation.PUBLISH); + } + + private void assertFailedRollKeepsCheckpoint(FailedOperation operation) throws Exception { + File oldFile = checkpointFile(0); + MemTableInfo table = memTable(1); + try (CheckpointManager manager = + spy(new CheckpointManager("checkpoint-roll", directory.getPath()))) { + addCheckpoint(manager, table); + manager.fsyncCheckpointFile(); + writeWalEntry(table, 10); + byte[] original = Files.readAllBytes(oldFile.toPath()); + + failCandidate(manager, operation); + config.setCheckpointFileSizeThresholdInByte(original.length); + manager.fsyncCheckpointFile(); + assertArrayEquals(original, Files.readAllBytes(oldFile.toPath())); + assertArrayEquals( + new File[] {oldFile}, directory.listFiles((dir, name) -> !name.endsWith(".wal"))); + assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus()); + + // A later checkpoint write must still reach the old writer. + config.setCheckpointFileSizeThresholdInByte(Long.MAX_VALUE); + addCheckpoint(manager, memTable(2)); + manager.fsyncCheckpointFile(); + assertEquals(2, CheckpointRecoverUtils.recoverMemTableInfo(directory).getMaxMemTableId()); + } + assertReplay(table, 10); + } + + private void failCandidate(CheckpointManager manager, FailedOperation operation) + throws IOException { + IOException failure = new IOException("injected checkpoint failure: " + operation); + doAnswer( + invocation -> { + if (operation == FailedOperation.OPEN) { + throw failure; + } + CheckpointWriter candidate = spy(new CheckpointWriter(invocation.getArgument(0))); + switch (operation) { + case WRITE_MAX_ID: + doThrow(failure).when(candidate).write(any(ByteBuffer.class)); + break; + case WRITE_SNAPSHOT: + doCallRealMethod().doThrow(failure).when(candidate).write(any(ByteBuffer.class)); + break; + case FORCE: + doThrow(failure).when(candidate).force(); + break; + case PUBLISH: + doThrow(failure).when(candidate).publish(any(File.class)); + break; + default: + throw new AssertionError(operation); + } + return candidate; + }) + .when(manager) + .openCheckpointWriter(any(File.class)); + } + + @Test + public void testRollWithOversizedSnapshotThenAppendAndRecover() throws Exception { + MemTableInfo first = memTable(1); + MemTableInfo second = memTable(2); + config.setCheckpointFileSizeThresholdInByte(1); + try (CheckpointManager manager = + new CheckpointManager("checkpoint-roll", directory.getPath())) { + assertTrue(checkpointFile(0).length() > 1); + assertFalse(checkpointFile(1).exists()); + addCheckpoint(manager, first); + manager.fsyncCheckpointFile(); + manager.fsyncCheckpointFile(); + assertFalse(checkpointFile(0).exists()); + assertTrue(checkpointFile(1).length() > 1); + assertFalse(checkpointFile(2).exists()); + + config.setCheckpointFileSizeThresholdInByte(Long.MAX_VALUE); + addCheckpoint(manager, second); + manager.fsyncCheckpointFile(); + CheckpointRecoverUtils.CheckpointInfo recovered = + CheckpointRecoverUtils.recoverMemTableInfo(directory); + assertEquals(2, recovered.getMaxMemTableId()); + assertEquals(Set.of(1L, 2L), recovered.getMemTableId2Info().keySet()); + writeWalEntry(second, 20); + } + assertReplay(second, 20); + } + + @Test + public void testRecoverySkipsUnpublishedCandidateAndIncompleteLegacyCheckpoint() + throws Exception { + MemTableInfo table = memTable(1); + try (CheckpointManager manager = + new CheckpointManager("checkpoint-roll", directory.getPath())) { + addCheckpoint(manager, table); + manager.fsyncCheckpointFile(); + } + writeWalEntry(table, 10); + // An older writer could append incremental records after failing to write its snapshot. + writeCheckpointFile( + checkpointFile(1), + new Checkpoint(CheckpointType.CREATE_MEMORY_TABLE, Collections.singletonList(memTable(2)))); + // Simulate interruption before publishing a candidate with an empty global snapshot. + writeCheckpointFile( + new File(directory, checkpointFile(2).getName() + ".tmp"), + new Checkpoint(CheckpointType.GLOBAL_MEMORY_TABLE_INFO, Collections.emptyList())); + assertEquals(1, CheckpointRecoverUtils.recoverMemTableInfo(directory).getMaxMemTableId()); + assertReplay(table, 10); + } + + private File checkpointFile(long version) { + return new File(directory, CheckpointFileUtils.getLogFileName(version)); + } + + private MemTableInfo memTable(long id) { + IMemTable table = mock(IMemTable.class); + when(table.getMemTableId()).thenReturn(id); + when(table.getDataRegionId()).thenReturn("0"); + File tsFile = new File(folder.getRoot(), "sequence/root.recovery/0/0/" + id + "-1-0-0.tsfile"); + return new MemTableInfo(table, tsFile.getAbsolutePath(), 0); + } + + private void addCheckpoint(CheckpointManager manager, MemTableInfo table) { + manager.makeCreateMemTableCPInMemory(table); + manager.makeCreateMemTableCPOnDisk(table.getMemTableId()); + } + + private void writeCheckpointFile(File file, Checkpoint checkpoint) throws IOException { + try (CheckpointWriter writer = new CheckpointWriter(file)) { + ByteBuffer data = ByteBuffer.allocate(Long.BYTES + checkpoint.serializedSize()); + data.putLong(100); + checkpoint.serialize(data); + writer.write(data); + } + } + + private void writeWalEntry(MemTableInfo table, long time) throws Exception { + WALEntry entry = new WALInfoEntry(table.getMemTableId(), getInsertRowNode(DEVICE, time)); + ByteBuffer data = ByteBuffer.allocate(entry.serializedSize()); + entry.serialize(new WALByteBufferForTest(data)); + WALMetaData metadata = new WALMetaData(); + metadata.add(data.position(), -1, table.getMemTableId()); + try (WALWriter writer = new WALWriter(new File(directory, "_0-0-0.wal"))) { + writer.write(data, metadata); + } + } + + private void assertReplay(MemTableInfo table, long time) throws Exception { + File tsFile = new File(table.getTsFilePath()); + UnsealedTsFileRecoverPerformer performer = mock(UnsealedTsFileRecoverPerformer.class); + WALRecoverListener listener = new WALRecoverListener(tsFile.getAbsolutePath()); + when(performer.getTsFileResource()).thenReturn(new TsFileResource(tsFile)); + when(performer.getRecoverListener()).thenReturn(listener); + recovery.addRecoverPerformer(performer); + + CountDownLatch latch = new CountDownLatch(1); + new WALNodeRecoverTask(directory, latch).run(); + assertEquals(0, latch.getCount()); + ArgumentCaptor replay = ArgumentCaptor.forClass(WALEntry.class); + verify(performer).redoLog(replay.capture()); + assertEquals(WALRecoverListener.Status.SUCCESS, listener.waitForResult()); + assertEquals(table.getMemTableId(), replay.getValue().getMemTableId()); + assertEquals(time, ((InsertRowNode) replay.getValue().getValue()).getTime()); + } + + private enum FailedOperation { + OPEN, + WRITE_MAX_ID, + WRITE_SNAPSHOT, + FORCE, + PUBLISH + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFormatExtensionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFormatExtensionTest.java index 3ead148f06a6d..c84a77bd3ee3b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFormatExtensionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFormatExtensionTest.java @@ -58,6 +58,7 @@ import java.net.URL; import java.net.URLClassLoader; import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -130,7 +131,7 @@ public Enumeration getResources(String name) throws IOException { assertTrue( result.getClass().getMethod("getFailures").invoke(result).toString(), (boolean) result.getClass().getMethod("wasSuccessful").invoke(result)); - assertEquals(16, result.getClass().getMethod("getRunCount").invoke(result)); + assertEquals(20, result.getClass().getMethod("getRunCount").invoke(result)); } finally { Thread.currentThread().setContextClassLoader(previous); } @@ -146,7 +147,9 @@ public void setUp() throws Exception { provider = mock(WALFormatProvider.class); context = mock(WALFileContext.class); when(provider.isEnabled()).thenReturn(true); - when(provider.create(any(), any())).thenReturn(context); + doAnswer(invocation -> writeHeader(invocation.getArgument(1))) + .when(provider) + .create(any(), any()); when(provider.open(any(), any())).thenReturn(context); StubProvider.delegate = provider; } @@ -158,13 +161,13 @@ public void tearDown() { @Test public void testInitializationFailureClosesCallerChannels() throws Exception { - File file = folder.newFile(); + File file = extensionFile(); + byte[] original = Files.readAllBytes(file.toPath()); WALFormatException failure = WALFormatException.unsupported(); - when(provider.create(any(), any())).thenThrow(failure); + doThrow(failure).when(provider).create(any(), any()); assertSame(failure, assertThrows(IOException.class, () -> new WALWriter(file))); - ArgumentCaptor created = ArgumentCaptor.forClass(FileChannel.class); - verify(provider).create(eq(file), created.capture()); - assertFalse(created.getValue().isOpen()); + assertCreateChannelClosed(file); + assertArrayEquals(original, Files.readAllBytes(file.toPath())); when(provider.open(any(), any())).thenThrow(failure); assertSame(failure, assertThrows(IOException.class, () -> new WALInputStream(file))); ArgumentCaptor opened = ArgumentCaptor.forClass(FileChannel.class); @@ -172,6 +175,86 @@ public void testInitializationFailureClosesCallerChannels() throws Exception { assertFalse(opened.getValue().isOpen()); } + @Test + public void testFailedInitializationRemovesNewEmptyFile() throws Exception { + File file = new File(folder.getRoot(), "empty.wal"); + IOException failure = new IOException("injected failure before writing the WAL header"); + doThrow(failure).when(provider).create(any(), any()); + + assertSame(failure, assertThrows(IOException.class, () -> new WALWriter(file))); + assertCreateChannelClosed(file); + assertFalse(file.exists()); + } + + @Test + public void testFailedInitializationKeepsPreexistingEmptyFile() throws Exception { + File file = folder.newFile(); + IOException failure = new IOException("injected failure before writing the WAL header"); + doThrow(failure).when(provider).create(any(), any()); + + assertSame(failure, assertThrows(IOException.class, () -> new WALWriter(file))); + assertCreateChannelClosed(file); + assertArrayEquals(new byte[0], Files.readAllBytes(file.toPath())); + } + + @Test + public void testFailedInitializationRemovesNewPartialFile() throws Exception { + File file = new File(folder.getRoot(), "partial.wal"); + IOException failure = new IOException("injected failure during the WAL header write"); + doAnswer( + invocation -> { + FileChannel channel = invocation.getArgument(1); + ByteBuffer prefix = ByteBuffer.wrap(StubProvider.MAGIC, 0, 2); + while (prefix.hasRemaining()) { + channel.write(prefix); + } + throw failure; + }) + .doAnswer(invocation -> writeHeader(invocation.getArgument(1))) + .when(provider) + .create(any(), any()); + + assertSame(failure, assertThrows(IOException.class, () -> new WALWriter(file))); + assertCreateChannelClosed(file); + assertFalse(file.exists()); + try (WALWriter ignored = new WALWriter(file)) { + assertEquals(WALFileVersion.EXTENDED, WALFileVersion.getVersion(file)); + } + } + + @Test + public void testInitializationFailureSuppressesContextCloseFailure() throws Exception { + File file = new File(folder.getRoot(), "close.wal"); + IllegalStateException closeFailure = new IllegalStateException("injected close failure"); + doAnswer( + invocation -> { + FileChannel channel = invocation.getArgument(1); + channel.close(); + return context; + }) + .when(provider) + .create(any(), any()); + doThrow(closeFailure).when(context).close(); + + IOException failure = assertThrows(ClosedChannelException.class, () -> new WALWriter(file)); + assertArrayEquals(new Throwable[] {closeFailure}, failure.getSuppressed()); + assertFalse(file.exists()); + } + + private void assertCreateChannelClosed(File file) throws IOException { + ArgumentCaptor channel = ArgumentCaptor.forClass(FileChannel.class); + verify(provider).create(eq(file), channel.capture()); + assertFalse(channel.getValue().isOpen()); + } + + private WALFileContext writeHeader(FileChannel channel) throws IOException { + ByteBuffer magic = ByteBuffer.wrap(StubProvider.MAGIC); + while (magic.hasRemaining()) { + channel.write(magic); + } + return context; + } + @Test public void testWriterDelegatesSegmentsMetadataAndClose() throws Exception { when(context.writeData(any(), anyLong(), any(), anyInt(), any())) @@ -611,10 +694,6 @@ private WALFormatProvider implementation() throws IOException { @Override public WALFileContext create(File file, FileChannel channel) throws IOException { - ByteBuffer magic = ByteBuffer.wrap(MAGIC); - while (magic.hasRemaining()) { - channel.write(magic); - } return implementation().create(file, channel); } From 6636a16131742156bda4e9824fa466aef8381353 Mon Sep 17 00:00:00 2001 From: HTHou Date: Mon, 16 Nov 2026 12:08:08 +0800 Subject: [PATCH 2/4] [WAL] Preserve checkpoints across legacy rolls and publication failures --- .../dataregion/wal/buffer/WALBuffer.java | 4 +- .../wal/checkpoint/CheckpointManager.java | 52 ++++-- .../dataregion/wal/io/CheckpointWriter.java | 84 +++++++++- .../wal/recover/CheckpointRecoverUtils.java | 44 +++--- .../wal/recover/WALNodeRecoverTask.java | 23 ++- .../wal/utils/CheckpointFileUtils.java | 33 +++- .../CheckpointRollRecoveryTest.java | 148 +++++++++++++++--- .../wal/recover/WALNodeRecoverTaskTest.java | 10 ++ 8 files changed, 328 insertions(+), 70 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index 22ff479d409c6..9dda10a18bec9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -661,8 +661,6 @@ public void run() { return; } - makeMemTableCheckpoints(); - long walFileVersionId = currentWALFileVersion; currentWALFileWriter.updateFileStatus(fileStatus); @@ -679,6 +677,8 @@ public void run() { // flush buffer to os double compressionRatio = 1.0; try { + makeMemTableCheckpoints(); + checkpointManager.checkCheckpointPublication(); compressionRatio = currentWALFileWriter.write(syncingBuffer, info.metaData); } catch (Throwable e) { logger.error( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java index 5f734cb2ab2a7..bbff37863f88a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java @@ -39,7 +39,6 @@ import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.LinkOption; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -79,6 +78,8 @@ public class CheckpointManager implements AutoCloseable { private ILogWriter currentLogWriter; // size of the file before any incremental checkpoints are appended private long initialCheckpointSize; + // Both names may survive after a failed publication and rollback. Stop appending until recovery. + private volatile IOException checkpointPublicationFailure; // endregion @@ -139,7 +140,7 @@ public void makeCreateMemTableCPOnDisk(long memTableId) { long start = System.nanoTime(); try { MemTableInfo memTableInfo = memTableId2Info.get(memTableId); - if (memTableInfo == null) { + if (memTableInfo == null || checkpointPublicationFailure != null) { return; } Checkpoint checkpoint = @@ -159,7 +160,7 @@ public void makeFlushMemTableCP(long memTableId) { long start = System.nanoTime(); try { MemTableInfo memTableInfo = memTableId2Info.get(memTableId); - if (memTableInfo == null) { + if (memTableInfo == null || checkpointPublicationFailure != null) { return; } memTableInfo.setFlushed(); @@ -198,11 +199,21 @@ private void writeCheckpoint(ILogWriter writer, Checkpoint checkpoint) throws IO } } + public void checkCheckpointPublication() throws IOException { + IOException failure = checkpointPublicationFailure; + if (failure != null) { + throw failure; + } + } + // region Task to fsync checkpoint file /** Fsync checkpoints to the disk. */ public void fsyncCheckpointFile() { infoLock.lock(); try { + if (checkpointPublicationFailure != null) { + return; + } try { currentLogWriter.force(); } catch (IOException e) { @@ -251,18 +262,13 @@ private void rollLogWriterIfNeeded() throws IOException { } private CheckpointWriter initializeLogWriter(File file, boolean append) throws IOException { - // The .tmp suffix keeps a candidate out of checkpoint recovery, even after a crash. - Path candidate = - append - ? null - : Files.createTempFile( - file.toPath().toAbsolutePath().getParent(), file.getName(), ".tmp"); + File candidate = append ? null : CheckpointFileUtils.createCheckpointCandidate(file); CheckpointWriter writer = null; try { - writer = openCheckpointWriter(candidate == null ? file : candidate.toFile()); + writer = openCheckpointWriter(candidate == null ? file : candidate); writeInitialCheckpoint(writer); if (candidate != null) { - writer.publish(file); + publishCheckpoint(writer, file, candidate); } return writer; } catch (IOException | RuntimeException | Error e) { @@ -277,7 +283,7 @@ private CheckpointWriter initializeLogWriter(File file, boolean append) throws I } if (candidate != null) { try { - Files.deleteIfExists(candidate); + Files.deleteIfExists(candidate.toPath()); } catch (IOException | RuntimeException | Error deleteException) { e.addSuppressed(deleteException); } @@ -286,6 +292,28 @@ private CheckpointWriter initializeLogWriter(File file, boolean append) throws I } } + private void publishCheckpoint(CheckpointWriter writer, File file, File candidate) + throws IOException { + try { + writer.publish(file); + } catch (IOException | RuntimeException | Error failure) { + if (file.equals(writer.getLogFile())) { + // The final name became visible before synchronization failed. Make its removal durable + // before accepting any more writes through the old checkpoint. + try { + writer.publish(candidate); + } catch (IOException | RuntimeException | Error rollbackFailure) { + if (rollbackFailure != failure) { + failure.addSuppressed(rollbackFailure); + } + checkpointPublicationFailure = + failure instanceof IOException ioFailure ? ioFailure : new IOException(failure); + } + } + throw failure; + } + } + CheckpointWriter openCheckpointWriter(File file) throws IOException { return new CheckpointWriter(file); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java index 4d27b29fd3b4d..4fb5d2cad8355 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java @@ -21,8 +21,20 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint; +import com.sun.jna.Memory; +import com.sun.jna.Native; +import com.sun.jna.Platform; +import com.sun.jna.platform.win32.Kernel32; +import com.sun.jna.platform.win32.Kernel32Util; +import com.sun.jna.platform.win32.Win32Exception; +import com.sun.jna.platform.win32.WinBase; +import com.sun.jna.platform.win32.WinDef.DWORD; +import com.sun.jna.platform.win32.WinNT; + +import java.io.Closeable; import java.io.File; import java.io.IOException; +import java.nio.channels.FileChannel; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.LinkOption; @@ -31,6 +43,8 @@ /** CheckpointWriter writes the binary {@link Checkpoint} into .checkpoint file. */ public class CheckpointWriter extends LogWriter { + private static final int FILE_RENAME_INFO = 3; + private File currentFile; public CheckpointWriter(File logFile) throws IOException { @@ -38,15 +52,79 @@ public CheckpointWriter(File logFile) throws IOException { currentFile = logFile; } - /** Publishes a forced checkpoint in the same directory, keeping its writer open for appends. */ + /** Durably renames a forced checkpoint in the same directory, keeping its writer open. */ public void publish(File targetFile) throws IOException { + Path source = currentFile.toPath(); Path target = targetFile.toPath(); // The manager owns the directory. Check for a stale target since ATOMIC_MOVE may replace it. if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { throw new FileAlreadyExistsException(target.toString()); } - Files.move(currentFile.toPath(), target, StandardCopyOption.ATOMIC_MOVE); - currentFile = targetFile; + if (Platform.isWindows()) { + publishOnWindows(source, targetFile); + } else { + Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); + currentFile = targetFile; + } + forceParentDirectory(target.toAbsolutePath().getParent()); + } + + protected void forceParentDirectory(Path directory) throws IOException { + if (!Platform.isWindows()) { + try (FileChannel channel = FileChannel.open(directory)) { + channel.force(true); + } + } + } + + private void publishOnWindows(Path source, File target) throws IOException { + // Renaming through a write-through handle flushes the resulting NTFS metadata changes. + Kernel32 kernel = Kernel32.INSTANCE; + WinNT.HANDLE handle = + kernel.CreateFile( + windowsPath(source), + WinNT.GENERIC_WRITE | WinNT.DELETE, + WinNT.FILE_SHARE_READ | WinNT.FILE_SHARE_WRITE | WinNT.FILE_SHARE_DELETE, + null, + WinNT.OPEN_EXISTING, + WinNT.FILE_FLAG_WRITE_THROUGH, + null); + if (WinBase.INVALID_HANDLE_VALUE.equals(handle)) { + throw new IOException(new Win32Exception(Native.getLastError())); + } + + try (Closeable closeHandle = () -> Kernel32Util.closeHandle(handle)) { + String targetName = windowsPath(target.toPath()); + int nameBytes = targetName.length() * Native.WCHAR_SIZE; + // FILE_RENAME_INFO has a zeroed flag and parent handle before the byte length and name. + int lengthOffset = 2 * Native.POINTER_SIZE; + int nameOffset = lengthOffset + Integer.BYTES; + int bufferSize = nameOffset + nameBytes + Native.WCHAR_SIZE; + try (Memory renameInfo = new Memory(bufferSize)) { + renameInfo.clear(); + renameInfo.setInt(lengthOffset, nameBytes); + renameInfo.setWideString(nameOffset, targetName); + if (!kernel.SetFileInformationByHandle( + handle, FILE_RENAME_INFO, renameInfo, new DWORD(bufferSize))) { + IOException failure = new IOException(new Win32Exception(Native.getLastError())); + if (!Files.exists(source, LinkOption.NOFOLLOW_LINKS)) { + // The rename may have taken effect before the metadata flush failed. + currentFile = target; + } + throw failure; + } + currentFile = target; + } + } catch (Win32Exception e) { + throw new IOException(e); + } + } + + private static String windowsPath(Path path) { + String absolutePath = path.toAbsolutePath().normalize().toString(); + return absolutePath.startsWith("\\\\") + ? "\\\\?\\UNC\\" + absolutePath.substring(2) + : "\\\\?\\" + absolutePath; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/CheckpointRecoverUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/CheckpointRecoverUtils.java index a8b780a3a9aa2..e593a5e5bebde 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/CheckpointRecoverUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/CheckpointRecoverUtils.java @@ -26,7 +26,9 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.utils.CheckpointFileUtils; import java.io.File; +import java.util.ArrayDeque; import java.util.Collections; +import java.util.Deque; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -43,39 +45,35 @@ public static CheckpointInfo recoverMemTableInfo(File logDirectory) { } // desc sort by version id CheckpointFileUtils.descSortByVersionId(checkpointFiles); - // find last valid .checkpoint file and load checkpoints from it + // Read back to the latest complete snapshot. Legacy rolls can have readable incremental + // checkpoints even if writing their snapshot failed. + Deque readers = new ArrayDeque<>(); for (File checkpointFile : checkpointFiles) { CheckpointReader reader = new CheckpointReader(checkpointFile); List checkpoints = reader.getCheckpoints(); - // A file left by an older failed roll may have incremental records without a snapshot. - if (!checkpoints.isEmpty() - && checkpoints.get(0).getType() == CheckpointType.GLOBAL_MEMORY_TABLE_INFO) { - return recoverMemTableInfo(reader.getMaxMemTableId(), checkpoints); + if (checkpoints.isEmpty()) { + continue; + } + readers.addFirst(reader); + if (checkpoints.get(0).getType() == CheckpointType.GLOBAL_MEMORY_TABLE_INFO) { + break; } } - return new CheckpointInfo(0, Collections.emptyMap()); - } - private static CheckpointInfo recoverMemTableInfo( - long maxMemTableId, List checkpoints) { - // recover memTables information by checkpoints + // Apply the snapshot and the readable incremental checkpoints in version order. + long maxMemTableId = 0; Map memTableId2Info = new HashMap<>(); - for (Checkpoint checkpoint : checkpoints) { - switch (checkpoint.getType()) { - case GLOBAL_MEMORY_TABLE_INFO: - case CREATE_MEMORY_TABLE: - for (MemTableInfo memTableInfo : checkpoint.getMemTableInfos()) { + for (CheckpointReader reader : readers) { + maxMemTableId = Math.max(maxMemTableId, reader.getMaxMemTableId()); + for (Checkpoint checkpoint : reader.getCheckpoints()) { + for (MemTableInfo memTableInfo : checkpoint.getMemTableInfos()) { + if (checkpoint.getType() == CheckpointType.FLUSH_MEMORY_TABLE) { + memTableId2Info.remove(memTableInfo.getMemTableId()); + } else { maxMemTableId = Math.max(maxMemTableId, memTableInfo.getMemTableId()); memTableId2Info.put(memTableInfo.getMemTableId(), memTableInfo); } - break; - case FLUSH_MEMORY_TABLE: - for (MemTableInfo memTableInfo : checkpoint.getMemTableInfos()) { - memTableId2Info.remove(memTableInfo.getMemTableId()); - } - break; - default: - break; + } } } return new CheckpointInfo(maxMemTableId, memTableId2Info); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java index 950af933da4ab..a9d3e46a2d723 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java @@ -151,16 +151,10 @@ private void recoverNode() { logDirectory); return; } + deleteRecoveredCheckpoints(CheckpointFileUtils.listCheckpointCandidates(logDirectory)); if (config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS)) { // delete checkpoint info to avoid repeated recover - File[] checkpointFiles = CheckpointFileUtils.listAllCheckpointFiles(logDirectory); - for (File checkpointFile : checkpointFiles) { - try { - Files.delete(checkpointFile.toPath()); - } catch (IOException e) { - logger.error(StorageEngineMessages.ERROR_DELETE_CHECKPOINT_FILE, checkpointFile, e); - } - } + deleteRecoveredCheckpoints(CheckpointFileUtils.listAllCheckpointFiles(logDirectory)); // register wal node WALManager.getInstance() .registerWALNode( @@ -197,6 +191,19 @@ private void recoverNode() { } } + private static void deleteRecoveredCheckpoints(File[] checkpointFiles) { + if (checkpointFiles == null) { + return; + } + for (File checkpointFile : checkpointFiles) { + try { + Files.deleteIfExists(checkpointFile.toPath()); + } catch (IOException e) { + logger.error(StorageEngineMessages.ERROR_DELETE_CHECKPOINT_FILE, checkpointFile, e); + } + } + } + private long[] readLastFileInfoAndRepairIt() { File[] walFiles = WALFileUtils.listAllWALFiles(logDirectory); if (walFiles == null || walFiles.length == 0) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/CheckpointFileUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/CheckpointFileUtils.java index faed50c6de8be..f8a1dca0aefa0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/CheckpointFileUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/CheckpointFileUtils.java @@ -22,6 +22,10 @@ import org.apache.iotdb.db.i18n.StorageEngineMessages; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; import java.util.Arrays; import java.util.Comparator; import java.util.regex.Matcher; @@ -32,11 +36,20 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.WAL_VERSION_ID; public class CheckpointFileUtils { + private static final String CANDIDATE_SUFFIX = ".tmp"; + private static final String CHECKPOINT_NAME = + Pattern.quote(WAL_FILE_PREFIX) + + "(?<" + + WAL_VERSION_ID + + ">\\d+)" + + Pattern.quote(WAL_CHECKPOINT_FILE_SUFFIX); + // versionId is a self-incremented id number, helping to maintain the order of checkpoint files - public static final Pattern CHECKPOINT_FILE_NAME_PATTERN = - Pattern.compile( - String.format( - "%s(?<%s>\\d+)\\%s$", WAL_FILE_PREFIX, WAL_VERSION_ID, WAL_CHECKPOINT_FILE_SUFFIX)); + public static final Pattern CHECKPOINT_FILE_NAME_PATTERN = Pattern.compile(CHECKPOINT_NAME + "$"); + + // Files.createTempFile inserts an unsigned decimal number between the prefix and suffix. + private static final Pattern CANDIDATE_NAME_PATTERN = + Pattern.compile(CHECKPOINT_NAME + "\\d+" + Pattern.quote(CANDIDATE_SUFFIX)); /** Return true when this file is .checkpoint file. */ public static boolean checkpointFilenameFilter(File dir, String name) { @@ -48,6 +61,18 @@ public static File[] listAllCheckpointFiles(File dir) { return dir.listFiles(CheckpointFileUtils::checkpointFilenameFilter); } + public static File[] listCheckpointCandidates(File dir) { + return dir.listFiles( + file -> + CANDIDATE_NAME_PATTERN.matcher(file.getName()).matches() + && Files.isRegularFile(file.toPath(), LinkOption.NOFOLLOW_LINKS)); + } + + public static File createCheckpointCandidate(File target) throws IOException { + Path parent = target.toPath().toAbsolutePath().getParent(); + return Files.createTempFile(parent, target.getName(), CANDIDATE_SUFFIX).toFile(); + } + /** Parse version id from filename. */ public static int parseVersionId(String filename) { Matcher matcher = CHECKPOINT_FILE_NAME_PATTERN.matcher(filename); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointRollRecoveryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointRollRecoveryTest.java index dd1852118f37f..2e9c49efac531 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointRollRecoveryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointRollRecoveryTest.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALBuffer; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALInfoEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.io.CheckpointWriter; @@ -39,6 +40,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.recover.file.UnsealedTsFileRecoverPerformer; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.CheckpointFileUtils; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.WALFlushListener; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.WALRecoverListener; import org.junit.After; @@ -52,7 +54,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; +import java.nio.file.Path; import java.util.Collections; +import java.util.List; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -60,6 +64,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -131,6 +136,11 @@ public void testCandidatePublishFailureKeepsCheckpointForReplay() throws Excepti assertFailedRollKeepsCheckpoint(FailedOperation.PUBLISH); } + @Test + public void testDirectorySyncFailureRollsBackAndKeepsCheckpointForReplay() throws Exception { + assertFailedRollKeepsCheckpoint(FailedOperation.SYNC_DIRECTORY); + } + private void assertFailedRollKeepsCheckpoint(FailedOperation operation) throws Exception { File oldFile = checkpointFile(0); MemTableInfo table = memTable(1); @@ -166,7 +176,11 @@ private void failCandidate(CheckpointManager manager, FailedOperation operation) if (operation == FailedOperation.OPEN) { throw failure; } - CheckpointWriter candidate = spy(new CheckpointWriter(invocation.getArgument(0))); + File file = invocation.getArgument(0); + if (operation == FailedOperation.SYNC_DIRECTORY) { + return writerWithDirectorySyncFailure(file, failure, null); + } + CheckpointWriter candidate = spy(new CheckpointWriter(file)); switch (operation) { case WRITE_MAX_ID: doThrow(failure).when(candidate).write(any(ByteBuffer.class)); @@ -189,6 +203,74 @@ private void failCandidate(CheckpointManager manager, FailedOperation operation) .openCheckpointWriter(any(File.class)); } + @Test(timeout = 30000) + public void testFailedPublicationRollbackStopsWalWrites() throws Exception { + File oldFile = checkpointFile(0); + File nextFile = checkpointFile(1); + MemTableInfo table = memTable(1); + IOException syncFailure = new IOException("injected directory sync failure"); + IOException rollbackFailure = new IOException("injected rollback rename failure"); + try (CheckpointManager manager = + spy(new CheckpointManager("checkpoint-roll", directory.getPath())); + WALBuffer buffer = new WALBuffer("checkpoint-roll", directory.getPath(), manager, 0, 0)) { + addCheckpoint(manager, table); + manager.fsyncCheckpointFile(); + WALInfoEntry first = new WALInfoEntry(1, getInsertRowNode(DEVICE, 10), true); + buffer.write(first); + assertEquals(WALFlushListener.Status.SUCCESS, first.getWalFlushListener().waitForResult()); + long walSize = buffer.getCurrentWALOriginalFileSize(); + byte[] oldBytes = Files.readAllBytes(oldFile.toPath()); + + doAnswer( + invocation -> + writerWithDirectorySyncFailure( + invocation.getArgument(0), syncFailure, rollbackFailure)) + .when(manager) + .openCheckpointWriter(any(File.class)); + config.setCheckpointFileSizeThresholdInByte(oldBytes.length); + manager.fsyncCheckpointFile(); + assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus()); + assertTrue(nextFile.exists()); + byte[] nextBytes = Files.readAllBytes(nextFile.toPath()); + + commonConfig.setNodeStatus(NodeStatus.Running); + addCheckpoint(manager, memTable(2)); + WALInfoEntry second = new WALInfoEntry(2, getInsertRowNode(DEVICE, 20), true); + buffer.write(second); + WALFlushListener listener = second.getWalFlushListener(); + assertEquals(WALFlushListener.Status.FAILURE, listener.waitForResult()); + assertSame(syncFailure, listener.getCause()); + assertArrayEquals(new Throwable[] {rollbackFailure}, syncFailure.getSuppressed()); + assertEquals(walSize, buffer.getCurrentWALOriginalFileSize()); + assertArrayEquals(oldBytes, Files.readAllBytes(oldFile.toPath())); + assertArrayEquals(nextBytes, Files.readAllBytes(nextFile.toPath())); + } + assertReplay(table, 10); + } + + private CheckpointWriter writerWithDirectorySyncFailure( + File candidate, IOException syncFailure, IOException rollbackFailure) throws IOException { + return new CheckpointWriter(candidate) { + @Override + public void publish(File target) throws IOException { + if (target.equals(candidate) && rollbackFailure != null) { + throw rollbackFailure; + } + super.publish(target); + } + + @Override + protected void forceParentDirectory(Path parent) throws IOException { + if (getLogFile().equals(checkpointFile(1))) { + assertTrue(checkpointFile(0).isFile()); + assertTrue(checkpointFile(1).isFile()); + throw syncFailure; + } + super.forceParentDirectory(parent); + } + }; + } + @Test public void testRollWithOversizedSnapshotThenAppendAndRecover() throws Exception { MemTableInfo first = memTable(1); @@ -218,25 +300,53 @@ public void testRollWithOversizedSnapshotThenAppendAndRecover() throws Exception } @Test - public void testRecoverySkipsUnpublishedCandidateAndIncompleteLegacyCheckpoint() - throws Exception { - MemTableInfo table = memTable(1); - try (CheckpointManager manager = - new CheckpointManager("checkpoint-roll", directory.getPath())) { - addCheckpoint(manager, table); - manager.fsyncCheckpointFile(); - } - writeWalEntry(table, 10); - // An older writer could append incremental records after failing to write its snapshot. + public void testRecoveryReplaysLegacyIncrementsAfterSnapshotAndSkipsCandidate() throws Exception { + MemTableInfo first = memTable(1); + MemTableInfo flushed = memTable(2); + MemTableInfo latest = memTable(3); + writeCheckpointFile( + checkpointFile(0), + 2, + new Checkpoint(CheckpointType.GLOBAL_MEMORY_TABLE_INFO, List.of(first, flushed))); + // A legacy roll can contain a create or flush record even if its snapshot write failed. writeCheckpointFile( checkpointFile(1), - new Checkpoint(CheckpointType.CREATE_MEMORY_TABLE, Collections.singletonList(memTable(2)))); + 2, + new Checkpoint(CheckpointType.CREATE_MEMORY_TABLE, Collections.singletonList(latest))); + writeCheckpointFile( + checkpointFile(2), + 3, + new Checkpoint(CheckpointType.FLUSH_MEMORY_TABLE, Collections.singletonList(flushed))); // Simulate interruption before publishing a candidate with an empty global snapshot. writeCheckpointFile( - new File(directory, checkpointFile(2).getName() + ".tmp"), + CheckpointFileUtils.createCheckpointCandidate(checkpointFile(3)), + 100, new Checkpoint(CheckpointType.GLOBAL_MEMORY_TABLE_INFO, Collections.emptyList())); - assertEquals(1, CheckpointRecoverUtils.recoverMemTableInfo(directory).getMaxMemTableId()); - assertReplay(table, 10); + writeWalEntry(latest, 30); + CheckpointRecoverUtils.CheckpointInfo recovered = + CheckpointRecoverUtils.recoverMemTableInfo(directory); + assertEquals(3, recovered.getMaxMemTableId()); + assertEquals(Set.of(1L, 3L), recovered.getMemTableId2Info().keySet()); + assertReplay(latest, 30); + } + + @Test + public void testRecoveryReplaysLegacyIncrementsWithoutSnapshot() throws Exception { + MemTableInfo first = memTable(1); + writeCheckpointFile( + checkpointFile(1), + 0, + new Checkpoint(CheckpointType.CREATE_MEMORY_TABLE, Collections.singletonList(first))); + writeCheckpointFile( + checkpointFile(2), + 1, + new Checkpoint(CheckpointType.CREATE_MEMORY_TABLE, Collections.singletonList(memTable(2)))); + writeWalEntry(first, 10); + CheckpointRecoverUtils.CheckpointInfo recovered = + CheckpointRecoverUtils.recoverMemTableInfo(directory); + assertEquals(2, recovered.getMaxMemTableId()); + assertEquals(Set.of(1L, 2L), recovered.getMemTableId2Info().keySet()); + assertReplay(first, 10); } private File checkpointFile(long version) { @@ -256,10 +366,11 @@ private void addCheckpoint(CheckpointManager manager, MemTableInfo table) { manager.makeCreateMemTableCPOnDisk(table.getMemTableId()); } - private void writeCheckpointFile(File file, Checkpoint checkpoint) throws IOException { + private void writeCheckpointFile(File file, long maxMemTableId, Checkpoint checkpoint) + throws IOException { try (CheckpointWriter writer = new CheckpointWriter(file)) { ByteBuffer data = ByteBuffer.allocate(Long.BYTES + checkpoint.serializedSize()); - data.putLong(100); + data.putLong(maxMemTableId); checkpoint.serialize(data); writer.write(data); } @@ -299,6 +410,7 @@ private enum FailedOperation { WRITE_MAX_ID, WRITE_SNAPSHOT, FORCE, - PUBLISH + PUBLISH, + SYNC_DIRECTORY } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTaskTest.java index f7780a7fa9059..536c70db6f10e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTaskTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTaskTest.java @@ -38,6 +38,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter; import org.apache.iotdb.db.storageengine.dataregion.wal.node.WALNode; import org.apache.iotdb.db.storageengine.dataregion.wal.recover.file.UnsealedTsFileRecoverPerformer; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.CheckpointFileUtils; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALMode; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.WALRecoverListener; @@ -197,6 +198,10 @@ private void assertFailedRecovery(Exception failure, boolean finalization, boole } byte[] walBefore = Files.readAllBytes(wal.toPath()); byte[] checkpointBefore = Files.readAllBytes(checkpointFile.toPath()); + File candidate = CheckpointFileUtils.createCheckpointCandidate(new File(node, "_1.checkpoint")); + Files.writeString(candidate.toPath(), "V99-WAL"); + File unrelatedTemp = new File(node, "_1.checkpoint.backup.tmp"); + Files.createFile(unrelatedTemp.toPath()); UnsealedTsFileRecoverPerformer failed = performers.get(0); when(failed.canWrite()).thenReturn(true); if (finalization) { @@ -222,6 +227,7 @@ private void assertFailedRecovery(Exception failure, boolean finalization, boole verify(healthy).endRecovery(); assertArrayEquals(walBefore, Files.readAllBytes(wal.toPath())); assertArrayEquals(checkpointBefore, Files.readAllBytes(checkpointFile.toPath())); + assertTrue(candidate.exists()); if (stableNode) { WALManager manager = WALManager.getInstance(); for (int i = 0; i < 3; i++) { @@ -262,6 +268,10 @@ private void assertFailedRecovery(Exception failure, boolean finalization, boole CountDownLatch retryLatch = new CountDownLatch(1); new WALNodeRecoverTask(node, retryLatch).run(); assertEquals(0, retryLatch.getCount()); + assertFalse(candidate.exists()); + if (config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS)) { + assertTrue(unrelatedTemp.exists()); + } for (UnsealedTsFileRecoverPerformer performer : retried) { assertEquals( WALRecoverListener.Status.SUCCESS, performer.getRecoverListener().waitForResult()); From 87d053f74d3abe0c97c835cb9cb4ceea2842cbf2 Mon Sep 17 00:00:00 2001 From: HTHou Date: Mon, 16 Nov 2026 12:42:56 +0800 Subject: [PATCH 3/4] [WAL] Propagate checkpoint I/O failures and respect recovery version gaps --- .../dataregion/wal/buffer/WALBuffer.java | 2 +- .../wal/checkpoint/CheckpointManager.java | 18 +-- .../wal/recover/CheckpointRecoverUtils.java | 13 +- .../CheckpointRollRecoveryTest.java | 124 ++++++++++++++++-- .../wal/recover/WALRecoverManagerTest.java | 12 +- 5 files changed, 146 insertions(+), 23 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index 9dda10a18bec9..3c3f1ecee565a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -770,7 +770,7 @@ private void failListeners(Exception e) { } } - private void makeMemTableCheckpoints() { + private void makeMemTableCheckpoints() throws IOException { if (info.checkpoints.isEmpty()) { return; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java index bbff37863f88a..4859e39513016 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java @@ -135,7 +135,7 @@ public void makeCreateMemTableCPInMemory(MemTableInfo memTableInfo) { } /** Make checkpoint for create memTable info on disk. */ - public void makeCreateMemTableCPOnDisk(long memTableId) { + public void makeCreateMemTableCPOnDisk(long memTableId) throws IOException { infoLock.lock(); long start = System.nanoTime(); try { @@ -155,7 +155,7 @@ public void makeCreateMemTableCPOnDisk(long memTableId) { } /** Make checkpoint for flush memTable info. */ - public void makeFlushMemTableCP(long memTableId) { + public void makeFlushMemTableCP(long memTableId) throws IOException { infoLock.lock(); long start = System.nanoTime(); try { @@ -163,12 +163,12 @@ public void makeFlushMemTableCP(long memTableId) { if (memTableInfo == null || checkpointPublicationFailure != null) { return; } - memTableInfo.setFlushed(); - memTableId2Info.remove(memTableId); Checkpoint checkpoint = new Checkpoint( CheckpointType.FLUSH_MEMORY_TABLE, Collections.singletonList(memTableInfo)); logByCachedByteBuffer(checkpoint); + memTableInfo.setFlushed(); + memTableId2Info.remove(memTableId); } finally { WRITING_METRICS.recordMakeCheckpointCost( CheckpointType.FLUSH_MEMORY_TABLE, System.nanoTime() - start); @@ -176,12 +176,13 @@ public void makeFlushMemTableCP(long memTableId) { } } - private void logByCachedByteBuffer(Checkpoint checkpoint) { + private void logByCachedByteBuffer(Checkpoint checkpoint) throws IOException { try { writeCheckpoint(currentLogWriter, checkpoint); } catch (IOException e) { logger.error(StorageEngineMessages.FAIL_TO_MAKE_CHECKPOINT, checkpoint, e); DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); + throw e; } } @@ -207,8 +208,8 @@ public void checkCheckpointPublication() throws IOException { } // region Task to fsync checkpoint file - /** Fsync checkpoints to the disk. */ - public void fsyncCheckpointFile() { + /** Fsyncs the active checkpoint before attempting to roll. */ + public void fsyncCheckpointFile() throws IOException { infoLock.lock(); try { if (checkpointPublicationFailure != null) { @@ -223,8 +224,7 @@ public void fsyncCheckpointFile() { identifier, e); DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); - CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e); - return; + throw e; } try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/CheckpointRecoverUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/CheckpointRecoverUtils.java index e593a5e5bebde..1a82ad51e7d2c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/CheckpointRecoverUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/CheckpointRecoverUtils.java @@ -45,19 +45,28 @@ public static CheckpointInfo recoverMemTableInfo(File logDirectory) { } // desc sort by version id CheckpointFileUtils.descSortByVersionId(checkpointFiles); - // Read back to the latest complete snapshot. Legacy rolls can have readable incremental - // checkpoints even if writing their snapshot failed. + // Read back to a complete snapshot within the newest contiguous run of readable checkpoints. + // A missing version may have removed mappings that are still present in an older snapshot. Deque readers = new ArrayDeque<>(); + int nextVersion = -1; for (File checkpointFile : checkpointFiles) { + int version = CheckpointFileUtils.parseVersionId(checkpointFile.getName()); + if (!readers.isEmpty() && version != nextVersion) { + break; + } CheckpointReader reader = new CheckpointReader(checkpointFile); List checkpoints = reader.getCheckpoints(); if (checkpoints.isEmpty()) { + if (!readers.isEmpty()) { + break; + } continue; } readers.addFirst(reader); if (checkpoints.get(0).getType() == CheckpointType.GLOBAL_MEMORY_TABLE_INFO) { break; } + nextVersion = version - 1; } // Apply the snapshot and the readable incremental checkpoints in version order. diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointRollRecoveryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointRollRecoveryTest.java index 2e9c49efac531..d4cfbb72f2906 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointRollRecoveryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointRollRecoveryTest.java @@ -59,6 +59,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; import static org.apache.iotdb.db.storageengine.dataregion.wal.WALTestUtils.getInsertRowNode; import static org.junit.Assert.assertArrayEquals; @@ -215,9 +216,7 @@ public void testFailedPublicationRollbackStopsWalWrites() throws Exception { WALBuffer buffer = new WALBuffer("checkpoint-roll", directory.getPath(), manager, 0, 0)) { addCheckpoint(manager, table); manager.fsyncCheckpointFile(); - WALInfoEntry first = new WALInfoEntry(1, getInsertRowNode(DEVICE, 10), true); - buffer.write(first); - assertEquals(WALFlushListener.Status.SUCCESS, first.getWalFlushListener().waitForResult()); + assertEquals(WALFlushListener.Status.SUCCESS, writeWalRow(buffer, 1, 10).waitForResult()); long walSize = buffer.getCurrentWALOriginalFileSize(); byte[] oldBytes = Files.readAllBytes(oldFile.toPath()); @@ -235,9 +234,7 @@ public void testFailedPublicationRollbackStopsWalWrites() throws Exception { commonConfig.setNodeStatus(NodeStatus.Running); addCheckpoint(manager, memTable(2)); - WALInfoEntry second = new WALInfoEntry(2, getInsertRowNode(DEVICE, 20), true); - buffer.write(second); - WALFlushListener listener = second.getWalFlushListener(); + WALFlushListener listener = writeWalRow(buffer, 2, 20); assertEquals(WALFlushListener.Status.FAILURE, listener.waitForResult()); assertSame(syncFailure, listener.getCause()); assertArrayEquals(new Throwable[] {rollbackFailure}, syncFailure.getSuppressed()); @@ -248,6 +245,64 @@ public void testFailedPublicationRollbackStopsWalWrites() throws Exception { assertReplay(table, 10); } + @Test(timeout = 30000) + public void testActiveCheckpointForceFailureFailsWalBatch() throws Exception { + assertActiveCheckpointFailureFailsWalBatch(true); + } + + @Test(timeout = 30000) + public void testActiveCheckpointWriteFailureFailsWalBatch() throws Exception { + assertActiveCheckpointFailureFailsWalBatch(false); + } + + private void assertActiveCheckpointFailureFailsWalBatch(boolean failForce) throws Exception { + MemTableInfo first = memTable(1); + File activeFile = checkpointFile(1); + byte[] durableCheckpoint; + try (CheckpointManager manager = + spy(new CheckpointManager("checkpoint-roll", directory.getPath())); + WALBuffer buffer = new WALBuffer("checkpoint-roll", directory.getPath(), manager, 0, 0)) { + // Publish a real successor and capture its writer so the fault affects the active file. + AtomicReference rolledWriter = new AtomicReference<>(); + doAnswer( + invocation -> { + CheckpointWriter writer = spy(new CheckpointWriter(invocation.getArgument(0))); + rolledWriter.set(writer); + return writer; + }) + .when(manager) + .openCheckpointWriter(any(File.class)); + config.setCheckpointFileSizeThresholdInByte(1); + addCheckpoint(manager, first); + manager.fsyncCheckpointFile(); + CheckpointWriter active = rolledWriter.get(); + assertEquals(activeFile, active.getLogFile()); + durableCheckpoint = Files.readAllBytes(activeFile.toPath()); + assertEquals(WALFlushListener.Status.SUCCESS, writeWalRow(buffer, 1, 10).waitForResult()); + long walSize = buffer.getCurrentWALOriginalFileSize(); + + IOException failure = new IOException("injected active checkpoint I/O failure"); + if (failForce) { + doThrow(failure).doCallRealMethod().when(active).force(); + } else { + doThrow(failure).when(active).write(any(ByteBuffer.class)); + } + MemTableInfo second = memTable(2); + manager.makeCreateMemTableCPInMemory(second); + buffer.write( + new WALInfoEntry(2, new Checkpoint(CheckpointType.CREATE_MEMORY_TABLE, List.of(second)))); + WALFlushListener listener = writeWalRow(buffer, 2, 20); + assertEquals(WALFlushListener.Status.FAILURE, listener.waitForResult()); + assertSame(failure, listener.getCause()); + assertEquals(walSize, buffer.getCurrentWALOriginalFileSize()); + assertFalse(checkpointFile(2).exists()); + } + + // Recover from the bytes that were durable before the active checkpoint failed. + Files.write(activeFile.toPath(), durableCheckpoint); + assertReplay(first, 10); + } + private CheckpointWriter writerWithDirectorySyncFailure( File candidate, IOException syncFailure, IOException rollbackFailure) throws IOException { return new CheckpointWriter(candidate) { @@ -349,23 +404,74 @@ public void testRecoveryReplaysLegacyIncrementsWithoutSnapshot() throws Exceptio assertReplay(first, 10); } + @Test + public void testRecoveryReplaysLegacyIncrementsAfterMissingVersion() throws Exception { + assertRecoveryReplaysLegacyIncrementsAfterGap(false); + } + + @Test + public void testRecoveryReplaysLegacyIncrementsAfterUnreadableVersion() throws Exception { + assertRecoveryReplaysLegacyIncrementsAfterGap(true); + } + + private void assertRecoveryReplaysLegacyIncrementsAfterGap(boolean keepEmptyFile) + throws Exception { + MemTableInfo stale = memTable(1); + MemTableInfo replacement = memTable(2, stale.getTsFilePath()); + writeCheckpointFile( + checkpointFile(0), + 1, + new Checkpoint(CheckpointType.GLOBAL_MEMORY_TABLE_INFO, List.of(stale))); + // The checkpoint that flushed the old memtable is missing or has no readable records. + if (keepEmptyFile) { + Files.createFile(checkpointFile(1).toPath()); + } + writeCheckpointFile( + checkpointFile(2), + 2, + new Checkpoint(CheckpointType.CREATE_MEMORY_TABLE, List.of(replacement))); + writeCheckpointFile( + checkpointFile(3), + 3, + new Checkpoint(CheckpointType.CREATE_MEMORY_TABLE, List.of(memTable(3)))); + Files.createFile(checkpointFile(5).toPath()); + writeWalEntry(replacement, 20); + + CheckpointRecoverUtils.CheckpointInfo recovered = + CheckpointRecoverUtils.recoverMemTableInfo(directory); + assertReplay(replacement, 20); + assertEquals(3, recovered.getMaxMemTableId()); + assertEquals(Set.of(2L, 3L), recovered.getMemTableId2Info().keySet()); + } + private File checkpointFile(long version) { return new File(directory, CheckpointFileUtils.getLogFileName(version)); } private MemTableInfo memTable(long id) { + File tsFile = new File(folder.getRoot(), "sequence/root.recovery/0/0/" + id + "-1-0-0.tsfile"); + return memTable(id, tsFile.getAbsolutePath()); + } + + private MemTableInfo memTable(long id, String tsFilePath) { IMemTable table = mock(IMemTable.class); when(table.getMemTableId()).thenReturn(id); when(table.getDataRegionId()).thenReturn("0"); - File tsFile = new File(folder.getRoot(), "sequence/root.recovery/0/0/" + id + "-1-0-0.tsfile"); - return new MemTableInfo(table, tsFile.getAbsolutePath(), 0); + return new MemTableInfo(table, tsFilePath, 0); } - private void addCheckpoint(CheckpointManager manager, MemTableInfo table) { + private void addCheckpoint(CheckpointManager manager, MemTableInfo table) throws IOException { manager.makeCreateMemTableCPInMemory(table); manager.makeCreateMemTableCPOnDisk(table.getMemTableId()); } + private WALFlushListener writeWalRow(WALBuffer buffer, long memTableId, long time) + throws Exception { + WALInfoEntry entry = new WALInfoEntry(memTableId, getInsertRowNode(DEVICE, time), true); + buffer.write(entry); + return entry.getWalFlushListener(); + } + private void writeCheckpointFile(File file, long maxMemTableId, Checkpoint checkpoint) throws IOException { try (CheckpointWriter writer = new CheckpointWriter(file)) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java index 2b268dbc8d16f..0746b7022313a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java @@ -159,7 +159,11 @@ public void testNormalProcedure() throws Exception { } private void prepareCheckpointAndWALFileForNormal() - throws MetadataException, ExecutionException, InterruptedException, QueryProcessException { + throws MetadataException, + ExecutionException, + InterruptedException, + QueryProcessException, + IOException { // write useless .wal files, start write threads to write concurrently int threadsNum = 5; ExecutorService executorService = Executors.newFixedThreadPool(threadsNum); @@ -227,7 +231,11 @@ public void testMemTableSnapshot() throws Exception { } private void prepareCheckpointAndWALFileForSnapshot() - throws MetadataException, ExecutionException, InterruptedException, QueryProcessException { + throws MetadataException, + ExecutionException, + InterruptedException, + QueryProcessException, + IOException { // write useless .wal files, start write threads to write concurrently int threadsNum = 5; ExecutorService executorService = Executors.newFixedThreadPool(threadsNum); From 2345e3242675c18572a025b71534ddbcf470f1e6 Mon Sep 17 00:00:00 2001 From: HTHou Date: Mon, 16 Nov 2026 15:04:33 +0800 Subject: [PATCH 4/4] [WAL] Replay all active memtables for each TsFile --- .../wal/recover/WALNodeRecoverTask.java | 59 +++++---- .../wal/recover/WALRecoverManager.java | 9 +- .../wal/recover/WALNodeRecoverTaskTest.java | 124 +++++++++++++++--- 3 files changed, 139 insertions(+), 53 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java index a9d3e46a2d723..1a1840fb2775d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java @@ -62,12 +62,13 @@ import java.util.concurrent.atomic.AtomicLong; import static org.apache.iotdb.consensus.iot.log.ConsensusReqReader.DEFAULT_SEARCH_INDEX; +import static org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils.getTsFileRelativePath; /** This task is responsible for the recovery of one wal node. */ public class WALNodeRecoverTask implements Runnable { private static final Logger logger = LoggerFactory.getLogger(WALNodeRecoverTask.class); private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private static final WALRecoverManager walRecoverManger = WALRecoverManager.getInstance(); + private static final WALRecoverManager walRecoverManager = WALRecoverManager.getInstance(); // this directory store one wal node's .wal and .checkpoint files private final File logDirectory; @@ -76,8 +77,11 @@ public class WALNodeRecoverTask implements Runnable { // version id of first valid .wal file private long firstValidVersionId = Long.MAX_VALUE; private Map memTableId2Info; - private Map memTableId2RecoverPerformer = new HashMap<>(); - private final Set failedMemTables = new HashSet<>(); + private final Map memTableId2RecoverPerformer = + new HashMap<>(); + private final Map tsFilePath2RecoverPerformer = + new HashMap<>(); + private final Set failedRecoverPerformers = new HashSet<>(); private boolean recoveryFailed; public WALNodeRecoverTask(File logDirectory, CountDownLatch allNodesRecoveredLatch) { @@ -126,14 +130,16 @@ private void recoverNode() { } catch (Exception e) { recoveryFailed = true; WALManager.getInstance().markRecoveryFailed(logDirectory.getName(), e); - for (UnsealedTsFileRecoverPerformer recoverPerformer : memTableId2RecoverPerformer.values()) { - recoverPerformer.getRecoverListener().fail(e); + for (UnsealedTsFileRecoverPerformer recoverPerformer : tsFilePath2RecoverPerformer.values()) { + if (failedRecoverPerformers.add(recoverPerformer)) { + recoverPerformer.getRecoverListener().fail(e); + } } if (WALFormatException.isFatal(e)) { throw new IllegalStateException(e); } } finally { - for (UnsealedTsFileRecoverPerformer recoverPerformer : memTableId2RecoverPerformer.values()) { + for (UnsealedTsFileRecoverPerformer recoverPerformer : tsFilePath2RecoverPerformer.values()) { try { if (recoveryFailed || !recoverPerformer.canWrite()) { recoverPerformer.close(); @@ -314,33 +320,37 @@ private void recoverInfoFromCheckpoints() { oldVal = memTableIdCounter.get(); } } - // update firstValidVersionId and get recover performer from WALRecoverManager - memTableId2RecoverPerformer = new HashMap<>(); + // Claim each TsFile's performer once and route all of its active memTables to it. for (MemTableInfo memTableInfo : memTableId2Info.values()) { firstValidVersionId = Math.min(firstValidVersionId, memTableInfo.getFirstFileVersionId()); - UnsealedTsFileRecoverPerformer recoverPerformer = - walRecoverManger.removeRecoverPerformer(new File(memTableInfo.getTsFilePath())); - if (recoverPerformer != null) { - memTableId2RecoverPerformer.put(memTableInfo.getMemTableId(), recoverPerformer); + File tsFile = new File(memTableInfo.getTsFilePath()); + try { + String relativePath = getTsFileRelativePath(tsFile.getCanonicalPath()); + UnsealedTsFileRecoverPerformer recoverPerformer = + tsFilePath2RecoverPerformer.computeIfAbsent( + relativePath, walRecoverManager::removeRecoverPerformer); + if (recoverPerformer != null) { + memTableId2RecoverPerformer.put(memTableInfo.getMemTableId(), recoverPerformer); + } + } catch (IOException e) { + logger.error(StorageEngineMessages.FAIL_TO_REMOVE_RECOVER_PERFORMER, tsFile, e); } } } private void recoverTsFiles() { - if (memTableId2RecoverPerformer.isEmpty()) { + if (tsFilePath2RecoverPerformer.isEmpty()) { return; } // make preparation for recovery - for (Map.Entry entry : - memTableId2RecoverPerformer.entrySet()) { - UnsealedTsFileRecoverPerformer recoverPerformer = entry.getValue(); + for (UnsealedTsFileRecoverPerformer recoverPerformer : tsFilePath2RecoverPerformer.values()) { try { recoverPerformer.startRecovery(); } catch (Exception e) { recoveryFailed = true; WALManager.getInstance().markRecoveryFailed(logDirectory.getName(), e); - failedMemTables.add(entry.getKey()); + failedRecoverPerformers.add(recoverPerformer); recoverPerformer.getRecoverListener().fail(e); if (WALFormatException.isFatal(e)) { throw new IllegalStateException(e); @@ -371,14 +381,15 @@ private void recoverTsFiles() { // see WALInfoEntry#serialize, entry type buffer.position(Byte.BYTES); long memTableId = buffer.getLong(); - if (!memTableId2Info.containsKey(memTableId) || failedMemTables.contains(memTableId)) { + UnsealedTsFileRecoverPerformer recoverPerformer = + memTableId2RecoverPerformer.get(memTableId); + if (!memTableId2Info.containsKey(memTableId) + || failedRecoverPerformers.contains(recoverPerformer)) { continue; } buffer.clear(); WALEntry walEntry = WALEntry.deserialize(new DataInputStream(new ByteArrayInputStream(buffer.array()))); - UnsealedTsFileRecoverPerformer recoverPerformer = - memTableId2RecoverPerformer.get(walEntry.getMemTableId()); if (recoverPerformer != null) { recoverPerformer.redoLog(walEntry); } else { @@ -416,19 +427,17 @@ private void recoverTsFiles() { private void endRecovery() { // end recovering all recover performers - for (Map.Entry entry : - memTableId2RecoverPerformer.entrySet()) { - if (failedMemTables.contains(entry.getKey())) { + for (UnsealedTsFileRecoverPerformer recoverPerformer : tsFilePath2RecoverPerformer.values()) { + if (failedRecoverPerformers.contains(recoverPerformer)) { continue; } - UnsealedTsFileRecoverPerformer recoverPerformer = entry.getValue(); try { recoverPerformer.endRecovery(); recoverPerformer.getRecoverListener().succeed(); } catch (Exception e) { recoveryFailed = true; WALManager.getInstance().markRecoveryFailed(logDirectory.getName(), e); - failedMemTables.add(entry.getKey()); + failedRecoverPerformers.add(recoverPerformer); recoverPerformer.getRecoverListener().fail(e); if (WALFormatException.isFatal(e)) { throw new IllegalStateException(e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManager.java index c6ae45b497c7a..43106938b4785 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManager.java @@ -242,13 +242,8 @@ public WALRecoverListener addRecoverPerformer(UnsealedTsFileRecoverPerformer rec return recoverPerformer.getRecoverListener(); } - UnsealedTsFileRecoverPerformer removeRecoverPerformer(File file) { - try { - return absolutePath2RecoverPerformer.remove(getTsFileRelativePath(file.getCanonicalPath())); - } catch (IOException e) { - logger.error(StorageEngineMessages.FAIL_TO_REMOVE_RECOVER_PERFORMER, file, e); - } - return null; + UnsealedTsFileRecoverPerformer removeRecoverPerformer(String tsFileRelativePath) { + return absolutePath2RecoverPerformer.remove(tsFileRelativePath); } public ExceptionalCountDownLatch getAllDataRegionScannedLatch() { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTaskTest.java index 536c70db6f10e..d0cb794d5bdab 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTaskTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTaskTest.java @@ -45,6 +45,9 @@ import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Before; @@ -63,6 +66,7 @@ import java.util.List; import java.util.concurrent.CountDownLatch; +import static org.apache.iotdb.db.storageengine.dataregion.wal.WALTestUtils.getInsertRowNode; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -73,6 +77,7 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -139,6 +144,68 @@ public void testIoTConsensusFinalizationFailureBlocksNodeReuse() throws Exceptio false); } + @Test(timeout = 30000) + public void testRecoveryReplaysAllMemTablesForSameTsFile() throws Exception { + File node = folder.newFolder("node"); + String relativePath = "sequence/root.recovery/0/0/1-1-0-0.tsfile"; + File tsFile = new File(folder.getRoot(), relativePath); + // A legacy checkpoint may still refer to the file's previous data directory. + File previousPath = new File(folder.getRoot(), "previous/" + relativePath); + Files.createDirectories(tsFile.toPath().getParent()); + Files.createFile(tsFile.toPath()); + File oldCheckpoint = new File(node, "_0.checkpoint"); + File nextCheckpoint = new File(node, "_1.checkpoint"); + writeCheckpoint( + oldCheckpoint, + 1, + new Checkpoint( + CheckpointType.GLOBAL_MEMORY_TABLE_INFO, List.of(memTable(1, previousPath, 0)))); + writeCheckpoint( + nextCheckpoint, + 2, + new Checkpoint(CheckpointType.CREATE_MEMORY_TABLE, List.of(memTable(2, tsFile, 0)))); + String device = "root.recovery.d1"; + File wal = new File(node, "_0-0-0.wal"); + writeWalEntries( + wal, + List.of( + new WALInfoEntry(1, getInsertRowNode(device, 10)), + new WALInfoEntry(2, getInsertRowNode(device, 20)))); + + UnsealedTsFileRecoverPerformer failed = register(tsFile); + IOException failure = new IOException("injected initialization failure"); + doThrow(failure).when(failed).startRecovery(); + new WALNodeRecoverTask(node, new CountDownLatch(1)).run(); + assertEquals(WALRecoverListener.Status.FAILURE, failed.getRecoverListener().waitForResult()); + assertSame(failure, failed.getRecoverListener().getCause()); + verify(failed).startRecovery(); + verify(failed, never()).redoLog(any()); + verify(failed, never()).endRecovery(); + verify(failed).close(); + assertTrue(oldCheckpoint.exists()); + assertTrue(nextCheckpoint.exists()); + assertTrue(wal.exists()); + + UnsealedTsFileRecoverPerformer performer = + spy(new UnsealedTsFileRecoverPerformer(new TsFileResource(tsFile), true, false, p -> {})); + WALRecoverListener listener = recovery.addRecoverPerformer(performer); + new WALNodeRecoverTask(node, new CountDownLatch(1)).run(); + assertEquals(WALRecoverListener.Status.SUCCESS, listener.waitForResult()); + verify(performer).startRecovery(); + verify(performer).endRecovery(); + verify(performer).close(); + assertFalse(node.exists()); + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { + Statistics statistics = + reader + .readTimeseriesMetadata(IDeviceID.Factory.DEFAULT_FACTORY.create(device), "s1", true) + .getStatistics(); + assertEquals(2, statistics.getCount()); + assertEquals(10, statistics.getStartTime()); + assertEquals(20, statistics.getEndTime()); + } + } + private void assertFailedRecovery(Exception failure, boolean finalization, boolean retry) throws Exception { File node = folder.newFolder("node"); @@ -156,10 +223,7 @@ private void assertFailedRecovery(Exception failure, boolean finalization, boole File file = new File(folder.getRoot(), "sequence/root.recovery/0/0/" + id + "-1-0-0.tsfile"); files.add(file); performers.add(register(file)); - IMemTable table = mock(IMemTable.class); - when(table.getMemTableId()).thenReturn(id); - when(table.getDataRegionId()).thenReturn("0"); - tables.add(new MemTableInfo(table, file.getAbsolutePath(), firstVersion)); + tables.add(memTable(id, file, firstVersion)); InsertRowNode row = new InsertRowNode( new PlanNodeId(""), @@ -177,25 +241,11 @@ private void assertFailedRecovery(Exception failure, boolean finalization, boole } entries.add(new WALInfoEntry(id, row)); } - Checkpoint checkpoint = new Checkpoint(CheckpointType.GLOBAL_MEMORY_TABLE_INFO, tables); - ByteBuffer checkpointBytes = ByteBuffer.allocate(Long.BYTES + checkpoint.serializedSize()); - checkpointBytes.putLong(2); - checkpoint.serialize(checkpointBytes); File checkpointFile = new File(node, "_0.checkpoint"); - try (CheckpointWriter writer = new CheckpointWriter(checkpointFile)) { - writer.write(checkpointBytes, false); - } + writeCheckpoint( + checkpointFile, 2, new Checkpoint(CheckpointType.GLOBAL_MEMORY_TABLE_INFO, tables)); File wal = new File(node, stableNode ? "_7-11-1.wal" : "_0-0-0.wal"); - try (WALWriter writer = new WALWriter(wal)) { - for (WALEntry entry : entries) { - ByteBuffer data = ByteBuffer.allocate(entry.serializedSize()); - entry.serialize(new WALByteBufferForTest(data)); - WALMetaData metadata = new WALMetaData(); - metadata.add( - data.position(), stableNode ? 10 + entry.getMemTableId() : -1, entry.getMemTableId()); - writer.write(data, metadata); - } - } + writeWalEntries(wal, entries); byte[] walBefore = Files.readAllBytes(wal.toPath()); byte[] checkpointBefore = Files.readAllBytes(checkpointFile.toPath()); File candidate = CheckpointFileUtils.createCheckpointCandidate(new File(node, "_1.checkpoint")); @@ -291,6 +341,38 @@ private void assertFailedRecovery(Exception failure, boolean finalization, boole } } + private static MemTableInfo memTable(long id, File tsFile, long firstVersion) { + IMemTable table = mock(IMemTable.class); + when(table.getMemTableId()).thenReturn(id); + when(table.getDataRegionId()).thenReturn("0"); + return new MemTableInfo(table, tsFile.getAbsolutePath(), firstVersion); + } + + private static void writeCheckpoint(File file, long maxMemTableId, Checkpoint checkpoint) + throws IOException { + ByteBuffer bytes = ByteBuffer.allocate(Long.BYTES + checkpoint.serializedSize()); + bytes.putLong(maxMemTableId); + checkpoint.serialize(bytes); + try (CheckpointWriter writer = new CheckpointWriter(file)) { + writer.write(bytes, false); + } + } + + private static void writeWalEntries(File wal, List entries) throws IOException { + try (WALWriter writer = new WALWriter(wal)) { + for (WALEntry entry : entries) { + ByteBuffer data = ByteBuffer.allocate(entry.serializedSize()); + entry.serialize(new WALByteBufferForTest(data)); + WALMetaData metadata = new WALMetaData(); + metadata.add( + data.position(), + ((InsertRowNode) entry.getValue()).getSearchIndex(), + entry.getMemTableId()); + writer.write(data, metadata); + } + } + } + private UnsealedTsFileRecoverPerformer register(File file) { UnsealedTsFileRecoverPerformer performer = mock(UnsealedTsFileRecoverPerformer.class); WALRecoverListener listener = new WALRecoverListener(file.getAbsolutePath());