diff --git a/example/session/src/main/java/org/apache/iotdb/ConsensusSubscriptionWalFileAnalyzer.java b/example/session/src/main/java/org/apache/iotdb/ConsensusSubscriptionWalFileAnalyzer.java index 0b8c165f9fdac..ee8bf55f5880a 100644 --- a/example/session/src/main/java/org/apache/iotdb/ConsensusSubscriptionWalFileAnalyzer.java +++ b/example/session/src/main/java/org/apache/iotdb/ConsensusSubscriptionWalFileAnalyzer.java @@ -23,10 +23,10 @@ import java.io.File; import java.io.IOException; -import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.StandardOpenOption; import java.util.Locale; /** @@ -79,8 +79,7 @@ private static void printUsage() { } private static WalFileAnalysis analyze(final File walFile) throws IOException { - try (RandomAccessFile raf = new RandomAccessFile(walFile, "r"); - FileChannel channel = raf.getChannel()) { + try (FileChannel channel = FileChannel.open(walFile.toPath(), StandardOpenOption.READ)) { final long totalBytes = channel.size(); final String version = detectVersion(channel, totalBytes); final int headMagicBytes = getHeadMagicBytes(version); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java index 53d7140a6f455..63c2041199d82 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java @@ -25,6 +25,7 @@ import org.apache.tsfile.utils.Pair; import java.io.*; +import java.nio.file.Files; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -92,8 +93,12 @@ public static int[] searchAvailablePorts() { // ignore } // Delete the lock file if the ports can't be used or some error happens - if (lockFile.exists() && !lockFile.delete()) { - IoTDBTestLogger.logger.error("Delete lockfile {} failed", lockFilePath); + try { + if (lockFile.exists() && !Files.deleteIfExists(lockFile.toPath())) { + IoTDBTestLogger.logger.error("Delete lockfile {} failed", lockFilePath); + } + } catch (IOException e) { + IoTDBTestLogger.logger.error("Delete lockfile {} failed", lockFilePath, e); } } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java index 3819bd32a8d62..af673a402d221 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java @@ -78,6 +78,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -680,8 +681,12 @@ public void cleanClusterEnvironment() { nodeWrapper.stopForcibly(); nodeWrapper.destroyDir(); final String lockPath = EnvUtils.getLockFilePath(nodeWrapper.getPort()); - if (!new File(lockPath).delete()) { - logger.error("Delete lock file {} failed", lockPath); + try { + if (!Files.deleteIfExists(new File(lockPath).toPath())) { + logger.error("Delete lock file {} failed", lockPath); + } + } catch (IOException e) { + logger.error("Delete lock file {} failed", lockPath, e); } } if (clientManager != null) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/datastructure/SerializableList.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/datastructure/SerializableList.java index fb1b86ad8bb92..8365de016f7a2 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/datastructure/SerializableList.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/datastructure/SerializableList.java @@ -27,9 +27,9 @@ import org.apache.tsfile.utils.PublicBAOS; import java.io.IOException; -import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; public interface SerializableList { @@ -87,7 +87,6 @@ class SerializationRecorder { protected int serializedElementSize; protected String fileName; - protected RandomAccessFile file; protected FileChannel fileChannel; public SerializationRecorder(String queryId) { @@ -127,28 +126,21 @@ public int getSerializedElementSize() { return serializedElementSize; } - public RandomAccessFile getFile() throws IOException { - if (file == null) { - if (fileName == null) { - fileName = AbstractTemporaryQueryDataFileService.getInstance().register(this); - } - file = new RandomAccessFile(SystemFileFactory.INSTANCE.getFile(fileName), "rw"); - } - return file; - } - public void closeFile() throws IOException { - if (file == null) { - return; - } closeFileChannel(); - file.close(); - file = null; } public FileChannel getFileChannel() throws IOException { if (fileChannel == null) { - fileChannel = getFile().getChannel(); + if (fileName == null) { + fileName = AbstractTemporaryQueryDataFileService.getInstance().register(this); + } + fileChannel = + FileChannel.open( + SystemFileFactory.INSTANCE.getFile(fileName).toPath(), + StandardOpenOption.CREATE, + StandardOpenOption.READ, + StandardOpenOption.WRITE); } return fileChannel; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/ProcedureInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/ProcedureInfo.java index f8e20e360ced4..7409258b8c9fa 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/ProcedureInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/ProcedureInfo.java @@ -163,31 +163,29 @@ public TSStatus deleteProcedure(DeleteProcedurePlan deleteProcedurePlan) { } private static Optional loadProcedure(Path procedureFilePath) { - try (FileInputStream fis = new FileInputStream(procedureFilePath.toFile())) { + try (FileChannel channel = FileChannel.open(procedureFilePath)) { Procedure procedure = null; - try (FileChannel channel = fis.getChannel()) { - final long fileSize = channel.size(); - if (fileSize > PROCEDURE_LOAD_BUFFER_SIZE) { - throw new IOException( - String.format( - ConfigNodeMessages - .EXCEPTION_PROCEDURE_FILE_ARG_EXCEEDS_THE_LOAD_BUFFER_LIMIT_ARG_ACTUAL_SIZE_ARG_62375B4C, - procedureFilePath, - PROCEDURE_LOAD_BUFFER_SIZE, - fileSize)); - } - ByteBuffer byteBuffer = ByteBuffer.allocate((int) fileSize); - if (fileSize > 0) { - IOUtils.readFully(channel, byteBuffer); - byteBuffer.flip(); - procedure = ProcedureFactory.getInstance().create(byteBuffer); - byteBuffer.clear(); - } - return Optional.ofNullable(procedure); + final long fileSize = channel.size(); + if (fileSize > PROCEDURE_LOAD_BUFFER_SIZE) { + throw new IOException( + String.format( + ConfigNodeMessages + .EXCEPTION_PROCEDURE_FILE_ARG_EXCEEDS_THE_LOAD_BUFFER_LIMIT_ARG_ACTUAL_SIZE_ARG_62375B4C, + procedureFilePath, + PROCEDURE_LOAD_BUFFER_SIZE, + fileSize)); } + ByteBuffer byteBuffer = ByteBuffer.allocate((int) fileSize); + if (fileSize > 0) { + IOUtils.readFully(channel, byteBuffer); + byteBuffer.flip(); + procedure = ProcedureFactory.getInstance().create(byteBuffer); + byteBuffer.clear(); + } + return Optional.ofNullable(procedure); } catch (Exception e) { LOGGER.error(ConfigNodeMessages.LOAD_FAILED_IT_WILL_BE_DELETED, procedureFilePath, e); - if (!procedureFilePath.toFile().delete()) { + if (!FileUtils.deleteFileIfExist(procedureFilePath.toFile())) { LOGGER.error( ConfigNodeMessages.DELETED_FAILED_TAKE_APPROPRIATE_ACTION, procedureFilePath, e); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java index e2d3ed3e162af..d98a05c8a8cdc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java @@ -676,7 +676,8 @@ public boolean processTakeSnapshot(File snapshotDir) throws IOException, TExcept dataNodeInfoReadWriteLock.readLock().unlock(); configNodeInfoReadWriteLock.readLock().unlock(); for (int retry = 0; retry < 5; retry++) { - if (!tmpFile.exists() || tmpFile.delete()) { + if (!tmpFile.exists() + || org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tmpFile)) { break; } else { LOGGER.warn( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java index e71b61d7b338c..da2466c1e2458 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java @@ -1042,7 +1042,8 @@ public boolean processTakeSnapshot(File snapshotDir) throws TException, IOExcept } finally { // with or without success, delete temporary files anyway for (int retry = 0; retry < 5; retry++) { - if (!tmpFile.exists() || tmpFile.delete()) { + if (!tmpFile.exists() + || org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tmpFile)) { break; } else { LOGGER.warn( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 9c599da5840c6..6553fa6566ea9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -781,7 +781,8 @@ public boolean processDatabaseSchemaSnapshot( return tmpFile.renameTo(snapshotFile); } finally { for (int retry = 0; retry < 5; retry++) { - if (!tmpFile.exists() || tmpFile.delete()) { + if (!tmpFile.exists() + || org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tmpFile)) { break; } else { LOGGER.warn( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplatePreSetTable.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplatePreSetTable.java index 985e005e967ab..3ebfc5f2c516d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplatePreSetTable.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplatePreSetTable.java @@ -130,7 +130,8 @@ public boolean processTakeSnapshot(File snapshotDir) throws IOException { return tmpFile.renameTo(snapshotFile); } finally { for (int retry = 0; retry < 5; retry++) { - if (!tmpFile.exists() || tmpFile.delete()) { + if (!tmpFile.exists() + || org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tmpFile)) { break; } else { LOGGER.warn( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplateTable.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplateTable.java index 29878b47dfeb6..45b8264c40f50 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplateTable.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplateTable.java @@ -237,7 +237,8 @@ public boolean processTakeSnapshot(File snapshotDir) throws IOException { return tmpFile.renameTo(snapshotFile); } finally { for (int retry = 0; retry < 5; retry++) { - if (!tmpFile.exists() || tmpFile.delete()) { + if (!tmpFile.exists() + || org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tmpFile)) { break; } else { LOGGER.warn( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureWAL.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureWAL.java index d6ff7e4891dcc..2505352914665 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureWAL.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureWAL.java @@ -27,8 +27,6 @@ import org.slf4j.LoggerFactory; import java.io.DataOutputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; @@ -56,19 +54,17 @@ public ProcedureWAL(Path walFilePath, IProcedureFactory procedureFactory) { */ @TestOnly public void save(Procedure procedure) throws IOException { - File walTmp = new File(walFilePath + ".tmp"); - Path walTmpPath = walTmp.toPath(); + Path walTmpPath = Path.of(walFilePath + ".tmp"); Files.deleteIfExists(walTmpPath); Files.createFile(walTmpPath); - try (FileOutputStream fos = new FileOutputStream(walTmp); - FileChannel channel = fos.getChannel(); + try (FileChannel channel = + FileChannel.open(walTmpPath, java.nio.file.StandardOpenOption.WRITE); PublicBAOS publicBAOS = new PublicBAOS(); DataOutputStream dataOutputStream = new DataOutputStream(publicBAOS)) { procedure.serialize(dataOutputStream); channel.write(ByteBuffer.wrap(publicBAOS.getBuf(), 0, publicBAOS.size())); channel.force(true); - fos.getFD().sync(); } Files.deleteIfExists(walFilePath); Files.move(walTmpPath, walFilePath); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/writelog/io/SingleFileLogReader.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/writelog/io/SingleFileLogReader.java index d54b48f75d6f4..6681d213704bb 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/writelog/io/SingleFileLogReader.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/writelog/io/SingleFileLogReader.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.confignode.writelog.io; +import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; @@ -29,10 +30,8 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.util.NoSuchElementException; import java.util.zip.CRC32; @@ -147,9 +146,8 @@ public boolean isFileCorrupted() { } private void truncateBrokenLogs() { - try (FileOutputStream outputStream = new FileOutputStream(filepath, true); - FileChannel channel = outputStream.getChannel()) { - channel.truncate(unbrokenLogsSize); + try { + FileUtils.truncateFile(new File(filepath), unbrokenLogsSize); } catch (IOException e) { logger.error(ConfigNodeMessages.FAIL_TO_TRUNCATE_LOG_FILE_TO_SIZE, unbrokenLogsSize, e); } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java index 430507a40489f..d40ebe55d6579 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java @@ -84,13 +84,13 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -529,8 +529,9 @@ private void writeSnapshotFragment(File targetFile, ByteBuffer fileChunk, long f if (!Files.exists(parentDir)) { Files.createDirectories(parentDir); } - try (FileOutputStream fos = new FileOutputStream(targetFile.getAbsolutePath(), true); - FileChannel channel = fos.getChannel()) { + try (FileChannel channel = + FileChannel.open( + targetFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { channel.write(fileChunk.slice(), fileOffset); } } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java index 6c0366c546fe8..8c956c14a11e2 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java @@ -53,6 +53,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.DirectoryNotEmptyException; import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.util.Collection; @@ -276,8 +277,9 @@ public long takeSnapshot() throws IOException { private void deleteIncompleteSnapshot(File snapshotDir) throws IOException { // this takeSnapshot failed, clean up files and directories // statemachine is supposed to clear snapshotDir on failure - boolean isEmpty = snapshotDir.delete(); - if (!isEmpty) { + try { + Files.deleteIfExists(snapshotDir.toPath()); + } catch (DirectoryNotEmptyException e) { logger.info(RatisMessages.SNAPSHOT_DIR_INCOMPLETE_DELETING, snapshotDir.getAbsolutePath()); FileUtils.deleteFully(snapshotDir); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java index 8b7946998ac1a..d58c65158ac0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java @@ -36,11 +36,11 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.StandardOpenOption; import java.util.List; import java.util.Optional; import java.util.concurrent.BlockingQueue; @@ -93,7 +93,6 @@ public class PageCacheDeletionBuffer implements DeletionBuffer { private volatile ByteBuffer serializeBuffer; // Current Logging file. private volatile File logFile; - private volatile FileOutputStream logStream; private volatile FileChannel logChannel; // Max progressIndex among current .deletion file. Used by PersistTask for naming .deletion file. // Since deletions are written serially, DAL is also written serially. This ensures that the @@ -121,8 +120,12 @@ public void start() { new File( baseDirectory, String.format("_%d-%d%s", 0, 0, DeletionResourceManager.DELETION_FILE_SUFFIX)); - this.logStream = new FileOutputStream(logFile, true); - this.logChannel = logStream.getChannel(); + this.logChannel = + FileChannel.open( + logFile.toPath(), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.APPEND); // Create file && write magic string if (!logFile.exists() || logFile.length() == 0) { this.logChannel.write( @@ -186,9 +189,6 @@ private void fsyncCurrentLoggingFile() throws IOException { private void closeCurrentLoggingFile(boolean notifySuccess) throws IOException { LOGGER.info(DataNodePipeMessages.DELETION_PERSIST_CURRENT_FILE_HAS_BEEN_CLOSED, dataRegionId); // Close old resource to fsync. - if (this.logStream != null) { - this.logStream.close(); - } if (this.logChannel != null) { this.logChannel.close(); } @@ -243,8 +243,12 @@ private void switchLoggingFile() throws IOException { progressIndex.getRebootTimes(), progressIndex.getMemTableFlushOrderId(), DeletionResourceManager.DELETION_FILE_SUFFIX)); - this.logStream = new FileOutputStream(logFile, true); - this.logChannel = logStream.getChannel(); + this.logChannel = + FileChannel.open( + logFile.toPath(), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.APPEND); // Create file && write magic string if (!logFile.exists() || logFile.length() == 0) { this.logChannel.write( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/recover/DeletionReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/recover/DeletionReader.java index 8477e1e805934..b11805471a85b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/recover/DeletionReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/recover/DeletionReader.java @@ -29,11 +29,11 @@ import java.io.Closeable; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; @@ -45,15 +45,13 @@ public class DeletionReader implements Closeable { private final int regionId; private final Consumer removeHook; private final File logFile; - private final FileInputStream fileInputStream; private final FileChannel fileChannel; public DeletionReader(File logFile, int regionId, Consumer removeHook) throws IOException { this.logFile = logFile; this.regionId = regionId; - this.fileInputStream = new FileInputStream(logFile); - this.fileChannel = fileInputStream.getChannel(); + this.fileChannel = FileChannel.open(logFile.toPath(), StandardOpenOption.READ); this.removeHook = removeHook; } @@ -95,6 +93,5 @@ public List readAllDeletions() throws IOException { @Override public void close() throws IOException { this.fileChannel.close(); - this.fileInputStream.close(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/iotconsensusv2/IoTConsensusV2Receiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/iotconsensusv2/IoTConsensusV2Receiver.java index e185a424947e8..9303e4c01f41c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/iotconsensusv2/IoTConsensusV2Receiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/iotconsensusv2/IoTConsensusV2Receiver.java @@ -365,7 +365,8 @@ private TIoTConsensusV2TransferResp handleTransferFilePiece( // filename. However, for other files (mod, snapshot, etc.) the content varies for the // same name in different times, then we must rewrite the file to apply the newest // version. - writingFileWriter.setLength(0); + org.apache.iotdb.commons.utils.FileUtils.truncateFile(writingFile, 0); + writingFileWriter.seek(0); } final TSStatus status = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java index db60fc4645b63..8dfc4c1a8e632 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java @@ -24,7 +24,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodePipeMessages; -import org.apache.tsfile.external.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -114,10 +113,11 @@ private static void moveAsideAndCollect( } catch (final IOException e) { LOGGER.warn( DataNodePipeMessages.PIPE_HARDLINK_DIR_MOVE_FAILED_DELETING_SYNC, pipeHardLinkDir, e); + org.apache.iotdb.commons.utils.FileUtils.deleteFileOrDirectory(pipeHardLinkDir, true); LOGGER.info( DataNodePipeMessages.PIPE_HARDLINK_DIR_FOUND_DELETING_IT_RESULT, pipeHardLinkDir, - FileUtils.deleteQuietly(pipeHardLinkDir)); + !pipeHardLinkDir.exists()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTableModelTsFileBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTableModelTsFileBuilder.java index ce7e81e348125..57476d604b4b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTableModelTsFileBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTableModelTsFileBuilder.java @@ -189,7 +189,8 @@ List> writeTableModelTabletsToTsFiles( } for (final Pair sealedFile : sealedFiles) { - final boolean deleteSuccess = FileUtils.deleteQuietly(sealedFile.right); + final boolean deleteSuccess = + org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(sealedFile.right); LOGGER.warn( DataNodePipeMessages.BATCH_ID_DELETE_THE_TSFILE_AFTER_FAILED, currentBatchId.get(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTreeModelTsFileBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTreeModelTsFileBuilder.java index 7d6967db7ec42..ad4440ae79d81 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTreeModelTsFileBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTreeModelTsFileBuilder.java @@ -180,7 +180,8 @@ private List> writeTabletsToTsFiles() } for (final Pair sealedFile : sealedFiles) { - final boolean deleteSuccess = FileUtils.deleteQuietly(sealedFile.right); + final boolean deleteSuccess = + org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(sealedFile.right); LOGGER.warn( DataNodePipeMessages.BATCH_ID_DELETE_THE_TSFILE_AFTER_FAILED, currentBatchId.get(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTsFileBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTsFileBuilder.java index c34f1fa96b4a3..538154e667f99 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTsFileBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTsFileBuilder.java @@ -91,7 +91,7 @@ private File getNextBaseDir() throws DiskSpaceInsufficientException { .getNextWithRetry( folder -> { File dir = new File(folder, Long.toString(currentBatchId.get())); - FileUtils.deleteQuietly(dir); + org.apache.iotdb.commons.utils.FileUtils.deleteFileOrDirectory(dir, true); if (dir.mkdirs()) { LOGGER.info( DataNodePipeMessages.BATCH_ID_CREATE_BATCH_DIR_SUCCESSFULLY_BATCH, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index 48d391658403a..4ab1b25c41117 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -55,7 +55,6 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.EncryptUtils; -import org.apache.tsfile.external.commons.io.FileUtils; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -470,7 +469,7 @@ && handleSingleMiniFile(i)) { DataNodeQueryMessages.EMPTY_FILE_DETECTED_WILL_SKIP_LOADING_THIS_FILE, tsFile.getAbsolutePath()); if (isDeleteAfterLoad) { - FileUtils.deleteQuietly(tsFile); + org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tsFile); } } finally { // reset the session info to the original one diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/SchemaLogReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/SchemaLogReader.java index ae7ae7d0adf45..b3b26e3ac31a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/SchemaLogReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/SchemaLogReader.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.logfile; import org.apache.iotdb.commons.file.SystemFileFactory; +import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -32,10 +33,8 @@ import java.io.EOFException; import java.io.File; import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.channels.FileChannel; import java.util.NoSuchElementException; /** @@ -143,16 +142,15 @@ public boolean isFileCorrupted() { } private void truncateBrokenLogs() { - try (FileOutputStream outputStream = new FileOutputStream(logFile, true); - FileChannel channel = outputStream.getChannel()) { - if (currentIndex != channel.size()) { + try { + final long fileSize = logFile.length(); + if (currentIndex != fileSize) { LOGGER.warn( DataNodeSchemaMessages.LOG_FILE_END_CORRUPTED_TRUNCATE, logFile.getName(), currentIndex, - channel.size()); - channel.truncate(currentIndex); - channel.force(true); + fileSize); + FileUtils.truncateFile(logFile, currentIndex); } isFileCorrupted = false; } catch (IOException e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/SchemaLogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/SchemaLogWriter.java index f4cc8fdd7665e..0e63a4d38e521 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/SchemaLogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/SchemaLogWriter.java @@ -26,9 +26,12 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; import java.nio.file.Files; +import java.nio.file.StandardOpenOption; /** * This class provides the common ability to write a log storing T. @@ -41,7 +44,8 @@ public class SchemaLogWriter implements AutoCloseable { private final File logFile; - private final FileOutputStream fileOutputStream; + private final FileChannel fileChannel; + private final OutputStream fileOutputStream; private final ISerializer serializer; @@ -62,7 +66,8 @@ public SchemaLogWriter( } logFile = SystemFileFactory.INSTANCE.getFile(schemaDir + File.separator + logFileName); - fileOutputStream = new FileOutputStream(logFile, true); + fileChannel = openFileChannel(logFile); + fileOutputStream = Channels.newOutputStream(fileChannel); this.serializer = serializer; this.forceEachWrite = forceEachWrite; @@ -71,7 +76,8 @@ public SchemaLogWriter( public SchemaLogWriter(String logFilePath, ISerializer serializer, boolean forceEachWrite) throws IOException { logFile = SystemFileFactory.INSTANCE.getFile(logFilePath); - fileOutputStream = new FileOutputStream(logFile, true); + fileChannel = openFileChannel(logFile); + fileOutputStream = Channels.newOutputStream(fileChannel); this.serializer = serializer; this.forceEachWrite = forceEachWrite; @@ -91,11 +97,11 @@ public synchronized void force() throws IOException { return; } hasSynced = true; - fileOutputStream.getFD().sync(); + fileChannel.force(true); } private void syncBufferToDisk() throws IOException { - fileOutputStream.getFD().sync(); + fileChannel.force(true); hasSynced = true; } @@ -113,6 +119,14 @@ public synchronized void close() throws IOException { } public long position() throws IOException { - return fileOutputStream.getChannel().position(); + return fileChannel.position(); + } + + private static FileChannel openFileChannel(File file) throws IOException { + return FileChannel.open( + file.toPath(), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.APPEND); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/SchemaFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/SchemaFile.java index 665bafb956f78..968a4792e9ad0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/SchemaFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/SchemaFile.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.schema.node.role.IDatabaseMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; +import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.commons.utils.IOUtils; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; @@ -45,11 +46,11 @@ import java.io.File; import java.io.IOException; import java.io.PrintWriter; -import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.Iterator; /** @@ -123,7 +124,7 @@ private SchemaFile( pmtFile.createNewFile(); } - this.channel = new RandomAccessFile(pmtFile, "rw").getChannel(); + this.channel = openReadWriteChannel(pmtFile); this.headerContent = ByteBuffer.allocate(SchemaFileConfig.FILE_HEADER_SIZE); // will be overwritten if to init this.dataTTL = ttl; @@ -138,7 +139,7 @@ private SchemaFile(File file) throws IOException, MetadataException { pmtFile = file; filePath = pmtFile.getPath(); logPath = file.getParent() + File.separator + SchemaConstant.PBTREE_LOG_FILE_NAME; - channel = new RandomAccessFile(file, "rw").getChannel(); + channel = openReadWriteChannel(file); headerContent = ByteBuffer.allocate(SchemaFileConfig.FILE_HEADER_SIZE); if (channel.size() <= 0) { @@ -293,11 +294,19 @@ public void clear() throws IOException, MetadataException { } pmtFile.createNewFile(); - channel = new RandomAccessFile(pmtFile, "rw").getChannel(); + channel = openReadWriteChannel(pmtFile); headerContent = ByteBuffer.allocate(SchemaFileConfig.FILE_HEADER_SIZE); initFileHeader(); } + private static FileChannel openReadWriteChannel(File file) throws IOException { + return FileChannel.open( + file.toPath(), + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE); + } + public String inspect() throws MetadataException, IOException { return inspect(null); } @@ -464,7 +473,7 @@ public boolean createSnapshot(File snapshotDir) { SystemFileFactory.INSTANCE.getFile(snapshotDir, SchemaConstant.PBTREE_SNAPSHOT); try { sync(); - if (schemaFileSnapshot.exists() && !schemaFileSnapshot.delete()) { + if (schemaFileSnapshot.exists() && !FileUtils.deleteFileIfExist(schemaFileSnapshot)) { logger.error( DataNodeSchemaMessages.FAILED_TO_DELETE_OLD_PBTREE_SNAPSHOT, schemaFileSnapshot.getName()); @@ -474,7 +483,7 @@ public boolean createSnapshot(File snapshotDir) { return true; } catch (IOException e) { logger.error(DataNodeSchemaMessages.FAILED_TO_CREATE_SCHEMA_FILE_SNAPSHOT, e.getMessage(), e); - schemaFileSnapshot.delete(); + FileUtils.deleteFileIfExist(schemaFileSnapshot); return false; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/log/SchemaFileLogReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/log/SchemaFileLogReader.java index d2300384fa7ee..a763ffe556514 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/log/SchemaFileLogReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/log/SchemaFileLogReader.java @@ -32,6 +32,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -54,7 +55,7 @@ public SchemaFileLogReader(String logFilePath) throws IOException { } public List collectUpdatedEntries() throws IOException, SchemaFileLogCorruptedException { - if (inputStream == null || inputStream.getChannel().size() == 0) { + if (inputStream == null || Files.size(logFile.toPath()) == 0) { return Collections.emptyList(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageIOChannel.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageIOChannel.java index 4e9d6524f1996..f56e0c15d1b3c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageIOChannel.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageIOChannel.java @@ -26,7 +26,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.schemafile.log.SchemaFileLogWriter; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; @@ -95,9 +94,14 @@ private long recoverFromLog(String logPath) throws IOException, MetadataExceptio // complete log file if (!res.isEmpty()) { - try (FileOutputStream outputStream = new FileOutputStream(logPath, true)) { - outputStream.write(new byte[] {SchemaFileConfig.SF_COMMIT_MARK}); - return outputStream.getChannel().size(); + try (FileChannel logChannel = + FileChannel.open( + new File(logPath).toPath(), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.APPEND)) { + logChannel.write(ByteBuffer.wrap(new byte[] {SchemaFileConfig.SF_COMMIT_MARK})); + return logChannel.size(); } } return 0L; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java index 69e21e442de96..314e971e3acd7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java @@ -137,7 +137,7 @@ public static TagManager loadFromSnapshot( File tagSnapshot = SystemFileFactory.INSTANCE.getFile(snapshotDir, SchemaConstant.TAG_LOG_SNAPSHOT); File tagFile = SystemFileFactory.INSTANCE.getFile(sgSchemaDirPath, SchemaConstant.TAG_LOG); - if (tagFile.exists() && !tagFile.delete()) { + if (tagFile.exists() && !FileUtils.deleteFileIfExist(tagFile)) { logger.warn(DataNodeSchemaMessages.FAILED_TO_DELETE_EXISTING_WHEN_LOADING, tagFile.getName()); } @@ -145,7 +145,7 @@ public static TagManager loadFromSnapshot( org.apache.tsfile.external.commons.io.FileUtils.copyFile(tagSnapshot, tagFile); return new TagManager(sgSchemaDirPath, regionStatistics); } catch (IOException e) { - if (!tagFile.delete()) { + if (!FileUtils.deleteFileIfExist(tagFile)) { logger.warn( DataNodeSchemaMessages.FAILED_TO_DELETE_EXISTING_WHEN_COPY_FAILURE, tagFile.getName()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index eb63312ebb1de..cfdc63edbdfbe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -3233,7 +3233,7 @@ public void deleteByTable(RelationalDeleteDataNode node) throws IOException { } catch (IOException e) { logger.error(StorageEngineMessages.FAILED_TO_CHECK_OBJECT_FILES, e.getMessage()); } - FileUtils.deleteQuietly(objectTableDir); + org.apache.iotdb.commons.utils.FileUtils.deleteFileOrDirectory(objectTableDir, true); } FileMetrics.getInstance().decreaseObjectFileNum(count.get()); FileMetrics.getInstance().decreaseObjectFileSize(totalSize.get()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java index a243912240f0c..daf9de6489dc6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java @@ -155,7 +155,8 @@ public synchronized void removeDataRegionRatio(String dataRegionId) { dataRegionCompressionRatio.getRight()) + "." + dataRegionId); - if (!oldDataRegionFile.delete() && oldDataRegionFile.exists()) { + if (!org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(oldDataRegionFile) + && oldDataRegionFile.exists()) { LOGGER.warn(StorageEngineMessages.CANNOT_DELETE_OLD_COMPRESSION_FILE, oldDataRegionFile); } } @@ -237,7 +238,7 @@ private void recoverDataRegionRatio(File[] ratioFiles) { for (File ratioFile : ratioFiles) { if (ratioFile != null) { - if (!ratioFile.delete()) { + if (!org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(ratioFile)) { LOGGER.warn(StorageEngineMessages.CANNOT_DELETE_RATIO_FILE, ratioFile.getAbsolutePath()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/ModificationFileV1.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/ModificationFileV1.java index a0d625f016dff..5b938db9d8ff2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/ModificationFileV1.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/ModificationFileV1.java @@ -154,7 +154,9 @@ public void setFilePath(String filePath) { public void remove() throws IOException { close(); - boolean deleted = FSFactoryProducer.getFSFactory().getFile(filePath).delete(); + boolean deleted = + FSFactoryProducer.getFSFactory() + .deleteIfExists(FSFactoryProducer.getFSFactory().getFile(filePath)); if (!deleted) { logger.warn(StorageEngineMessages.DELETE_MODIFICATION_FILE_FAILED, filePath); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/io/LocalTextModificationAccessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/io/LocalTextModificationAccessor.java index ecc4be4f90972..2f1d5599a0766 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/io/LocalTextModificationAccessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/io/LocalTextModificationAccessor.java @@ -31,12 +31,16 @@ import org.slf4j.LoggerFactory; import java.io.BufferedReader; +import java.io.EOFException; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; -import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.NoSuchFileException; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -201,11 +205,11 @@ public void writeMeetException(Modification mod) throws IOException { @Override public void truncate(long size) { - try (FileOutputStream outputStream = - new FileOutputStream(FSFactoryProducer.getFSFactory().getFile(filePath), true)) { - outputStream.getChannel().truncate(size); + try { + org.apache.iotdb.commons.utils.FileUtils.truncateFile( + FSFactoryProducer.getFSFactory().getFile(filePath), size); logger.warn(StorageEngineMessages.MODIFICATIONS_WILL_BE_TRUNCATED, filePath, size); - } catch (FileNotFoundException e) { + } catch (NoSuchFileException e) { logger.debug(NO_MODIFICATION_MSG, filePath); } catch (IOException e) { logger.error( @@ -219,19 +223,20 @@ public void truncate(long size) { @Override public void mayTruncateLastLine() { - try (RandomAccessFile file = new RandomAccessFile(filePath, "r")) { - long filePointer = file.length() - 1; + try (FileChannel channel = + FileChannel.open( + FSFactoryProducer.getFSFactory().getFile(filePath).toPath(), StandardOpenOption.READ)) { + long filePointer = channel.size() - 1; if (filePointer <= 0) { return; } - file.seek(filePointer); - byte lastChar = file.readByte(); + ByteBuffer byteBuffer = ByteBuffer.allocate(Byte.BYTES); + byte lastChar = readByte(channel, byteBuffer, filePointer); if (lastChar != '\n') { while (filePointer > -1 && lastChar != '\n') { - file.seek(filePointer); + lastChar = readByte(channel, byteBuffer, filePointer); filePointer--; - lastChar = file.readByte(); } logger.warn(StorageEngineMessages.LAST_LINE_OF_MODS_INCOMPLETE); truncate(filePointer + 2); @@ -241,6 +246,16 @@ public void mayTruncateLastLine() { } } + private static byte readByte(FileChannel channel, ByteBuffer byteBuffer, long position) + throws IOException { + byteBuffer.clear(); + if (channel.read(byteBuffer, position) < Byte.BYTES) { + throw new EOFException(); + } + byteBuffer.flip(); + return byteBuffer.get(); + } + private static String encodeModification(Modification mod) { if (mod instanceof Deletion) { return encodeDeletion((Deletion) mod); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index a5ec8a555eb05..4b33759af1c40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -1619,7 +1619,7 @@ public void upgradeModFile(ExecutorService upgradeModFileThreadPool) throws IOEx @SuppressWarnings({"java:S4042", "java:S899", "ResultOfMethodCallIgnored"}) private ModificationFile doUpgradeModFile(ModificationFileV1 oldModFile) throws IOException { ModificationFile newMFile = ModificationFile.getExclusiveMods(this); - newMFile.getFile().delete(); + FileUtils.deleteFileIfExist(newMFile.getFile()); try { for (Modification oldMod : oldModFile.getModifications()) { newMFile.write(new TreeDeletionEntry((Deletion) oldMod)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheWriter.java index 7cef1dc4792ab..b11c7da451a95 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheWriter.java @@ -26,18 +26,17 @@ import java.io.File; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; import java.nio.file.Files; +import java.nio.file.StandardOpenOption; public class FileTimeIndexCacheWriter implements ILogWriter { private static final Logger logger = LoggerFactory.getLogger(FileTimeIndexCacheWriter.class); private final File logFile; - private FileOutputStream fileOutputStream; private FileChannel channel; private final boolean forceEachWrite; @@ -46,8 +45,7 @@ public FileTimeIndexCacheWriter(File logFile, boolean forceEachWrite) this.logFile = logFile; this.forceEachWrite = forceEachWrite; - fileOutputStream = new FileOutputStream(logFile, true); - channel = fileOutputStream.getChannel(); + channel = openChannel(logFile); } @Override @@ -82,8 +80,6 @@ public void close() throws IOException { if (channel.isOpen()) { channel.force(false); } - fileOutputStream.close(); - fileOutputStream = null; channel.close(); channel = null; } @@ -96,8 +92,12 @@ public void clearFile() throws IOException { logger.warn( StorageEngineMessages.PARTITION_LOG_FILE_ALREADY_EXISTS, logFile.getAbsolutePath()); } - fileOutputStream = new FileOutputStream(logFile, true); - channel = fileOutputStream.getChannel(); + channel = + FileChannel.open( + logFile.toPath(), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.APPEND); } @Override @@ -108,4 +108,18 @@ public String toString() { public File getLogFile() { return logFile; } + + private static FileChannel openChannel(File file) throws FileNotFoundException { + try { + return FileChannel.open( + file.toPath(), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.APPEND); + } catch (IOException e) { + FileNotFoundException exception = new FileNotFoundException(e.getMessage()); + exception.initCause(e); + throw exception; + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 37ccc3cb6cdd5..52675dae37e7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -31,11 +31,11 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; /** * LogWriter writes the binary logs into a file, including writing {@link WALEntry} into .wal file @@ -45,7 +45,6 @@ public abstract class LogWriter implements ILogWriter { private static final Logger logger = LoggerFactory.getLogger(LogWriter.class); protected final File logFile; - protected final FileOutputStream logStream; protected final FileChannel logChannel; protected long originalSize = 0; @@ -68,8 +67,12 @@ public abstract class LogWriter implements ILogWriter { protected LogWriter(File logFile, WALFileVersion version) throws IOException { this.logFile = logFile; - this.logStream = new FileOutputStream(logFile, true); - this.logChannel = this.logStream.getChannel(); + this.logChannel = + FileChannel.open( + logFile.toPath(), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.APPEND); if ((!logFile.exists() || logFile.length() == 0) && (version == WALFileVersion.V2 || version == WALFileVersion.V3)) { this.logChannel.write(ByteBuffer.wrap(version.getVersionBytes())); @@ -174,7 +177,6 @@ public void close() throws IOException { } } finally { logChannel.close(); - logStream.close(); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java index 728b3356b8887..a59ebeaa21db4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java @@ -398,7 +398,7 @@ private void deleteOutdatedFilesAndUpdateMetric() { long versionId = WALFileUtils.parseVersionId(currentWal.getName()); if (canDeleteFile(fileArrIdx, walFileStatus, versionId)) { long fileSize = currentWal.length(); - if (currentWal.delete()) { + if (org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(currentWal)) { deleteFileSize += fileSize; buffer.removeMemTableIdsOfWal(versionId); successfullyDeleted.add(versionId); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java index 46fe0129c2b24..5b470ee1c2bb9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java @@ -98,7 +98,7 @@ protected void recoverWithWriter() throws DataRegionException, IOException { if (versionNumber != TSFileConfig.VERSION_NUMBER) { // cannot rewrite a file with V3 header, delete it first writer.close(); - tsFile.delete(); + org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tsFile); writer = new RestorableTsFileIOWriter( tsFile, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java index 89854e23f04df..ce3ddccde92c2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java @@ -32,7 +32,6 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.rpc.TSStatusCode; -import org.apache.tsfile.external.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -125,11 +124,14 @@ public Optional visitLoadTsFile( .getTsFiles() .forEach( tsfile -> { - FileUtils.deleteQuietly(tsfile); + org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tsfile); final String tsFilePath = tsfile.getAbsolutePath(); - FileUtils.deleteQuietly(new File(LoadUtil.getTsFileResourcePath(tsFilePath))); - FileUtils.deleteQuietly(new File(LoadUtil.getTsFileModsV1Path(tsFilePath))); - FileUtils.deleteQuietly(new File(LoadUtil.getTsFileModsV2Path(tsFilePath))); + org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist( + new File(LoadUtil.getTsFileResourcePath(tsFilePath))); + org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist( + new File(LoadUtil.getTsFileModsV1Path(tsFilePath))); + org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist( + new File(LoadUtil.getTsFileModsV2Path(tsFilePath))); }); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java index 1922e335a290e..d96f5c1cd09f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java @@ -33,7 +33,6 @@ import org.apache.iotdb.db.storageengine.load.util.LoadUtil; import org.apache.iotdb.rpc.TSStatusCode; -import org.apache.tsfile.external.commons.io.FileUtils; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.record.Tablet; import org.slf4j.Logger; @@ -172,11 +171,14 @@ public Optional visitLoadFile( .getTsFiles() .forEach( tsfile -> { - FileUtils.deleteQuietly(tsfile); + org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tsfile); final String tsFilePath = tsfile.getAbsolutePath(); - FileUtils.deleteQuietly(new File(LoadUtil.getTsFileResourcePath(tsFilePath))); - FileUtils.deleteQuietly(new File(LoadUtil.getTsFileModsV1Path(tsFilePath))); - FileUtils.deleteQuietly(new File(LoadUtil.getTsFileModsV2Path(tsFilePath))); + org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist( + new File(LoadUtil.getTsFileResourcePath(tsFilePath))); + org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist( + new File(LoadUtil.getTsFileModsV1Path(tsFilePath))); + org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist( + new File(LoadUtil.getTsFileModsV2Path(tsFilePath))); }); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/DirectoryChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/DirectoryChecker.java index e7076cfbe186b..77bf6f4572912 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/DirectoryChecker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/DirectoryChecker.java @@ -29,20 +29,22 @@ import java.io.File; import java.io.IOException; -import java.io.RandomAccessFile; +import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.StandardCharsets; import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; public class DirectoryChecker { private static final Logger logger = LoggerFactory.getLogger(DirectoryChecker.class); private static final String LOCK_FILE_NAME = ".iotdb-lock"; - private final List randomAccessFileList = new ArrayList<>(); + private final List fileChannelList = new ArrayList<>(); private final List fileList = new ArrayList<>(); private DirectoryChecker() {} @@ -51,7 +53,7 @@ public static DirectoryChecker getInstance() { return DirectoryCheckerHolder.INSTANCE; } - @SuppressWarnings("java:S2095") // will be closed by randomAccessFileList + @SuppressWarnings("java:S2095") // will be closed by fileChannelList public void registerDirectory(File dir) throws ConfigurationException, IOException { if (dir.exists() && !dir.isDirectory()) { throw new ConfigurationException( @@ -69,8 +71,12 @@ public void registerDirectory(File dir) throws ConfigurationException, IOExcepti } } File file = new File(dir, LOCK_FILE_NAME); - RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw"); - FileChannel channel = randomAccessFile.getChannel(); + FileChannel channel = + FileChannel.open( + file.toPath(), + StandardOpenOption.CREATE, + StandardOpenOption.READ, + StandardOpenOption.WRITE); FileLock lock = null; try { // Try acquiring the lock without blocking. This method returns @@ -81,18 +87,26 @@ public void registerDirectory(File dir) throws ConfigurationException, IOExcepti } // File is already locked other virtual machine if (lock == null) { - randomAccessFile.close(); + String lockOwner = Files.readString(file.toPath()); + channel.close(); throw new ConfigurationException( String.format( StorageEngineMessages .STORAGE_EXCEPTION_CONFLICT_IS_DETECTED_IN_DIRECTORY_S_WHICH_MAY_BE_BEING_USED_CB5C77FC, dir.getAbsolutePath(), - randomAccessFile.readLine())); + lockOwner)); } - randomAccessFile.writeBytes(ProcessIdUtils.getProcessId()); + channel.truncate(0); + channel.position(0); + ByteBuffer processId = + ByteBuffer.wrap(ProcessIdUtils.getProcessId().getBytes(StandardCharsets.UTF_8)); + while (processId.hasRemaining()) { + channel.write(processId); + } + channel.force(true); // add to list fileList.add(file); - randomAccessFileList.add(randomAccessFile); + fileChannelList.add(channel); } public boolean isCrossDisk(String[] dirs) throws IOException { @@ -125,8 +139,8 @@ private Path mountOf(Path p) throws IOException { public void deregisterAll() { try { - for (RandomAccessFile randomAccessFile : randomAccessFileList) { - randomAccessFile.close(); + for (FileChannel fileChannel : fileChannelList) { + fileChannel.close(); // it will release lock automatically after close } for (File file : fileList) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java index 0e85f4d2072d5..4daa503ce1a0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java @@ -58,12 +58,13 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.io.RandomAccessFile; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.Base64; import java.util.Collections; import java.util.Iterator; @@ -1154,7 +1155,7 @@ private void deleteTopicProgressFiles(final String topicFileKey) { } private static void deleteFileIfExists(final File file) { - if (file.exists() && !file.delete()) { + if (file.exists() && !org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(file)) { LOGGER.warn( DataNodePipeMessages .PIPE_LOG_FAILED_TO_DELETE_CONSENSUS_SUBSCRIPTION_PROGRESS_FILE_51C57096, @@ -1220,11 +1221,15 @@ private void persistRegionProgress( private void overwriteRegionPayload( final File metaFile, final long payloadOffset, final byte[] payload) throws IOException { - try (final RandomAccessFile randomAccessFile = new RandomAccessFile(metaFile, "rw")) { - randomAccessFile.seek(payloadOffset); - randomAccessFile.write(payload); + try (final FileChannel channel = + FileChannel.open(metaFile.toPath(), StandardOpenOption.WRITE)) { + channel.position(payloadOffset); + final ByteBuffer payloadBuffer = ByteBuffer.wrap(payload); + while (payloadBuffer.hasRemaining()) { + channel.write(payloadBuffer); + } if (SubscriptionConfig.getInstance().isSubscriptionConsensusCommitFsyncEnabled()) { - randomAccessFile.getFD().sync(); + channel.force(true); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java index 67c61fcc1bc78..b85b136af61be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java @@ -29,9 +29,11 @@ import java.io.File; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.file.Files; +import java.nio.file.StandardOpenOption; public class ObjectWriter implements AutoCloseable { @@ -39,7 +41,7 @@ public class ObjectWriter implements AutoCloseable { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private final FileOutputStream fos; + private final FileChannel channel; private final File file; @@ -58,14 +60,15 @@ public ObjectWriter(File filePath) throws FileNotFoundException { } } file = filePath; - fos = new FileOutputStream(filePath, true); + channel = openChannel(filePath); } public void write(boolean isGeneratedByConsensus, long offset, byte[] content) throws IOException { if (file.length() != offset) { if (isGeneratedByConsensus || offset == 0) { - fos.getChannel().truncate(offset); + org.apache.iotdb.commons.utils.FileUtils.truncateFile(file, offset); + channel.position(offset); } else { throw new IOException( String.format( @@ -78,11 +81,27 @@ public void write(boolean isGeneratedByConsensus, long offset, byte[] content) if (file.length() + content.length > config.getMaxObjectSizeInByte()) { throw new IOException(DataNodeMiscMessages.FILE_LENGTH_LARGER_THAN_MAX); } - fos.write(content); + ByteBuffer buffer = ByteBuffer.wrap(content); + while (buffer.hasRemaining()) { + channel.write(buffer); + } } @Override public void close() throws Exception { - fos.close(); + channel.close(); + } + + private static FileChannel openChannel(File file) throws FileNotFoundException { + try { + FileChannel channel = + FileChannel.open(file.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE); + channel.position(channel.size()); + return channel; + } catch (IOException e) { + FileNotFoundException exception = new FileNotFoundException(e.getMessage()); + exception.initCause(e); + throw exception; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/writelog/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/writelog/LogWriter.java index 5421dd7665185..a68cf17b668e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/writelog/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/writelog/LogWriter.java @@ -25,11 +25,11 @@ import java.io.File; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; import java.util.zip.CRC32; /** @@ -40,7 +40,6 @@ public class LogWriter implements ILogWriter { private static final Logger logger = LoggerFactory.getLogger(LogWriter.class); private final File logFile; - private FileOutputStream fileOutputStream; private FileChannel channel; private final CRC32 checkSummer = new CRC32(); private final ByteBuffer lengthBuffer = ByteBuffer.allocate(4); @@ -51,15 +50,13 @@ public LogWriter(File logFile, boolean forceEachWrite) throws FileNotFoundExcept this.logFile = logFile; this.forceEachWrite = forceEachWrite; - fileOutputStream = new FileOutputStream(logFile, true); - channel = fileOutputStream.getChannel(); + channel = openChannel(logFile); } @Override public void write(ByteBuffer logBuffer) throws IOException { if (channel == null) { - fileOutputStream = new FileOutputStream(logFile, true); - channel = fileOutputStream.getChannel(); + channel = openChannel(logFile); } logBuffer.flip(); int logSize = logBuffer.limit(); @@ -104,8 +101,6 @@ public void close() throws IOException { if (channel.isOpen()) { channel.force(true); } - fileOutputStream.close(); - fileOutputStream = null; channel.close(); channel = null; } @@ -115,4 +110,18 @@ public void close() throws IOException { public String toString() { return "LogWriter{" + "logFile=" + logFile + '}'; } + + private static FileChannel openChannel(File file) throws FileNotFoundException { + try { + return FileChannel.open( + file.toPath(), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.APPEND); + } catch (IOException e) { + FileNotFoundException exception = new FileNotFoundException(e.getMessage()); + exception.initCause(e); + throw exception; + } + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/role/LocalFileRoleAccessor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/role/LocalFileRoleAccessor.java index cda9271bb044a..df7d7f284271c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/role/LocalFileRoleAccessor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/role/LocalFileRoleAccessor.java @@ -307,8 +307,7 @@ public boolean deleteEntity(String entityName) throws IOException { if (!roleProfile.exists() && !backFile.exists()) { return false; } - if ((roleProfile.exists() && !roleProfile.delete()) - || (backFile.exists() && !backFile.delete())) { + if (!FileUtils.deleteFileIfExist(roleProfile) || !FileUtils.deleteFileIfExist(backFile)) { throw new IOException(String.format(AuthMessages.CANNOT_DELETE_ROLE_FILE, entityName)); } return true; @@ -352,7 +351,7 @@ public boolean processTakeSnapshot(File snapshotDir) throws TException, IOExcept result = FileUtils.copyDir(roleFolder, roleTmpSnapshotDir); result &= roleTmpSnapshotDir.renameTo(roleSnapshotDir); } finally { - if (roleTmpSnapshotDir.exists() && !roleTmpSnapshotDir.delete()) { + if (roleTmpSnapshotDir.exists() && !FileUtils.deleteFileIfExist(roleTmpSnapshotDir)) { FileUtils.deleteFileOrDirectory(roleTmpSnapshotDir); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/LocalFileUserAccessor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/LocalFileUserAccessor.java index b29c064ba6283..5ad0d08fc4257 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/LocalFileUserAccessor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/LocalFileUserAccessor.java @@ -230,8 +230,7 @@ private boolean deleteURole(String username) throws IOException { if (!uRoleProfile.exists() && !backProfile.exists()) { return true; } - if ((uRoleProfile.exists() && !uRoleProfile.delete()) - || (backProfile.exists() && !backProfile.delete())) { + if (!FileUtils.deleteFileIfExist(uRoleProfile) || !FileUtils.deleteFileIfExist(backProfile)) { throw new IOException(String.format(AuthMessages.CATCH_ERROR_DELETE_USER_ROLE, username)); } return true; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java index a9f8ffc42296f..51d4671f36b6f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java @@ -30,7 +30,6 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -128,7 +127,8 @@ public boolean hasFileUnderTemporaryRoot(String fileName) { } private void removeFromTemporaryLibRoot(long requestId) { - FileUtils.deleteQuietly(getDirUnderTempRootByRequestId(requestId)); + org.apache.iotdb.commons.utils.FileUtils.deleteFileOrDirectory( + getDirUnderTempRootByRequestId(requestId), true); } public void saveTextAsFileUnderTemporaryRoot(String text, String fileName) throws IOException { @@ -228,12 +228,10 @@ protected void saveToDir(ByteBuffer byteBuffer, String destination) throws IOExc } Files.createFile(path); } - // FileOutPutStream is not in append mode by default, so the file will be - // overridden if it - // already exists. - try (FileOutputStream outputStream = new FileOutputStream(destination)) { - outputStream.getChannel().write(byteBuffer); - outputStream.getFD().sync(); + try (FileChannel channel = + FileChannel.open(path, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + channel.write(byteBuffer); + channel.force(true); } } catch (IOException e) { LOGGER.warn( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java index 5d465e7376e96..c82e41a49aa69 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java @@ -161,7 +161,7 @@ private void recover() throws IOException { return; } if (formalFile.exists() && tmpFile.exists()) { - if (!tmpFile.delete()) { + if (!FileUtils.deleteFileIfExist(tmpFile)) { LOGGER.warn( CommonMessages .LOG_DELETE_SYSTEM_PROPERTIES_TMP_FILE_FAIL_YOU_MAY_MANUALLY_DELETE_F81C4A53, @@ -182,7 +182,7 @@ private void replaceFormalFile() throws IOException { CommonMessages .EXCEPTION_TMP_SYSTEM_PROPERTIES_FILE_MUST_EXIST_CALL_REPLACEFORMALFILE_FA63B976); } - if (formalFile.exists() && !formalFile.delete()) { + if (formalFile.exists() && !FileUtils.deleteFileIfExist(formalFile)) { String msg = String.format( "Delete formal system properties file fail: %s", formalFile.getAbsoluteFile()); @@ -198,8 +198,8 @@ public void resetFilePath(String filePath) { } public void delete() { - this.formalFile.delete(); - this.tmpFile.delete(); + FileUtils.deleteFileIfExist(this.formalFile); + FileUtils.deleteFileIfExist(this.tmpFile); if (this.formalFile.exists() || this.tmpFile.exists()) { LOGGER.warn( CommonMessages diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/queue/serializer/PlainQueueSerializer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/queue/serializer/PlainQueueSerializer.java index 10087878d93ec..ddc41bdeec0ff 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/queue/serializer/PlainQueueSerializer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/queue/serializer/PlainQueueSerializer.java @@ -21,15 +21,14 @@ import org.apache.iotdb.commons.i18n.PipeMessages; import org.apache.iotdb.commons.pipe.datastructure.queue.ConcurrentIterableLinkedQueue; -import org.apache.iotdb.commons.utils.IOUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.EOFException; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.util.Objects; import java.util.function.Function; @@ -62,22 +61,21 @@ public void loadQueueFromFile( ConcurrentIterableLinkedQueue queue, Function elementDeserializationFunction) throws IOException { - try (FileChannel channel = inputStream.getChannel()) { - queue.setFirstIndex(ReadWriteIOUtils.readLong(inputStream)); - while (true) { - if (inputStream.available() == 0) { - return; - } - int capacity = ReadWriteIOUtils.readInt(inputStream); - ByteBuffer buffer = ByteBuffer.allocate(capacity); - IOUtils.readFully(channel, buffer); - buffer.flip(); - E element = elementDeserializationFunction.apply(buffer); - if (element == null) { - throw new IOException(PipeMessages.FAILED_TO_LOAD_SNAPSHOT); - } - queue.add(element); + queue.setFirstIndex(ReadWriteIOUtils.readLong(inputStream)); + while (true) { + if (inputStream.available() == 0) { + return; + } + int capacity = ReadWriteIOUtils.readInt(inputStream); + byte[] bytes = inputStream.readNBytes(capacity); + if (bytes.length != capacity) { + throw new EOFException(); + } + E element = elementDeserializationFunction.apply(ByteBuffer.wrap(bytes)); + if (element == null) { + throw new IOException(PipeMessages.FAILED_TO_LOAD_SNAPSHOT); } + queue.add(element); } } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java index af2e327ad42cb..806b5d0c5eeef 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java @@ -434,7 +434,9 @@ protected final TPipeTransferResp handleTransferFilePiece( // of the file. So the receiver should reset the offset of the writing file to the beginning // of the file. if (isRequestThroughAirGap && req.getStartWritingOffset() < writingFileWriter.length()) { - writingFileWriter.setLength(req.getStartWritingOffset()); + org.apache.iotdb.commons.utils.FileUtils.truncateFile( + writingFile, req.getStartWritingOffset()); + writingFileWriter.seek(req.getStartWritingOffset()); } if (!isWritingFileOffsetCorrect(req.getStartWritingOffset())) { @@ -442,7 +444,8 @@ protected final TPipeTransferResp handleTransferFilePiece( // If the file is a tsFile, then the content will not be changed for a specific filename. // However, for other files (mod, snapshot, etc.) the content varies for the same name in // different times, then we must rewrite the file to apply the newest version. - writingFileWriter.setLength(0); + org.apache.iotdb.commons.utils.FileUtils.truncateFile(writingFile, 0); + writingFileWriter.seek(0); } final TSStatus status = diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java index dd799b0f9a7fa..a0611a8cfc6bc 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java @@ -35,6 +35,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.channels.FileChannel; import java.nio.file.DirectoryNotEmptyException; import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileSystems; @@ -42,6 +43,7 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.text.CharacterIterator; import java.text.StringCharacterIterator; import java.util.ArrayList; @@ -92,6 +94,19 @@ public static boolean deleteFileIfExist(File file) { } } + /** + * Truncate a file through its NIO file system provider. + * + *

Using {@link FileChannel#open(Path, java.nio.file.OpenOption...)} allows a configured + * default {@link java.nio.file.spi.FileSystemProvider} to intercept the truncation. + */ + public static void truncateFile(File file, long size) throws IOException { + try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.WRITE)) { + channel.truncate(size); + channel.force(true); + } + } + public static void createLink(Path link, Path existing, boolean fallBackToCopy) throws IOException { try { @@ -197,7 +212,7 @@ private static void deleteDirectoryAndEmptyParent(File folder, LongConsumer dele File[] files = parentFolder.listFiles(); if (parentFolder.isDirectory() && (files == null || files.length == 0)) { acquireRemovePermit(parentFolder, deleteRateLimiter); - if (!parentFolder.delete()) { + if (!deleteFileIfExist(parentFolder)) { LOGGER.warn(UtilMessages.DELETE_FOLDER_FAILED, parentFolder.getAbsolutePath()); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/IOUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/IOUtils.java index 88c929e709260..4adebea8645ba 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/IOUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/IOUtils.java @@ -267,7 +267,7 @@ public static void writeObjectPrivilege( public static void replaceFile(File newFile, File oldFile) throws IOException { if (!newFile.renameTo(oldFile)) { // some OSs need to delete the old file before renaming to it - if (!oldFile.delete()) { + if (!FileUtils.deleteFileIfExist(oldFile)) { throw new IOException( String.format(UtilMessages.CANNOT_DELETE_OLD_USER_FILE, oldFile.getPath())); } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/FileUtilsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/FileUtilsTest.java index 402634b084bde..875a5a2bfcd4d 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/FileUtilsTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/FileUtilsTest.java @@ -74,6 +74,16 @@ public void testGetIllegalError4DirectoryRejectsEmptyPath() { Assert.assertNull(FileUtils.getIllegalError4Directory("valid_dir")); } + @Test + public void testTruncateFile() throws IOException { + File file = new File(tmpDir, "truncate-file"); + Files.write(file.toPath(), new byte[] {1, 2, 3, 4}); + + FileUtils.truncateFile(file, 2); + + Assert.assertArrayEquals(new byte[] {1, 2}, Files.readAllBytes(file.toPath())); + } + @Test public void testDeleteFileOrDirectoryWithRateLimiter() throws IOException { File deleteDir = new File(tmpDir, "deleteWithRateLimiter"); diff --git a/pom.xml b/pom.xml index dbca5078c3326..0c7c07c63d893 100644 --- a/pom.xml +++ b/pom.xml @@ -77,6 +77,7 @@ true 5.1.9 3.0.2 + 3.10 1.16 1.28.0 2.13.1 @@ -1024,6 +1025,38 @@ ${spotless.skip} + + + de.thetaphi + forbiddenapis + ${forbiddenapis.version} + + true + + false + + ${maven.multiModuleProjectDirectory}/src/main/forbidden-apis/secure-erase + + + + org/apache/iotdb/confignode/service/ConfigNode.class + org/apache/iotdb/db/service/DataNode.class + + + + + check-secure-erase-file-apis + process-classes + + check + + + + org.eluder.coveralls coveralls-maven-plugin diff --git a/src/main/forbidden-apis/secure-erase b/src/main/forbidden-apis/secure-erase new file mode 100644 index 0000000000000..6c2030d16fe4f --- /dev/null +++ b/src/main/forbidden-apis/secure-erase @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@defaultMessage Use Files or FileChannel APIs so SecureFileSystemProvider can intercept the operation. +java.io.File#delete() +java.io.File#deleteOnExit() +java.io.FileInputStream#getChannel() +java.io.FileOutputStream#getChannel() +java.io.RandomAccessFile#getChannel() +java.io.RandomAccessFile#setLength(long)