Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions .github/workflows/nio2-file-api-check.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
name: Check NIO.2-compatible File APIs in Changed Java Files

on:
pull_request:
branches:
- master
- 'dev/*'
- 'rel/*'
- "rc/*"
- 'force_ci/**'
paths-ignore:
- 'docs/**'
- 'site/**'
# allow manually run the action:
workflow_dispatch:

jobs:
nio2-file-api-check:
runs-on: ubuntu-latest

steps:
- name: Checkout code
uses: actions/checkout@v5

- name: Check changed Java lines for non-NIO.2 file APIs
shell: bash
run: |
set -euo pipefail

# Fetch the target branch. Fall back to master when manually triggered.
BASE_REF="${GITHUB_BASE_REF:-master}"
git fetch origin "$BASE_REF"

git switch -c check_branch

# Get Java diffs only. This check intentionally only scans added lines.
echo "Get Java diff of the changes"
DIFF=$(git diff "origin/$BASE_REF" check_branch -- '*.java')

if [ -z "$DIFF" ]; then
echo "No Java changes detected."
exit 0
fi

# These APIs bypass java.nio.file.spi.FileSystemProvider interception. Use NIO.2
# alternatives such as Files.delete/deleteIfExists and FileChannel.open(Path, ...)
# or the project helper FileUtils.truncateFile instead.
echo "Check the diff for non-NIO.2-compatible file APIs"
NON_NIO2_PATTERN='^\+.*(\.delete\(\)|\.deleteOnExit\(\)|getChannel\(\)\.truncate\(|\.setLength\(|new RandomAccessFile\()'
VIOLATIONS=$(echo "$DIFF" | grep -E "$NON_NIO2_PATTERN" || true)

if [ -z "$VIOLATIONS" ]; then
echo "No non-NIO.2-compatible file APIs found in changed content."
exit 0
fi

echo "Non-NIO.2-compatible file API usage found in the changes."
echo "Please use provider-compatible APIs before merging:"
echo "- Files.delete(...) or Files.deleteIfExists(...) instead of File.delete()/deleteOnExit()"
echo "- FileChannel.open(Path, ...) or FileUtils.truncateFile(...) instead of getChannel().truncate()/setLength()"
echo "$VIOLATIONS" | tee -a output.log
exit 1
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ private static Optional<Procedure> loadProcedure(Path procedureFilePath) {
}
} catch (Exception e) {
LOGGER.error(ConfigNodeMessages.LOAD_FAILED_IT_WILL_BE_DELETED, procedureFilePath, e);
if (!procedureFilePath.toFile().delete()) {
if (!FileUtils.deleteFileIfExist(procedureFilePath.toFile())) {
LOGGER.error(
ConfigNodeMessages.DELETED_FAILED_TAKE_APPROPRIATE_ACTION, procedureFilePath, e);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -676,7 +676,8 @@ public boolean processTakeSnapshot(File snapshotDir) throws IOException, TExcept
dataNodeInfoReadWriteLock.readLock().unlock();
configNodeInfoReadWriteLock.readLock().unlock();
for (int retry = 0; retry < 5; retry++) {
if (!tmpFile.exists() || tmpFile.delete()) {
if (!tmpFile.exists()
|| org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tmpFile)) {
break;
} else {
LOGGER.warn(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1042,7 +1042,8 @@ public boolean processTakeSnapshot(File snapshotDir) throws TException, IOExcept
} finally {
// with or without success, delete temporary files anyway
for (int retry = 0; retry < 5; retry++) {
if (!tmpFile.exists() || tmpFile.delete()) {
if (!tmpFile.exists()
|| org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tmpFile)) {
break;
} else {
LOGGER.warn(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -781,7 +781,8 @@ public boolean processDatabaseSchemaSnapshot(
return tmpFile.renameTo(snapshotFile);
} finally {
for (int retry = 0; retry < 5; retry++) {
if (!tmpFile.exists() || tmpFile.delete()) {
if (!tmpFile.exists()
|| org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tmpFile)) {
break;
} else {
LOGGER.warn(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,8 @@ public boolean processTakeSnapshot(File snapshotDir) throws IOException {
return tmpFile.renameTo(snapshotFile);
} finally {
for (int retry = 0; retry < 5; retry++) {
if (!tmpFile.exists() || tmpFile.delete()) {
if (!tmpFile.exists()
|| org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tmpFile)) {
break;
} else {
LOGGER.warn(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,8 @@ public boolean processTakeSnapshot(File snapshotDir) throws IOException {
return tmpFile.renameTo(snapshotFile);
} finally {
for (int retry = 0; retry < 5; retry++) {
if (!tmpFile.exists() || tmpFile.delete()) {
if (!tmpFile.exists()
|| org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tmpFile)) {
break;
} else {
LOGGER.warn(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.iotdb.confignode.writelog.io;

import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
import org.apache.iotdb.confignode.i18n.ConfigNodeMessages;

Expand All @@ -29,10 +30,8 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.NoSuchElementException;
import java.util.zip.CRC32;

Expand Down Expand Up @@ -147,9 +146,8 @@ public boolean isFileCorrupted() {
}

private void truncateBrokenLogs() {
try (FileOutputStream outputStream = new FileOutputStream(filepath, true);
FileChannel channel = outputStream.getChannel()) {
channel.truncate(unbrokenLogsSize);
try {
FileUtils.truncateFile(new File(filepath), unbrokenLogsSize);
} catch (IOException e) {
logger.error(ConfigNodeMessages.FAIL_TO_TRUNCATE_LOG_FILE_TO_SIZE, unbrokenLogsSize, e);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@

import java.io.File;
import java.io.IOException;
import java.nio.file.DirectoryNotEmptyException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.Collection;
Expand Down Expand Up @@ -276,8 +277,9 @@ public long takeSnapshot() throws IOException {
private void deleteIncompleteSnapshot(File snapshotDir) throws IOException {
// this takeSnapshot failed, clean up files and directories
// statemachine is supposed to clear snapshotDir on failure
boolean isEmpty = snapshotDir.delete();
if (!isEmpty) {
try {
Files.deleteIfExists(snapshotDir.toPath());
} catch (DirectoryNotEmptyException e) {
logger.info(RatisMessages.SNAPSHOT_DIR_INCOMPLETE_DELETING, snapshotDir.getAbsolutePath());
FileUtils.deleteFully(snapshotDir);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.DataNodePipeMessages;

import org.apache.tsfile.external.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -114,10 +113,11 @@ private static void moveAsideAndCollect(
} catch (final IOException e) {
LOGGER.warn(
DataNodePipeMessages.PIPE_HARDLINK_DIR_MOVE_FAILED_DELETING_SYNC, pipeHardLinkDir, e);
org.apache.iotdb.commons.utils.FileUtils.deleteFileOrDirectory(pipeHardLinkDir, true);
LOGGER.info(
DataNodePipeMessages.PIPE_HARDLINK_DIR_FOUND_DELETING_IT_RESULT,
pipeHardLinkDir,
FileUtils.deleteQuietly(pipeHardLinkDir));
!pipeHardLinkDir.exists());
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,8 @@ List<Pair<String, File>> writeTableModelTabletsToTsFiles(
}

for (final Pair<String, File> sealedFile : sealedFiles) {
final boolean deleteSuccess = FileUtils.deleteQuietly(sealedFile.right);
final boolean deleteSuccess =
org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(sealedFile.right);
LOGGER.warn(
DataNodePipeMessages.BATCH_ID_DELETE_THE_TSFILE_AFTER_FAILED,
currentBatchId.get(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,8 @@ private List<Pair<String, File>> writeTabletsToTsFiles()
}

for (final Pair<String, File> sealedFile : sealedFiles) {
final boolean deleteSuccess = FileUtils.deleteQuietly(sealedFile.right);
final boolean deleteSuccess =
org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(sealedFile.right);
LOGGER.warn(
DataNodePipeMessages.BATCH_ID_DELETE_THE_TSFILE_AFTER_FAILED,
currentBatchId.get(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ private File getNextBaseDir() throws DiskSpaceInsufficientException {
.getNextWithRetry(
folder -> {
File dir = new File(folder, Long.toString(currentBatchId.get()));
FileUtils.deleteQuietly(dir);
org.apache.iotdb.commons.utils.FileUtils.deleteFileOrDirectory(dir, true);
if (dir.mkdirs()) {
LOGGER.info(
DataNodePipeMessages.BATCH_ID_CREATE_BATCH_DIR_SUCCESSFULLY_BATCH,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.encrypt.EncryptParameter;
import org.apache.tsfile.encrypt.EncryptUtils;
import org.apache.tsfile.external.commons.io.FileUtils;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.file.metadata.TimeseriesMetadata;
Expand Down Expand Up @@ -470,7 +469,7 @@ && handleSingleMiniFile(i)) {
DataNodeQueryMessages.EMPTY_FILE_DETECTED_WILL_SKIP_LOADING_THIS_FILE,
tsFile.getAbsolutePath());
if (isDeleteAfterLoad) {
FileUtils.deleteQuietly(tsFile);
org.apache.iotdb.commons.utils.FileUtils.deleteFileIfExist(tsFile);
}
} finally {
// reset the session info to the original one
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.iotdb.db.schemaengine.schemaregion.logfile;

import org.apache.iotdb.commons.file.SystemFileFactory;
import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.i18n.DataNodeSchemaMessages;

import org.apache.tsfile.utils.ReadWriteIOUtils;
Expand All @@ -32,10 +33,8 @@
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.channels.FileChannel;
import java.util.NoSuchElementException;

/**
Expand Down Expand Up @@ -143,16 +142,15 @@ public boolean isFileCorrupted() {
}

private void truncateBrokenLogs() {
try (FileOutputStream outputStream = new FileOutputStream(logFile, true);
FileChannel channel = outputStream.getChannel()) {
if (currentIndex != channel.size()) {
try {
final long fileSize = logFile.length();
if (currentIndex != fileSize) {
LOGGER.warn(
DataNodeSchemaMessages.LOG_FILE_END_CORRUPTED_TRUNCATE,
logFile.getName(),
currentIndex,
channel.size());
channel.truncate(currentIndex);
channel.force(true);
fileSize);
FileUtils.truncateFile(logFile, currentIndex);
}
isFileCorrupted = false;
} catch (IOException e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.iotdb.commons.schema.SchemaConstant;
import org.apache.iotdb.commons.schema.node.role.IDatabaseMNode;
import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory;
import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.commons.utils.IOUtils;
import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.commons.utils.TestOnly;
Expand All @@ -45,11 +46,11 @@
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Iterator;

/**
Expand Down Expand Up @@ -123,7 +124,7 @@ private SchemaFile(
pmtFile.createNewFile();
}

this.channel = new RandomAccessFile(pmtFile, "rw").getChannel();
this.channel = openReadWriteChannel(pmtFile);
this.headerContent = ByteBuffer.allocate(SchemaFileConfig.FILE_HEADER_SIZE);
// will be overwritten if to init
this.dataTTL = ttl;
Expand All @@ -138,7 +139,7 @@ private SchemaFile(File file) throws IOException, MetadataException {
pmtFile = file;
filePath = pmtFile.getPath();
logPath = file.getParent() + File.separator + SchemaConstant.PBTREE_LOG_FILE_NAME;
channel = new RandomAccessFile(file, "rw").getChannel();
channel = openReadWriteChannel(file);
headerContent = ByteBuffer.allocate(SchemaFileConfig.FILE_HEADER_SIZE);

if (channel.size() <= 0) {
Expand Down Expand Up @@ -293,11 +294,19 @@ public void clear() throws IOException, MetadataException {
}
pmtFile.createNewFile();

channel = new RandomAccessFile(pmtFile, "rw").getChannel();
channel = openReadWriteChannel(pmtFile);
headerContent = ByteBuffer.allocate(SchemaFileConfig.FILE_HEADER_SIZE);
initFileHeader();
}

private static FileChannel openReadWriteChannel(File file) throws IOException {
return FileChannel.open(
file.toPath(),
StandardOpenOption.READ,
StandardOpenOption.WRITE,
StandardOpenOption.CREATE);
}

public String inspect() throws MetadataException, IOException {
return inspect(null);
}
Expand Down Expand Up @@ -464,7 +473,7 @@ public boolean createSnapshot(File snapshotDir) {
SystemFileFactory.INSTANCE.getFile(snapshotDir, SchemaConstant.PBTREE_SNAPSHOT);
try {
sync();
if (schemaFileSnapshot.exists() && !schemaFileSnapshot.delete()) {
if (schemaFileSnapshot.exists() && !FileUtils.deleteFileIfExist(schemaFileSnapshot)) {
logger.error(
DataNodeSchemaMessages.FAILED_TO_DELETE_OLD_PBTREE_SNAPSHOT,
schemaFileSnapshot.getName());
Expand All @@ -474,7 +483,7 @@ public boolean createSnapshot(File snapshotDir) {
return true;
} catch (IOException e) {
logger.error(DataNodeSchemaMessages.FAILED_TO_CREATE_SCHEMA_FILE_SNAPSHOT, e.getMessage(), e);
schemaFileSnapshot.delete();
FileUtils.deleteFileIfExist(schemaFileSnapshot);
return false;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,15 +137,15 @@ public static TagManager loadFromSnapshot(
File tagSnapshot =
SystemFileFactory.INSTANCE.getFile(snapshotDir, SchemaConstant.TAG_LOG_SNAPSHOT);
File tagFile = SystemFileFactory.INSTANCE.getFile(sgSchemaDirPath, SchemaConstant.TAG_LOG);
if (tagFile.exists() && !tagFile.delete()) {
if (tagFile.exists() && !FileUtils.deleteFileIfExist(tagFile)) {
logger.warn(DataNodeSchemaMessages.FAILED_TO_DELETE_EXISTING_WHEN_LOADING, tagFile.getName());
}

try {
org.apache.tsfile.external.commons.io.FileUtils.copyFile(tagSnapshot, tagFile);
return new TagManager(sgSchemaDirPath, regionStatistics);
} catch (IOException e) {
if (!tagFile.delete()) {
if (!FileUtils.deleteFileIfExist(tagFile)) {
logger.warn(
DataNodeSchemaMessages.FAILED_TO_DELETE_EXISTING_WHEN_COPY_FAILURE, tagFile.getName());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3233,7 +3233,7 @@ public void deleteByTable(RelationalDeleteDataNode node) throws IOException {
} catch (IOException e) {
logger.error(StorageEngineMessages.FAILED_TO_CHECK_OBJECT_FILES, e.getMessage());
}
FileUtils.deleteQuietly(objectTableDir);
org.apache.iotdb.commons.utils.FileUtils.deleteFileOrDirectory(objectTableDir, true);
}
FileMetrics.getInstance().decreaseObjectFileNum(count.get());
FileMetrics.getInstance().decreaseObjectFileSize(totalSize.get());
Expand Down
Loading
Loading