Skip to content

feat(load): reject managed TsFile directories #18271

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
luoluoyuyu wants to merge 9 commits into
base: master
Choose a base branch
from
Open

feat(load): reject managed TsFile directories #18271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
e777e04
feat(load): reject managed TsFile directories
luoluoyuyu Jul 21, 2026
fdccefc
perf(load): cache internal TsFile directories
luoluoyuyu Jul 27, 2026
6bed0eb
fix(load): reject all local data directories
luoluoyuyu Jul 28, 2026
1c38947
test(load): cover pipe receiver outside data directory
luoluoyuyu Jul 29, 2026
828791b
fix(load): reject IoTDB data directory
luoluoyuyu Jul 29, 2026
962a829
fix(pipe): preserve load source during type conversion
luoluoyuyu Jul 29, 2026
9c32e1b
fix(pipe): preserve load source during scheduler retry
luoluoyuyu Jul 29, 2026
fcba830
fix(pipe): preserve load source during active load
luoluoyuyu Jul 29, 2026
6bb600b
fix(pipe): keep active load source validation
luoluoyuyu Jul 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -887,6 +887,28 @@ public void testLoadWithRelativePathName() throws Exception {
}
}

@Test
public void testLoadDataNodeInternalDataDirectoryIsRejectedWithoutLeakingPath() throws Exception {
final DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0);
final File dataDir = new File(dataNodeWrapper.getDataPath());

try (final Connection connection =
EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(dataNodeWrapper);
final Statement statement = connection.createStatement()) {
try {
statement.execute(String.format("load \"%s\"", dataDir.getAbsolutePath()));
Assert.fail("Expected LOAD from the DataNode internal data directory to be rejected.");
} catch (final SQLException e) {
Assert.assertTrue(
e.getMessage(),
e.getMessage()
.contains(
"Cannot load files because the specified directory contains IoTDB data."));
Assert.assertFalse(e.getMessage(), e.getMessage().contains(dataDir.getAbsolutePath()));
}
}
}

@Test
public void testLoadWithMods() throws Exception {
final long writtenPoint1;
Expand Down
3 changes: 3 additions & 0 deletions iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3013,6 +3013,9 @@ private DataNodeQueryMessages() {}
"Can not find %s on this machine, notice that load can only handle files on this machine.";
public static final String QUERY_EXCEPTION_LOAD_TSFILE_SOURCE_PATH_S_IS_OUTSIDE_ALLOWED_DIRECTORIES_85A6019F =
"Load TsFile source path %s is outside allowed directories %s.";
public static final String
QUERY_EXCEPTION_CANNOT_LOAD_FILES_BECAUSE_SPECIFIED_DIRECTORY_CONTAINS_IOTDB_DATA_B0A1B93D =
"Cannot load files because the specified directory contains IoTDB data.";
public static final String QUERY_EXCEPTION_FAILED_TO_RESOLVE_CANONICAL_PATH_FOR_LOAD_TSFILE_SOURCE_09CC9AC6 =
"Failed to resolve canonical path for Load TsFile source %s: %s";
public static final String QUERY_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_AE9DBDC0 =
Expand Down
3 changes: 3 additions & 0 deletions iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3633,6 +3633,9 @@ private DataNodeQueryMessages() {}
public static final String QUERY_EXCEPTION_LOAD_TSFILE_SOURCE_PATH_S_IS_OUTSIDE_ALLOWED_DIRECTORIES_85A6019F =

"加载 TsFile 的源路径 %s 位于允许目录 %s 之外。";
public static final String
QUERY_EXCEPTION_CANNOT_LOAD_FILES_BECAUSE_SPECIFIED_DIRECTORY_CONTAINS_IOTDB_DATA_B0A1B93D =
"指定目录包含 IoTDB 数据，无法加载文件。";
public static final String QUERY_EXCEPTION_FAILED_TO_RESOLVE_CANONICAL_PATH_FOR_LOAD_TSFILE_SOURCE_09CC9AC6 =
"无法解析 load TsFile source %s 的 canonical path：%s";
public static final String QUERY_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_AE9DBDC0 =
Expand Down
22 changes: 22 additions & 0 deletions iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,8 @@ public class IoTDBConfig {

private CanonicalPaths loadTsFileAllowedDirCanonicalPaths = canonicalPaths(loadTsFileAllowedDirs);

private volatile CanonicalPaths internalDataDirCanonicalPaths = new CanonicalPaths(new Path[0]);

private boolean loadTsFileSourcePathCheckEnabled = false;

/** Strategy of multiple directories. */
Expand Down Expand Up @@ -1429,6 +1431,7 @@ private void formulateFolders() {
queryDir = addDataHomeDir(queryDir);
sortTmpDir = addDataHomeDir(sortTmpDir);
formulateDataDirs(tierDataDirs);
formulateInternalDataDirs(tierDataDirs);
}

private void formulateDataDirs(String[][] tierDataDirs) {
Expand Down Expand Up @@ -1480,6 +1483,7 @@ void reloadDataDirs(String[][] newTierDataDirs) throws LoadConfigurationExceptio
}
}
this.tierDataDirs = newTierDataDirs;
formulateInternalDataDirs(newTierDataDirs);
reloadSystemMetrics();
}

Expand Down Expand Up @@ -1556,6 +1560,10 @@ public String[] getLocalDataDirs() {
.toArray(String[]::new);
}

public Path[] getInternalDataDirCanonicalPaths() throws FileNotFoundException {
return internalDataDirCanonicalPaths.getPaths();
}

public String[][] getTierDataDirs() {
return tierDataDirs;
}
Expand All @@ -1564,6 +1572,7 @@ public String[][] getTierDataDirs() {
public void setTierDataDirs(String[][] tierDataDirs) {
formulateDataDirs(tierDataDirs);
this.tierDataDirs = tierDataDirs;
formulateInternalDataDirs(tierDataDirs);
// TODO(szywilliam): rewrite the logic here when ratis supports complete snapshot semantic
setRatisDataRegionSnapshotDir(
tierDataDirs[0][0] + File.separator + IoTDBConstant.SNAPSHOT_FOLDER_NAME);
Expand Down Expand Up @@ -1663,6 +1672,19 @@ public void formulateLoadTsFileDirs(String[][] tierDataDirs) {
this.loadTsFileDirCanonicalPaths = canonicalPaths(newLoadTsFileDirs);
}

private void formulateInternalDataDirs(final String[][] tierDataDirs) {
final List<String> internalDataDirs = new ArrayList<>();
internalDataDirs.add(addDataHomeDir("data"));
for (final String[] tierDataDir : tierDataDirs) {
for (final String dataDir : tierDataDir) {
if (FSUtils.isLocal(dataDir)) {
internalDataDirs.add(dataDir);
}
}
}
internalDataDirCanonicalPaths = canonicalPaths(internalDataDirs.toArray(new String[0]));
}

private static CanonicalPaths canonicalPaths(final String[] dirs) {
final Path[] paths = new Path[dirs.length];
for (int i = 0; i < dirs.length; i++) {
Expand Down
2 changes: 1 addition & 1 deletion .../src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public TsFileLoader(File tsFile, String database) {
@Override
public void load(final SessionInfo sessionInfo) {
try {
LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath());
LoadTsFileStatement statement = LoadTsFileStatement.createForPipe(tsFile.getAbsolutePath());
statement.setDeleteAfterLoad(true);
statement.setConvertOnTypeMismatch(true);
statement.setDatabaseLevel(parseSgLevel());
Expand Down
2 changes: 1 addition & 1 deletion ...rc/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -676,7 +676,7 @@ static LoadTsFileStatement buildLoadTsFileStatementForSync(
final boolean validateTsFile,
final boolean shouldConvertDataTypeOnTypeMismatch)
throws FileNotFoundException {
final LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(fileAbsolutePath);
final LoadTsFileStatement statement = LoadTsFileStatement.createForPipe(fileAbsolutePath);
statement.setDeleteAfterLoad(true);
statement.setConvertOnTypeMismatch(shouldConvertDataTypeOnTypeMismatch);
statement.setVerifySchema(validateTsFile || shouldConvertDataTypeOnTypeMismatch);
Expand Down
22 changes: 16 additions & 6 deletions ...e/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -488,15 +488,20 @@ private boolean handleSingleMiniFile(final int i) throws FileNotFoundException {
isTableModelTsFile.get(i)
? loadTsFileDataTypeConverter
.convertForTableModel(
LoadTsFile.createUnchecked(
null, tsFiles.get(i).getPath(), Collections.emptyMap())
(isGeneratedByPipe
? LoadTsFile.createForPipe(
null, tsFiles.get(i).getPath(), Collections.emptyMap())
: LoadTsFile.createUnchecked(
null, tsFiles.get(i).getPath(), Collections.emptyMap()))
.setDatabase(databaseForTableData)
.setDeleteAfterLoad(isDeleteAfterLoad)
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
.orElse(null)
: loadTsFileDataTypeConverter
.convertForTreeModel(
LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath())
(isGeneratedByPipe
? LoadTsFileStatement.createForPipe(tsFiles.get(i).getPath())
: LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath()))
.setDeleteAfterLoad(isDeleteAfterLoad)
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
.orElse(null);
Expand Down Expand Up @@ -781,15 +786,20 @@ private void executeTabletConversionOnException(
isTableModelTsFile.get(i)
? loadTsFileDataTypeConverter
.convertForTableModel(
LoadTsFile.createUnchecked(
null, tsFiles.get(i).getPath(), Collections.emptyMap())
(isGeneratedByPipe
? LoadTsFile.createForPipe(
null, tsFiles.get(i).getPath(), Collections.emptyMap())
: LoadTsFile.createUnchecked(
null, tsFiles.get(i).getPath(), Collections.emptyMap()))
.setDatabase(databaseForTableData)
.setDeleteAfterLoad(isDeleteAfterLoad)
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
.orElse(null)
: loadTsFileDataTypeConverter
.convertForTreeModel(
LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath())
(isGeneratedByPipe
? LoadTsFileStatement.createForPipe(tsFiles.get(i).getPath())
: LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath()))
.setDeleteAfterLoad(isDeleteAfterLoad)
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
.orElse(null);
Expand Down
23 changes: 17 additions & 6 deletions ...ode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,19 +73,25 @@ public class LoadTsFile extends Statement {
private boolean needDecode4TimeColumn;

public LoadTsFile(NodeLocation location, String filePath, Map<String, String> loadAttributes) {
this(location, filePath, loadAttributes, true);
this(location, filePath, loadAttributes, true, true);
}

public static LoadTsFile createUnchecked(
NodeLocation location, String filePath, Map<String, String> loadAttributes) {
return new LoadTsFile(location, filePath, loadAttributes, false);
return new LoadTsFile(location, filePath, loadAttributes, false, true);
}

public static LoadTsFile createForPipe(
NodeLocation location, String filePath, Map<String, String> loadAttributes) {
return new LoadTsFile(location, filePath, loadAttributes, false, false);
}

private LoadTsFile(
NodeLocation location,
String filePath,
Map<String, String> loadAttributes,
boolean validateSourcePath) {
boolean validateSourcePath,
boolean validateInternalDataDir) {
super(location);
this.filePath =
requireNonNull(filePath, DataNodeQueryMessages.EXCEPTION_FILEPATH_IS_NULL_84CE8A66);
Expand All @@ -103,8 +109,11 @@ private LoadTsFile(

try {
this.tsFiles =
org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement.processTsFile(
new File(filePath), validateSourcePath);
validateInternalDataDir
? org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement
.processTsFile(new File(filePath), validateSourcePath)
: org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement
Comment on lines +113 to +115

@jt2594838 jt2594838 Jul 30, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Mind this

@luoluoyuyu luoluoyuyu Jul 30, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the reminder. processTsFileForPipe intentionally bypasses the managed-directory check because Pipe receiver/hardlink sources can be placed under IoTDB-managed paths. We have restricted this branch to createForPipe() call sites guarded by an existing isGeneratedByPipe context; normal table-model LOAD still uses processTsFile(...) and keeps the managed-directory validation.

.processTsFileForPipe(new File(filePath));
this.resources = new ArrayList<>();
this.writePointCountList = new ArrayList<>();
this.isTableModel = new ArrayList<>(Collections.nCopies(this.tsFiles.size(), true));
Expand Down Expand Up @@ -298,7 +307,9 @@ public List<LoadTsFile> getSubStatements() {
final Map<String, String> properties = this.loadAttributes;

final LoadTsFile subStatement =
LoadTsFile.createUnchecked(getLocation().orElse(null), filePath, properties);
isGeneratedByPipe
? LoadTsFile.createForPipe(getLocation().orElse(null), filePath, properties)
: LoadTsFile.createUnchecked(getLocation().orElse(null), filePath, properties);

// Copy all configuration properties
subStatement.databaseLevel = this.databaseLevel;
Expand Down
9 changes: 7 additions & 2 deletions ...rc/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -625,7 +625,10 @@ private void convertFailedTsFilesToTabletsAndRetry() {
failedNode.isTableModel()
? loadTsFileDataTypeConverter
.convertForTableModel(
LoadTsFile.createUnchecked(null, filePath, Collections.emptyMap())
(isGeneratedByPipe
? LoadTsFile.createForPipe(null, filePath, Collections.emptyMap())
: LoadTsFile.createUnchecked(
null, filePath, Collections.emptyMap()))
.setDatabase(failedNode.getDatabase())
.setDeleteAfterLoad(failedNode.isDeleteAfterLoad())
.setConvertOnTypeMismatch(true))
Expand Down Expand Up @@ -684,7 +687,9 @@ private LoadTsFileStatement buildRetryTreeLoadStatement(
final String filePath, final boolean deleteAfterLoad, final String database)
throws FileNotFoundException {
final LoadTsFileStatement statement =
LoadTsFileStatement.createUnchecked(filePath)
(isGeneratedByPipe
? LoadTsFileStatement.createForPipe(filePath)
: LoadTsFileStatement.createUnchecked(filePath))
.setDeleteAfterLoad(deleteAfterLoad)
.setConvertOnTypeMismatch(true);
if (database != null) {
Expand Down
67 changes: 58 additions & 9 deletions ...rc/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,19 @@ public class LoadTsFileStatement extends Statement {
private boolean needDecode4TimeColumn;

public LoadTsFileStatement(String filePath) throws FileNotFoundException {
this(filePath, true);
this(filePath, true, true);
}

public static LoadTsFileStatement createUnchecked(String filePath) throws FileNotFoundException {
return new LoadTsFileStatement(filePath, false);
return new LoadTsFileStatement(filePath, false, true);
}

private LoadTsFileStatement(String filePath, boolean validateSourcePath)
public static LoadTsFileStatement createForPipe(String filePath) throws FileNotFoundException {
return new LoadTsFileStatement(filePath, false, false);
}

private LoadTsFileStatement(
String filePath, boolean validateSourcePath, boolean validateInternalDataDir)
throws FileNotFoundException {
this.file = new File(filePath).getAbsoluteFile();
this.databaseLevel = IoTDBDescriptor.getInstance().getConfig().getDefaultDatabaseLevel();
Expand All @@ -96,19 +101,34 @@ private LoadTsFileStatement(String filePath, boolean validateSourcePath)
IoTDBDescriptor.getInstance().getConfig().getLoadTabletConversionThresholdBytes();
this.autoCreateDatabase = IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled();

this.tsFiles = processTsFile(file, validateSourcePath);
this.tsFiles = processTsFile(file, validateSourcePath, validateInternalDataDir);
this.resources = new ArrayList<>();
this.writePointCountList = new ArrayList<>();
this.isTableModel = new ArrayList<>(Collections.nCopies(this.tsFiles.size(), false));
this.statementType = StatementType.MULTI_BATCH_INSERT;
}

public static List<File> processTsFile(final File file) throws FileNotFoundException {
return processTsFile(file, true);
return processTsFile(file, true, true);
}

public static List<File> processTsFile(final File file, final boolean validateSourcePath)
throws FileNotFoundException {
return processTsFile(file, validateSourcePath, true);
}

public static List<File> processTsFileForPipe(final File file) throws FileNotFoundException {
return processTsFile(file, false, false);
}

private static List<File> processTsFile(
final File file, final boolean validateSourcePath, final boolean validateInternalDataDir)
throws FileNotFoundException {
final Path[] internalDataDirCanonicalPaths =
IoTDBDescriptor.getInstance().getConfig().getInternalDataDirCanonicalPaths();
if (validateInternalDataDir) {
validateNotLoadingInternalTsFile(file, internalDataDirCanonicalPaths);
}
if (validateSourcePath) {
validateLoadSourcePath(file);
}
Expand All @@ -124,7 +144,9 @@ public static List<File> processTsFile(final File file, final boolean validateSo
.QUERY_EXCEPTION_CAN_NOT_FIND_S_ON_THIS_MACHINE_NOTICE_THAT_LOAD_CAN_ONLY_B7886C0E,
file.getPath()));
}
tsFiles.addAll(findAllTsFile(file, validateSourcePath));
tsFiles.addAll(
findAllTsFile(
file, validateSourcePath, validateInternalDataDir, internalDataDirCanonicalPaths));
}
sortTsFiles(tsFiles);
return tsFiles;
Expand All @@ -146,7 +168,11 @@ protected LoadTsFileStatement() {
this.statementType = StatementType.MULTI_BATCH_INSERT;
}

private static List<File> findAllTsFile(File file, boolean validateSourcePath)
private static List<File> findAllTsFile(
File file,
boolean validateSourcePath,
boolean validateInternalDataDir,
Path[] internalDataDirCanonicalPaths)
throws FileNotFoundException {
final File[] files = file.listFiles();
if (files == null) {
Expand All @@ -155,13 +181,21 @@ private static List<File> findAllTsFile(File file, boolean validateSourcePath)

final List<File> tsFiles = new ArrayList<>();
for (File nowFile : files) {
if (validateInternalDataDir) {
validateNotLoadingInternalTsFile(nowFile, internalDataDirCanonicalPaths);
}
if (validateSourcePath) {
validateLoadSourcePath(nowFile);
}
if (nowFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) {
tsFiles.add(nowFile);
} else if (nowFile.isDirectory()) {
tsFiles.addAll(findAllTsFile(nowFile, validateSourcePath));
tsFiles.addAll(
findAllTsFile(
nowFile,
validateSourcePath,
validateInternalDataDir,
internalDataDirCanonicalPaths));
}
}
return tsFiles;
Expand Down Expand Up @@ -195,6 +229,19 @@ private static void validateLoadSourcePath(final File file) throws FileNotFoundE
Arrays.toString(allowedDirs)));
}

private static void validateNotLoadingInternalTsFile(
final File file, final Path[] internalDataDirCanonicalPaths) throws FileNotFoundException {
final Path sourcePath = canonicalPath(file);
for (final Path internalDataDirCanonicalPath : internalDataDirCanonicalPaths) {
if (sourcePath.startsWith(internalDataDirCanonicalPath)
|| internalDataDirCanonicalPath.startsWith(sourcePath)) {
throw new FileNotFoundException(
DataNodeQueryMessages
.QUERY_EXCEPTION_CANNOT_LOAD_FILES_BECAUSE_SPECIFIED_DIRECTORY_CONTAINS_IOTDB_DATA_B0A1B93D);
}
}
}

private static Path canonicalPath(final File file) throws FileNotFoundException {
try {
return file.getCanonicalFile().toPath();
Expand Down Expand Up @@ -482,7 +529,9 @@ public List<PartialPath> getPaths() {
loadAttributes.put(PIPE_GENERATED_KEY, String.valueOf(true));
}

return LoadTsFile.createUnchecked(null, file.getAbsolutePath(), loadAttributes);
return isGeneratedByPipe
? LoadTsFile.createForPipe(null, file.getAbsolutePath(), loadAttributes)
: LoadTsFile.createUnchecked(null, file.getAbsolutePath(), loadAttributes);
}

@Override
Expand Down
Loading
Loading