From 0579b197fe05c6fbb14f6d8a7fa1a07d8aa21c2b Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 10 Nov 2026 12:19:35 +0800 Subject: [PATCH 01/17] Batch scan TimeSeriesMetadata for wide-table queries A table scan that selects (nearly) all field columns of a device previously deserialized the device's TimeSeriesMetadata column by column, re-walking the device metadata index once per leaf node. Add a batch path that reads the whole device's metadata in a single pass via TsFileSequenceReader.readDeviceMetadata, populating the per-measurement cache and building the aligned metadata through the existing modification/loader logic. The batch path is taken when the selected field columns reach a configurable fraction (device_metadata_batch_scan_ratio, default 0.8) of the table's total field columns. --- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 14 +++ .../apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++ .../source/AlignedSeriesScanUtil.java | 3 +- .../operator/source/FileLoaderUtils.java | 95 +++++++++++++++++-- .../ExternalTsFileSeriesScanUtil.java | 3 +- .../DataNodeTableOperatorGenerator.java | 12 +++ .../plan/parameter/SeriesScanOptions.java | 9 ++ .../buffer/TimeSeriesMetadataCache.java | 54 +++++++++++ .../buffer/TimeSeriesMetadataCacheTest.java | 76 +++++++++++++++ .../conf/iotdb-system.properties.template | 6 ++ 10 files changed, 269 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index c6a0d4878969f..c1d3aad8e8b37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -853,6 +853,12 @@ public class IoTDBConfig implements DefaultEncodingProvider { /** time cost(ms) threshold for slow query. Unit: millisecond */ private long slowQueryThreshold = 10000; + /** + * When a table scan queries at least this fraction of a table's field columns, deserialize all + * the device's TimeSeriesMetadata in a single batch read instead of column by column. + */ + private double deviceMetadataBatchScanRatio = 0.8; + /** time window threshold for record of history queries. Unit: minute */ private int queryCostStatWindow = 0; @@ -2962,6 +2968,14 @@ public void setSlowQueryThreshold(long slowQueryThreshold) { this.slowQueryThreshold = slowQueryThreshold; } + public double getDeviceMetadataBatchScanRatio() { + return deviceMetadataBatchScanRatio; + } + + public void setDeviceMetadataBatchScanRatio(double deviceMetadataBatchScanRatio) { + this.deviceMetadataBatchScanRatio = deviceMetadataBatchScanRatio; + } + public int getQueryCostStatWindow() { return queryCostStatWindow; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 04fe3381b7541..dae9ba26b0864 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -519,6 +519,12 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException properties.getProperty( "query_timeout_threshold", Long.toString(conf.getQueryTimeoutThreshold())))); + conf.setDeviceMetadataBatchScanRatio( + Double.parseDouble( + properties.getProperty( + "device_metadata_batch_scan_ratio", + Double.toString(conf.getDeviceMetadataBatchScanRatio())))); + conf.setCopyToAllowedExportDirs( Arrays.stream( properties diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java index 331a5a5b267d1..517a5e0718bfc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java @@ -101,7 +101,8 @@ protected AbstractAlignedTimeSeriesMetadata loadTimeSeriesMetadata( scanOptions.getGlobalTimeFilter(), isSeq, ignoreAllNullRows, - null); + null, + scanOptions.getTotalFieldColumnCount()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 4374e974db5bc..dd08c320d1f37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.path.AlignedFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.exception.CorruptedTsFileException; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; @@ -62,6 +63,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -222,7 +224,8 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata( Filter globalTimeFilter, boolean isSeq, boolean ignoreAllNullRows, - long[] rootMeasurementMetadataIndexNodeOffset) + long[] rootMeasurementMetadataIndexNodeOffset, + int totalFieldColumnCount) throws IOException { final long t1 = System.nanoTime(); boolean loadFromMem = false; @@ -241,7 +244,8 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata( context, globalTimeFilter, ignoreAllNullRows, - rootMeasurementMetadataIndexNodeOffset); + rootMeasurementMetadataIndexNodeOffset, + totalFieldColumnCount); } else { // if the tsfile is unclosed, we just get it directly from TsFileResource loadFromMem = true; alignedTimeSeriesMetadata = @@ -333,7 +337,8 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr FragmentInstanceContext context, Filter globalTimeFilter, boolean ignoreAllNullRows, - long[] rootMeasurementMetadataIndexNodeOffset) + long[] rootMeasurementMetadataIndexNodeOffset, + int totalFieldColumnCount) throws IOException { AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null; // load all the TimeseriesMetadata of vector, the first one is for time column and the @@ -348,13 +353,33 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex // we should not ignore the non-exist of device in TsFileMetadata + boolean ignoreNotExists = + context.ignoreNotExistsDevice() + || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE; + + // For a wide device (a table scan that selects almost all field columns), deserialize the + // whole device's TimeSeriesMetadata in a single batch read instead of walking the device + // metadata index once per column. + if (shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColumnCount) + && ignoreNotExists) { + return loadAlignedTimeSeriesMetadataBatch( + resource, + alignedPath, + valueMeasurementList, + context, + globalTimeFilter, + ignoreAllNullRows, + cache, + filePath, + deviceId); + } + TimeseriesMetadata timeColumn = cache.get( filePath, new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""), allSensors, - context.ignoreNotExistsDevice() - || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, + ignoreNotExists, isDebug, context, rootMeasurementMetadataIndexNodeOffset); @@ -383,8 +408,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr filePath, new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement), allSensors, - context.ignoreNotExistsDevice() - || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, + ignoreNotExists, isDebug, context, rootMeasurementMetadataIndexNodeOffset); @@ -407,6 +431,63 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr return alignedTimeSeriesMetadata; } + private static boolean shouldUseDeviceMetadataBatchScan( + int selectedColumnCount, int totalFieldColumnCount) { + return totalFieldColumnCount > 0 + && (double) selectedColumnCount / totalFieldColumnCount + >= IoTDBDescriptor.getInstance().getConfig().getDeviceMetadataBatchScanRatio(); + } + + private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBatch( + TsFileResource resource, + AlignedFullPath alignedPath, + List valueMeasurementList, + FragmentInstanceContext context, + Filter globalTimeFilter, + boolean ignoreAllNullRows, + TimeSeriesMetadataCache cache, + String filePath, + IDeviceID deviceId) + throws IOException { + Map deviceMetadata = + cache.getDeviceTimeSeriesMetadata(filePath, resource.getTsFileID(), deviceId, context); + TimeseriesMetadata timeColumn = deviceMetadata.get(""); + if (timeColumn == null) { + return null; + } + // only need time column, like count_time aggregation + if (valueMeasurementList.isEmpty()) { + return setModifications( + resource, + timeColumn, + Collections.emptyList(), + alignedPath, + context, + globalTimeFilter, + false); + } + List valueTimeSeriesMetadataList = + new ArrayList<>(valueMeasurementList.size()); + // if all the queried aligned sensors does not exist, we will return null + boolean exist = false; + for (String measurement : valueMeasurementList) { + TimeseriesMetadata valueColumn = deviceMetadata.get(measurement); + exist = (exist || (valueColumn != null)); + valueTimeSeriesMetadataList.add(valueColumn); + } + if (!ignoreAllNullRows || exist) { + return setModifications( + resource, + timeColumn, + valueTimeSeriesMetadataList, + alignedPath, + context, + globalTimeFilter, + ignoreAllNullRows); + } + return null; + } + private static AbstractAlignedTimeSeriesMetadata setModifications( TsFileResource resource, TimeseriesMetadata timeColumnMetadata, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java index e907218fe8a7b..61a2832e66b40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java @@ -83,7 +83,8 @@ static AbstractAlignedTimeSeriesMetadata loadTimeSeriesMetadata( globalTimeFilter, resource.isSeq(), context.isIgnoreAllNullRows(), - new long[] {currentDeviceOffset.getStartOffset(), currentDeviceOffset.getEndOffset()}); + new long[] {currentDeviceOffset.getStartOffset(), currentDeviceOffset.getEndOffset()}, + 0); } @FunctionalInterface diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index 01584fe230776..8b6b9b36bb88c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -1185,6 +1185,9 @@ private void addSource( topKRuntimeFilter); seriesScanOptions.setTTLForTableView(viewTTL); seriesScanOptions.setIsTableViewForTreeModel(node instanceof TreeDeviceViewScanNode); + if (!(node instanceof TreeDeviceViewScanNode) && !(node instanceof ExternalTsFileScanNode)) { + seriesScanOptions.setTotalFieldColumnCount(computeTotalFieldColumnCount(node)); + } OperatorContext operatorContext = addOperatorContext(context, node.getPlanNodeId(), className); @@ -1222,6 +1225,15 @@ private BatchDeviceEntrySource createDeviceEntrySource(DeviceTableScanNode node) .orElseGet(() -> new InMemoryDeviceEntrySource(node.getDeviceEntries())); } + private int computeTotalFieldColumnCount(DeviceTableScanNode node) { + QualifiedObjectName qualifiedObjectName = node.getQualifiedObjectName(); + TsTable tsTable = + DataNodeTableCache.getInstance() + .getTableInCache( + qualifiedObjectName.getDatabaseName(), qualifiedObjectName.getObjectName()); + return tsTable == null ? 0 : tsTable.getFieldNum(); + } + private BatchDeviceEntrySource createAggregationDeviceEntrySource(AggregationTableScanNode node) { if (node.getDeviceEntryCount() == 0 && node.getGroupingKeys().isEmpty()) { return new InMemoryDeviceEntrySource(Collections.singletonList(null)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java index 4d40bbd32c4d4..2a173e3b09e0f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java @@ -56,6 +56,7 @@ public class SeriesScanOptions implements Accountable { private PaginationController paginationController; private boolean isTableViewForTreeModel; private long ttlForTableView = Long.MAX_VALUE; + private int totalFieldColumnCount = 0; private final TopKRuntimeFilter topKRuntimeFilter; private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance( @@ -178,6 +179,14 @@ public void setIsTableViewForTreeModel(boolean isTableViewForTreeModel) { this.isTableViewForTreeModel = isTableViewForTreeModel; } + public int getTotalFieldColumnCount() { + return totalFieldColumnCount; + } + + public void setTotalFieldColumnCount(int totalFieldColumnCount) { + this.totalFieldColumnCount = totalFieldColumnCount; + } + /** * pushLimitToEachDevice==false means that all devices return total limit rows. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 68457df0ebbcd..28f7eab222964 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -50,6 +50,7 @@ import java.io.IOException; import java.lang.ref.WeakReference; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -316,6 +317,59 @@ public TimeseriesMetadata get( } } + /** + * Load all TimeseriesMetadata of a device in a single batch read and populate the per-measurement + * cache. This is used by wide-table scans (a device with many columns) to avoid walking the + * device metadata index once per column / leaf node. + * + *

The returned map holds copies of the cached metadata (or the freshly read objects when the + * cache is disabled / for external tsfiles), so the caller may apply {@code setModified} and + * {@code setChunkMetadataLoader} without corrupting cached entries — matching {@link #get}. + * + * @param filePath path of the tsfile + * @param tsFileID id of the tsfile + * @param device target device + * @param queryContext query context used for metrics and to detect external tsfiles + * @return map from measurementId (including the time column, keyed by "") to TimeseriesMetadata; + * entries whose series is empty (count == 0) are mapped to null. + * @throws IOException if the metadata cannot be read from disk + */ + public Map getDeviceTimeSeriesMetadata( + String filePath, TsFileID tsFileID, IDeviceID device, QueryContext queryContext) + throws IOException { + long startTime = System.nanoTime(); + boolean externalTsFile = queryContext.isExternalTsFileScan(); + LongConsumer timeSeriesMetadataIoSizeRecorder = + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; + + try { + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); + Map deviceMetadata = reader.readDeviceMetadata(device); + + Map result = new HashMap<>(deviceMetadata.size()); + for (Map.Entry entry : deviceMetadata.entrySet()) { + TimeseriesMetadata metadata = entry.getValue(); + if (metadata.getStatistics().getCount() == 0) { + result.put(entry.getKey(), null); + continue; + } + if (CACHE_ENABLE && !externalTsFile) { + lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, entry.getKey()), metadata); + } + result.put(entry.getKey(), new TimeseriesMetadata(metadata)); + } + return result; + } finally { + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromDiskCount().incrementAndGet(); + queryContext + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskTime() + .getAndAdd(System.nanoTime() - startTime); + } + } + public double calculateTimeSeriesMetadataHitRatio() { return lruCache.stats().hitRate(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 87526eb594b9f..de70d297b82fe 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -30,7 +30,9 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -42,7 +44,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -280,4 +284,76 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { .setMayCacheNonExistSeries(mayCacheNonExistSeries); } } + + @Test + public void testGetDeviceTimeSeriesMetadata() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + try { + String filePath = "target/batch-device.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + TimeseriesMetadata timeColumn = + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS()); + + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + TimeseriesMetadata s1 = + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS()); + + // an empty series (count == 0) must be mapped to null and not cached + TimeseriesMetadata emptyS2 = + new TimeseriesMetadata( + (byte) 0, + 0, + "s2", + TSDataType.INT32, + Statistics.getStatsByType(TSDataType.INT32), + new PublicBAOS()); + + Map deviceMetadata = new HashMap<>(); + deviceMetadata.put("", timeColumn); + deviceMetadata.put("s1", s1); + deviceMetadata.put("s2", emptyS2); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.readDeviceMetadata(deviceID)).thenReturn(deviceMetadata); + + Map result = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, new QueryContext(false, false)); + + assertNotNull(result.get("")); + assertEquals("", result.get("").getMeasurementId()); + assertNotNull(result.get("s1")); + assertEquals("s1", result.get("s1").getMeasurementId()); + assertNull(result.get("s2")); + + // the non-empty entries should now be cached for per-measurement lookups + TimeseriesMetadata cached = + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"), + Collections.singleton("s1"), + true, + false, + new QueryContext(false, false)); + assertNotNull(cached); + assertEquals("s1", cached.getMeasurementId()); + } finally { + cache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } } diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index a3fe0970318c8..798da5494cb5c 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1267,6 +1267,12 @@ max_tsblock_line_number=1000 # Datatype: long slow_query_threshold=10000 +# When a table scan queries at least this fraction of a table's field columns, deserialize all the +# device's TimeSeriesMetadata in a single batch read instead of column by column. +# effectiveMode: restart +# Datatype: double +device_metadata_batch_scan_ratio=0.8 + # Time window threshold(min) for record of history queries. # effectiveMode: hot_reload # Datatype: int From 1be4e0c85465c15183daa2013d2a9bf370ef5494 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 10 Nov 2026 16:48:21 +0800 Subject: [PATCH 02/17] Fix NPE in batch TimeSeriesMetadata read by loading chunk metadata The batch path read a device's TimeseriesMetadata via TsFileSequenceReader.readDeviceMetadata, which deserializes only the header and statistics and leaves chunkMetadataList null. The downstream DiskAlignedChunkMetadataLoader dereferences that list in getCopiedChunkMetadataList, so a wide-table scan NPE'd at query time, and the null-chunk entries also polluted the shared per-measurement LRU cache (breaking later narrow queries on the same device). Switch to getDeviceTimeseriesMetadata, which deserializes chunk metadata too, and adapt the unit test to the list-based API, asserting that chunk metadata flows through to the result. --- .../buffer/TimeSeriesMetadataCache.java | 19 +++++++++++-------- .../buffer/TimeSeriesMetadataCacheTest.java | 16 ++++++++++------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 28f7eab222964..7f4ba60ab1f72 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -346,19 +346,22 @@ public Map getDeviceTimeSeriesMetadata( TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); - Map deviceMetadata = reader.readDeviceMetadata(device); - - Map result = new HashMap<>(deviceMetadata.size()); - for (Map.Entry entry : deviceMetadata.entrySet()) { - TimeseriesMetadata metadata = entry.getValue(); + // Use the chunk-metadata-including variant: the returned TimeseriesMetadata are passed to + // setModifications and then to DiskAlignedChunkMetadataLoader, which dereferences the chunk + // metadata list, so readDeviceMetadata (which skips chunk metadata) would NPE at query time. + List deviceMetadataList = reader.getDeviceTimeseriesMetadata(device); + + Map result = new HashMap<>(deviceMetadataList.size()); + for (TimeseriesMetadata metadata : deviceMetadataList) { + String measurementId = metadata.getMeasurementId(); if (metadata.getStatistics().getCount() == 0) { - result.put(entry.getKey(), null); + result.put(measurementId, null); continue; } if (CACHE_ENABLE && !externalTsFile) { - lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, entry.getKey()), metadata); + lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), metadata); } - result.put(entry.getKey(), new TimeseriesMetadata(metadata)); + result.put(measurementId, new TimeseriesMetadata(metadata)); } return result; } finally { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index de70d297b82fe..eec5a7a13af72 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -27,6 +27,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -44,7 +45,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -321,14 +321,16 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { Statistics.getStatsByType(TSDataType.INT32), new PublicBAOS()); - Map deviceMetadata = new HashMap<>(); - deviceMetadata.put("", timeColumn); - deviceMetadata.put("s1", s1); - deviceMetadata.put("s2", emptyS2); + List deviceMetadata = new ArrayList<>(); + deviceMetadata.add(timeColumn); + deviceMetadata.add(s1); + deviceMetadata.add(emptyS2); + // the batch read must carry chunk metadata, otherwise the downstream chunk loader NPEs + s1.setChunkMetadataList(Collections.singletonList(new ChunkMetadata())); TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - Mockito.when(reader.readDeviceMetadata(deviceID)).thenReturn(deviceMetadata); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata); Map result = cache.getDeviceTimeSeriesMetadata( @@ -338,6 +340,8 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { assertEquals("", result.get("").getMeasurementId()); assertNotNull(result.get("s1")); assertEquals("s1", result.get("s1").getMeasurementId()); + // chunk metadata must flow through to the result (and the cached copy) unchanged + assertNotNull(result.get("s1").getChunkMetadataList()); assertNull(result.get("s2")); // the non-empty entries should now be cached for per-measurement lookups From 758f4aea543974d205422c4cc62d34e1d6dd5940 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 10 Nov 2026 17:17:25 +0800 Subject: [PATCH 03/17] Clean up batch TimeSeriesMetadata scan path Remove the unreachable time-column-only branch in the batch loader (the batch gate already requires a non-empty selection), and validate device_metadata_batch_scan_ratio so a non-positive value falls back to the default instead of routing every table scan through the batch path. Document the valid range in the properties template. --- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 11 +++++++++-- .../execution/operator/source/FileLoaderUtils.java | 11 ----------- .../resources/conf/iotdb-system.properties.template | 1 + 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index dae9ba26b0864..df91d783ddb7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -519,11 +519,18 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException properties.getProperty( "query_timeout_threshold", Long.toString(conf.getQueryTimeoutThreshold())))); - conf.setDeviceMetadataBatchScanRatio( + double deviceMetadataBatchScanRatio = Double.parseDouble( properties.getProperty( "device_metadata_batch_scan_ratio", - Double.toString(conf.getDeviceMetadataBatchScanRatio())))); + Double.toString(conf.getDeviceMetadataBatchScanRatio()))); + // a non-positive ratio would route every table scan, even single-column ones, through the + // batch path; keep the current value instead + deviceMetadataBatchScanRatio = + deviceMetadataBatchScanRatio <= 0 + ? conf.getDeviceMetadataBatchScanRatio() + : deviceMetadataBatchScanRatio; + conf.setDeviceMetadataBatchScanRatio(deviceMetadataBatchScanRatio); conf.setCopyToAllowedExportDirs( Arrays.stream( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index dd08c320d1f37..857328a58838c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -455,17 +455,6 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa if (timeColumn == null) { return null; } - // only need time column, like count_time aggregation - if (valueMeasurementList.isEmpty()) { - return setModifications( - resource, - timeColumn, - Collections.emptyList(), - alignedPath, - context, - globalTimeFilter, - false); - } List valueTimeSeriesMetadataList = new ArrayList<>(valueMeasurementList.size()); // if all the queried aligned sensors does not exist, we will return null diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 798da5494cb5c..e8021790bbe83 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1269,6 +1269,7 @@ slow_query_threshold=10000 # When a table scan queries at least this fraction of a table's field columns, deserialize all the # device's TimeSeriesMetadata in a single batch read instead of column by column. +# Value must be in (0, 1]; a non-positive value falls back to the default. # effectiveMode: restart # Datatype: double device_metadata_batch_scan_ratio=0.8 From 2563c508aeeeecf21d4de53a7c6ea46148fa886b Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 10 Nov 2026 18:15:07 +0800 Subject: [PATCH 04/17] Expand batch TimeSeriesMetadata scan to device-level time index files Remove the ignoreNotExists gate so the batch path is also taken for files with a device-level time index, where previously only file-level-index files (or recovery) used it. When the batch read finds the device is missing from the file, replicate the per-column path: skip silently when ignoreNotExists is set, otherwise throw CorruptedTsFileException to surface the time-index/metadata inconsistency. --- .../operator/source/FileLoaderUtils.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 857328a58838c..3f5f174da85d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -360,8 +360,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // For a wide device (a table scan that selects almost all field columns), deserialize the // whole device's TimeSeriesMetadata in a single batch read instead of walking the device // metadata index once per column. - if (shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColumnCount) - && ignoreNotExists) { + if (shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColumnCount)) { return loadAlignedTimeSeriesMetadataBatch( resource, alignedPath, @@ -369,6 +368,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr context, globalTimeFilter, ignoreAllNullRows, + ignoreNotExists, cache, filePath, deviceId); @@ -445,12 +445,27 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa FragmentInstanceContext context, Filter globalTimeFilter, boolean ignoreAllNullRows, + boolean ignoreNotExists, TimeSeriesMetadataCache cache, String filePath, IDeviceID deviceId) throws IOException { Map deviceMetadata = cache.getDeviceTimeSeriesMetadata(filePath, resource.getTsFileID(), deviceId, context); + if (deviceMetadata.isEmpty()) { + // device not found in this file. With a file-level time index (or during recovery) that is + // expected, so skip; with a device-level time index the device should be present, so surface + // the inconsistency instead of silently dropping the device's data. + if (ignoreNotExists) { + return null; + } + throw new CorruptedTsFileException( + resource.getTsFile(), + CorruptedTsFileException.Stage.READ_TIMESERIES_METADATA, + DataNodeQueryMessages + .EXCEPTION_FAILED_TO_READ_TIMESERIES_METADATA_THE_TSFILE_MAY_BE_CORRUPTED_PLEASE_CHECK_THE_LOGS_FOR_THE_CORRUPTED_FILE_PATH_0B9E652E, + null); + } TimeseriesMetadata timeColumn = deviceMetadata.get(""); if (timeColumn == null) { return null; From 31ecea85e5650d4612c37d8a15d375e058b57656 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 10 Nov 2026 18:22:02 +0800 Subject: [PATCH 05/17] Hardcode device metadata batch scan ratio to 0.6 Remove the device_metadata_batch_scan_ratio config parameter and hardcode the threshold at 0.6 (previously configurable, default 0.8) so the batch path covers a wider set of scans without requiring configuration. --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 14 -------------- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 13 ------------- .../execution/operator/source/FileLoaderUtils.java | 10 +++++++--- .../conf/iotdb-system.properties.template | 7 ------- 4 files changed, 7 insertions(+), 37 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index c1d3aad8e8b37..c6a0d4878969f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -853,12 +853,6 @@ public class IoTDBConfig implements DefaultEncodingProvider { /** time cost(ms) threshold for slow query. Unit: millisecond */ private long slowQueryThreshold = 10000; - /** - * When a table scan queries at least this fraction of a table's field columns, deserialize all - * the device's TimeSeriesMetadata in a single batch read instead of column by column. - */ - private double deviceMetadataBatchScanRatio = 0.8; - /** time window threshold for record of history queries. Unit: minute */ private int queryCostStatWindow = 0; @@ -2968,14 +2962,6 @@ public void setSlowQueryThreshold(long slowQueryThreshold) { this.slowQueryThreshold = slowQueryThreshold; } - public double getDeviceMetadataBatchScanRatio() { - return deviceMetadataBatchScanRatio; - } - - public void setDeviceMetadataBatchScanRatio(double deviceMetadataBatchScanRatio) { - this.deviceMetadataBatchScanRatio = deviceMetadataBatchScanRatio; - } - public int getQueryCostStatWindow() { return queryCostStatWindow; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index df91d783ddb7e..04fe3381b7541 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -519,19 +519,6 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException properties.getProperty( "query_timeout_threshold", Long.toString(conf.getQueryTimeoutThreshold())))); - double deviceMetadataBatchScanRatio = - Double.parseDouble( - properties.getProperty( - "device_metadata_batch_scan_ratio", - Double.toString(conf.getDeviceMetadataBatchScanRatio()))); - // a non-positive ratio would route every table scan, even single-column ones, through the - // batch path; keep the current value instead - deviceMetadataBatchScanRatio = - deviceMetadataBatchScanRatio <= 0 - ? conf.getDeviceMetadataBatchScanRatio() - : deviceMetadataBatchScanRatio; - conf.setDeviceMetadataBatchScanRatio(deviceMetadataBatchScanRatio); - conf.setCopyToAllowedExportDirs( Arrays.stream( properties diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 3f5f174da85d8..25f532af27017 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -22,7 +22,6 @@ import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.path.AlignedFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; -import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.exception.CorruptedTsFileException; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; @@ -74,6 +73,12 @@ public class FileLoaderUtils { private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET = SeriesScanCostMetricSet.getInstance(); + /** + * Fraction of a table's field columns that a scan must select for it to use the batch + * device-metadata read instead of the per-column path. + */ + private static final double DEVICE_METADATA_BATCH_SCAN_RATIO = 0.6; + private FileLoaderUtils() { // util class } @@ -434,8 +439,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr private static boolean shouldUseDeviceMetadataBatchScan( int selectedColumnCount, int totalFieldColumnCount) { return totalFieldColumnCount > 0 - && (double) selectedColumnCount / totalFieldColumnCount - >= IoTDBDescriptor.getInstance().getConfig().getDeviceMetadataBatchScanRatio(); + && (double) selectedColumnCount / totalFieldColumnCount >= DEVICE_METADATA_BATCH_SCAN_RATIO; } private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBatch( diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index e8021790bbe83..a3fe0970318c8 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1267,13 +1267,6 @@ max_tsblock_line_number=1000 # Datatype: long slow_query_threshold=10000 -# When a table scan queries at least this fraction of a table's field columns, deserialize all the -# device's TimeSeriesMetadata in a single batch read instead of column by column. -# Value must be in (0, 1]; a non-positive value falls back to the default. -# effectiveMode: restart -# Datatype: double -device_metadata_batch_scan_ratio=0.8 - # Time window threshold(min) for record of history queries. # effectiveMode: hot_reload # Datatype: int From 2cc96048b01da2531d30ba50c5ea0192f4a20c54 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 11 Nov 2026 09:56:32 +0800 Subject: [PATCH 06/17] Synchronize batch metadata read and include device in corruption error Serialize concurrent reads of the same device+file in getDeviceTimeSeriesMetadata (matching the per-column get) so threads don't each deserialize the whole device and race on the shared reader. Also attach the missing device id to the CorruptedTsFileException thrown when a device is absent from a device-level time index file. --- .../operator/source/FileLoaderUtils.java | 5 +- .../buffer/TimeSeriesMetadataCache.java | 46 ++++++++++++------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 25f532af27017..98dcf4c7991e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.exception.CorruptedTsFileException; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; @@ -468,7 +469,9 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa CorruptedTsFileException.Stage.READ_TIMESERIES_METADATA, DataNodeQueryMessages .EXCEPTION_FAILED_TO_READ_TIMESERIES_METADATA_THE_TSFILE_MAY_BE_CORRUPTED_PLEASE_CHECK_THE_LOGS_FOR_THE_CORRUPTED_FILE_PATH_0B9E652E, - null); + new IOException( + String.format( + StorageEngineMessages.DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT, deviceId))); } TimeseriesMetadata timeColumn = deviceMetadata.get(""); if (timeColumn == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 7f4ba60ab1f72..30457bf0570cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -346,24 +346,15 @@ public Map getDeviceTimeSeriesMetadata( TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); - // Use the chunk-metadata-including variant: the returned TimeseriesMetadata are passed to - // setModifications and then to DiskAlignedChunkMetadataLoader, which dereferences the chunk - // metadata list, so readDeviceMetadata (which skips chunk metadata) would NPE at query time. - List deviceMetadataList = reader.getDeviceTimeseriesMetadata(device); - - Map result = new HashMap<>(deviceMetadataList.size()); - for (TimeseriesMetadata metadata : deviceMetadataList) { - String measurementId = metadata.getMeasurementId(); - if (metadata.getStatistics().getCount() == 0) { - result.put(measurementId, null); - continue; - } - if (CACHE_ENABLE && !externalTsFile) { - lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), metadata); + if (CACHE_ENABLE && !externalTsFile) { + // serialize concurrent reads of the same device+file (same lock key as get) so threads + // don't each deserialize the whole device and race on the shared reader + synchronized ( + devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, WeakReference::new)) { + return readDeviceTimeSeriesMetadata(reader, tsFileID, device, true); } - result.put(measurementId, new TimeseriesMetadata(metadata)); } - return result; + return readDeviceTimeSeriesMetadata(reader, tsFileID, device, false); } finally { queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromDiskCount().incrementAndGet(); queryContext @@ -373,6 +364,29 @@ public Map getDeviceTimeSeriesMetadata( } } + private Map readDeviceTimeSeriesMetadata( + TsFileSequenceReader reader, TsFileID tsFileID, IDeviceID device, boolean cacheEnabled) + throws IOException { + // Use the chunk-metadata-including variant: the returned TimeseriesMetadata are passed to + // setModifications and then to DiskAlignedChunkMetadataLoader, which dereferences the chunk + // metadata list, so readDeviceMetadata (which skips chunk metadata) would NPE at query time. + List deviceMetadataList = reader.getDeviceTimeseriesMetadata(device); + + Map result = new HashMap<>(deviceMetadataList.size()); + for (TimeseriesMetadata metadata : deviceMetadataList) { + String measurementId = metadata.getMeasurementId(); + if (metadata.getStatistics().getCount() == 0) { + result.put(measurementId, null); + continue; + } + if (cacheEnabled) { + lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), metadata); + } + result.put(measurementId, new TimeseriesMetadata(metadata)); + } + return result; + } + public double calculateTimeSeriesMetadataHitRatio() { return lruCache.stats().hitRate(); } From 4bcb8b62e9fccc91dd5ff669ba84651ff06b87e2 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 11 Nov 2026 10:05:04 +0800 Subject: [PATCH 07/17] Fix batch scan ratio denominator and cache empty series Count every value column (FIELD + ATTRIBUTE) as the batch scan ratio denominator instead of only FIELD columns, matching the selected-column numerator. Cache empty (count == 0) series as the null placeholder in the batch read, matching the per-column get, so later per-column reads don't re-read them. Add a unit test for the absent-device (empty map) path. --- .../operator/source/FileLoaderUtils.java | 3 +- .../DataNodeTableOperatorGenerator.java | 5 ++- .../buffer/TimeSeriesMetadataCache.java | 21 ++++++++++--- .../buffer/TimeSeriesMetadataCacheTest.java | 31 ++++++++++++++++++- 4 files changed, 53 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 98dcf4c7991e3..a154789e30cfb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -456,7 +456,8 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa IDeviceID deviceId) throws IOException { Map deviceMetadata = - cache.getDeviceTimeSeriesMetadata(filePath, resource.getTsFileID(), deviceId, context); + cache.getDeviceTimeSeriesMetadata( + filePath, resource.getTsFileID(), deviceId, context, ignoreNotExists); if (deviceMetadata.isEmpty()) { // device not found in this file. With a file-level time index (or during recovery) that is // expected, so skip; with a device-level time index the device should be present, so surface diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index 8b6b9b36bb88c..6d8b235bd1322 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -1231,7 +1231,10 @@ private int computeTotalFieldColumnCount(DeviceTableScanNode node) { DataNodeTableCache.getInstance() .getTableInCache( qualifiedObjectName.getDatabaseName(), qualifiedObjectName.getObjectName()); - return tsTable == null ? 0 : tsTable.getFieldNum(); + // count every value column (FIELD + ATTRIBUTE) the scan may select, matching the denominator + // used by shouldUseDeviceMetadataBatchScan; getColumnNum() also includes the tag (id) columns + // and the time column, hence the subtraction. + return tsTable == null ? 0 : tsTable.getColumnNum() - tsTable.getTagNum() - 1; } private BatchDeviceEntrySource createAggregationDeviceEntrySource(AggregationTableScanNode node) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 30457bf0570cf..c0d1eacee1837 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -335,7 +335,11 @@ public TimeseriesMetadata get( * @throws IOException if the metadata cannot be read from disk */ public Map getDeviceTimeSeriesMetadata( - String filePath, TsFileID tsFileID, IDeviceID device, QueryContext queryContext) + String filePath, + TsFileID tsFileID, + IDeviceID device, + QueryContext queryContext, + boolean ignoreNotExists) throws IOException { long startTime = System.nanoTime(); boolean externalTsFile = queryContext.isExternalTsFileScan(); @@ -351,10 +355,10 @@ public Map getDeviceTimeSeriesMetadata( // don't each deserialize the whole device and race on the shared reader synchronized ( devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, WeakReference::new)) { - return readDeviceTimeSeriesMetadata(reader, tsFileID, device, true); + return readDeviceTimeSeriesMetadata(reader, tsFileID, device, true, ignoreNotExists); } } - return readDeviceTimeSeriesMetadata(reader, tsFileID, device, false); + return readDeviceTimeSeriesMetadata(reader, tsFileID, device, false, ignoreNotExists); } finally { queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromDiskCount().incrementAndGet(); queryContext @@ -365,7 +369,11 @@ public Map getDeviceTimeSeriesMetadata( } private Map readDeviceTimeSeriesMetadata( - TsFileSequenceReader reader, TsFileID tsFileID, IDeviceID device, boolean cacheEnabled) + TsFileSequenceReader reader, + TsFileID tsFileID, + IDeviceID device, + boolean cacheEnabled, + boolean ignoreNotExists) throws IOException { // Use the chunk-metadata-including variant: the returned TimeseriesMetadata are passed to // setModifications and then to DiskAlignedChunkMetadataLoader, which dereferences the chunk @@ -377,6 +385,11 @@ private Map readDeviceTimeSeriesMetadata( String measurementId = metadata.getMeasurementId(); if (metadata.getStatistics().getCount() == 0) { result.put(measurementId, null); + if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { + lruCache.put( + new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), + NULL_EXISTS_CACHE_PLACE_HOLDER); + } continue; } if (cacheEnabled) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index eec5a7a13af72..67351702f8162 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -58,6 +58,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; public class TimeSeriesMetadataCacheTest { @@ -334,7 +335,7 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { Map result = cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, new QueryContext(false, false)); + filePath, tsFileID, deviceID, new QueryContext(false, false), false); assertNotNull(result.get("")); assertEquals("", result.get("").getMeasurementId()); @@ -360,4 +361,32 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { fileReaderManager.closeFileAndRemoveReader(tsFileID); } } + + @Test + public void testGetDeviceTimeSeriesMetadataForAbsentDevice() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + try { + String filePath = "target/batch-absent-device.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) + .thenReturn(Collections.emptyList()); + + Map result = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, new QueryContext(false, false), false); + + // an absent device yields an empty map, which loadAlignedTimeSeriesMetadataBatch then + // distinguishes from "present but empty time column" + assertTrue(result.isEmpty()); + } finally { + cache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } } From a6a563427f20d7826e03570329476eda79c629e2 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 11 Nov 2026 10:09:08 +0800 Subject: [PATCH 08/17] Add unit test for batch scan absent-device throw logic Cover the loadAlignedTimeSeriesMetadataBatch empty-map branch: a device absent from a device-level time index file throws CorruptedTsFileException, while with a file-level time index (or during recovery) it is skipped. --- .../operator/source/FileLoaderUtilsTest.java | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java new file mode 100644 index 0000000000000..9fab5e782a821 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source; + +import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.db.exception.CorruptedTsFileException; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; + +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; +import org.apache.tsfile.read.filter.basic.Filter; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class FileLoaderUtilsTest { + + @Test + public void testLoadAlignedTimeSeriesMetadataBatchAbsentDevice() throws Exception { + TsFileResource resource = Mockito.mock(TsFileResource.class); + Mockito.when(resource.getTsFileID()).thenReturn(new TsFileID()); + Mockito.when(resource.getTsFile()).thenReturn(new File("target/x.tsfile")); + + TimeSeriesMetadataCache cache = Mockito.mock(TimeSeriesMetadataCache.class); + Mockito.when( + cache.getDeviceTimeSeriesMetadata( + Mockito.anyString(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.anyBoolean())) + .thenReturn(Collections.emptyMap()); + + IDeviceID deviceId = Factory.DEFAULT_FACTORY.create("root.device"); + + Method method = + FileLoaderUtils.class.getDeclaredMethod( + "loadAlignedTimeSeriesMetadataBatch", + TsFileResource.class, + AlignedFullPath.class, + List.class, + FragmentInstanceContext.class, + Filter.class, + boolean.class, + boolean.class, + TimeSeriesMetadataCache.class, + String.class, + IDeviceID.class); + method.setAccessible(true); + + // a device absent from a device-level time index file is an inconsistency: surface it + try { + method.invoke( + null, + resource, + null, + Collections.emptyList(), + null, + null, + false, + false, + cache, + "target/x.tsfile", + deviceId); + fail("expected CorruptedTsFileException when the device is absent and ignoreNotExists=false"); + } catch (InvocationTargetException e) { + assertTrue(e.getCause() instanceof CorruptedTsFileException); + } + + // with a file-level time index (or during recovery) an absent device is expected: skip it + try { + Object result = + method.invoke( + null, + resource, + null, + Collections.emptyList(), + null, + null, + false, + true, + cache, + "target/x.tsfile", + deviceId); + assertNull(result); + } catch (InvocationTargetException e) { + fail("unexpected exception: " + e.getCause()); + } + } +} From 1199ab3f885a736a33428f7180b128ed659cf736 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 11 Nov 2026 10:12:47 +0800 Subject: [PATCH 09/17] Add end-to-end test for wide-table batch metadata scan Drive AlignedSeriesScanUtil with all value columns selected and a matching total field column count so the batch path is taken, and assert the aligned pages are still read correctly. --- .../series/AlignedSeriesScanBatchTest.java | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java new file mode 100644 index 0000000000000..abcfbb7c0b3ef --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; +import org.apache.iotdb.db.utils.EnvironmentUtils; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; + +/** Verifies the wide-table batch metadata scan (see FileLoaderUtils) reads data correctly. */ +public class AlignedSeriesScanBatchTest extends AbstractAlignedSeriesScanTest { + + @Test + public void testBatchScanReadsAlignedData() throws IOException, IllegalPathException { + AlignedFullPath scanPath = + new AlignedFullPath( + TEST_DEVICE, + Arrays.asList("s1", "s2", "s3"), + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32), + new MeasurementSchema("s3", TSDataType.INT32))); + + SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); + builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); + SeriesScanOptions scanOptions = builder.build(); + // all 3 value columns are selected, so selected/total = 1.0 >= 0.6 and the batch path is taken + scanOptions.setTotalFieldColumnCount(3); + + AlignedSeriesScanUtil seriesScanUtil = + new AlignedSeriesScanUtil( + scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); + + // the batch path must still assemble the same aligned pages: file 1 has 10 points + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + } +} From 42749a444cdf2ccd8fb948381700a7deb8777377 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 11 Nov 2026 11:27:58 +0800 Subject: [PATCH 10/17] Don't use batch metadata scan for single-column devices The batch read offers no benefit for a single-column device (one leaf either way) and it bypasses the per-column cache's debug logging, which IoTDBDebugQueryIT.tableTest relies on. Require at least two value columns before taking the batch path. --- .../execution/operator/source/FileLoaderUtils.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index a154789e30cfb..9e8d522b3c02c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -439,7 +439,9 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr private static boolean shouldUseDeviceMetadataBatchScan( int selectedColumnCount, int totalFieldColumnCount) { - return totalFieldColumnCount > 0 + // a single-column device gains nothing from a batch read (one leaf either way), and it would + // bypass the per-column cache's debug logging + return totalFieldColumnCount > 1 && (double) selectedColumnCount / totalFieldColumnCount >= DEVICE_METADATA_BATCH_SCAN_RATIO; } From 98aba2c20aa9415399df8f0da09e2370a870bfd2 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 16:27:22 +0800 Subject: [PATCH 11/17] Reuse the metadata cache for batch device scans getDeviceTimeSeriesMetadata always deserialized the device's whole metadata index and then re-populated the cache, never reading it, so repeated wide-table scans were slower than the per-column path that it populates. Check the per-measurement cache first, read the device only when one of the requested measurements is missing, and cache just those measurements (plus the null placeholders for absent ones). Align the absent-device behavior with the per-column get(): skip the device when the file's bloom filter proves it is missing, otherwise raise the read IOException so the existing wrapper records the suspicious-disk metric. Require a null device offset for the batch path instead of relying on the zero sentinel used by external scans, and extract the aligned metadata assembly shared by both paths. Count only FIELD columns in the ratio denominator to match the smaller numerator, and extend the tests to cover the cache hit, the bloom-filter decisions, the gating boundary and the batch path actually being taken. --- .../operator/source/FileLoaderUtils.java | 83 ++++++------ .../DataNodeTableOperatorGenerator.java | 7 +- .../buffer/TimeSeriesMetadataCache.java | 126 +++++++++++++++--- .../operator/source/FileLoaderUtilsTest.java | 98 ++------------ .../buffer/TimeSeriesMetadataCacheTest.java | 71 ++++++++-- .../series/AlignedSeriesScanBatchTest.java | 29 ++++ 6 files changed, 256 insertions(+), 158 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 9e8d522b3c02c..ca34d9f9ba495 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -25,7 +25,6 @@ import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.exception.CorruptedTsFileException; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; -import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; @@ -365,12 +364,15 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // For a wide device (a table scan that selects almost all field columns), deserialize the // whole device's TimeSeriesMetadata in a single batch read instead of walking the device - // metadata index once per column. - if (shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColumnCount)) { + // metadata index once per column. External tsfiles address metadata by offset, so they must + // stay on the per-column path. + if (rootMeasurementMetadataIndexNodeOffset == null + && shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColumnCount)) { return loadAlignedTimeSeriesMetadataBatch( resource, alignedPath, valueMeasurementList, + allSensors, context, globalTimeFilter, ignoreAllNullRows, @@ -406,10 +408,8 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr } else { List valueTimeSeriesMetadataList = new ArrayList<>(valueMeasurementList.size()); - // if all the queried aligned sensors does not exist, we will return null - boolean exist = false; for (String measurement : valueMeasurementList) { - TimeseriesMetadata valueColumn = + valueTimeSeriesMetadataList.add( cache.get( filePath, new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement), @@ -417,27 +417,23 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr ignoreNotExists, isDebug, context, - rootMeasurementMetadataIndexNodeOffset); - exist = (exist || (valueColumn != null)); - valueTimeSeriesMetadataList.add(valueColumn); - } - if (!ignoreAllNullRows || exist) { - alignedTimeSeriesMetadata = - setModifications( - resource, - timeColumn, - valueTimeSeriesMetadataList, - alignedPath, - context, - globalTimeFilter, - ignoreAllNullRows); + rootMeasurementMetadataIndexNodeOffset)); } + alignedTimeSeriesMetadata = + buildAlignedTimeSeriesMetadata( + resource, + alignedPath, + timeColumn, + valueTimeSeriesMetadataList, + context, + globalTimeFilter, + ignoreAllNullRows); } } return alignedTimeSeriesMetadata; } - private static boolean shouldUseDeviceMetadataBatchScan( + static boolean shouldUseDeviceMetadataBatchScan( int selectedColumnCount, int totalFieldColumnCount) { // a single-column device gains nothing from a batch read (one leaf either way), and it would // bypass the per-column cache's debug logging @@ -449,6 +445,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa TsFileResource resource, AlignedFullPath alignedPath, List valueMeasurementList, + Set allSensors, FragmentInstanceContext context, Filter globalTimeFilter, boolean ignoreAllNullRows, @@ -459,22 +456,11 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa throws IOException { Map deviceMetadata = cache.getDeviceTimeSeriesMetadata( - filePath, resource.getTsFileID(), deviceId, context, ignoreNotExists); + filePath, resource.getTsFileID(), deviceId, allSensors, context, ignoreNotExists); if (deviceMetadata.isEmpty()) { - // device not found in this file. With a file-level time index (or during recovery) that is - // expected, so skip; with a device-level time index the device should be present, so surface - // the inconsistency instead of silently dropping the device's data. - if (ignoreNotExists) { - return null; - } - throw new CorruptedTsFileException( - resource.getTsFile(), - CorruptedTsFileException.Stage.READ_TIMESERIES_METADATA, - DataNodeQueryMessages - .EXCEPTION_FAILED_TO_READ_TIMESERIES_METADATA_THE_TSFILE_MAY_BE_CORRUPTED_PLEASE_CHECK_THE_LOGS_FOR_THE_CORRUPTED_FILE_PATH_0B9E652E, - new IOException( - String.format( - StorageEngineMessages.DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT, deviceId))); + // the cache applies the same bloom-filter / ignoreNotExists rules as the per-column get(), + // so an empty map means the device is expected to be absent from this file + return null; } TimeseriesMetadata timeColumn = deviceMetadata.get(""); if (timeColumn == null) { @@ -482,12 +468,31 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa } List valueTimeSeriesMetadataList = new ArrayList<>(valueMeasurementList.size()); + for (String measurement : valueMeasurementList) { + valueTimeSeriesMetadataList.add(deviceMetadata.get(measurement)); + } + return buildAlignedTimeSeriesMetadata( + resource, + alignedPath, + timeColumn, + valueTimeSeriesMetadataList, + context, + globalTimeFilter, + ignoreAllNullRows); + } + + private static AbstractAlignedTimeSeriesMetadata buildAlignedTimeSeriesMetadata( + TsFileResource resource, + AlignedFullPath alignedPath, + TimeseriesMetadata timeColumn, + List valueTimeSeriesMetadataList, + FragmentInstanceContext context, + Filter globalTimeFilter, + boolean ignoreAllNullRows) { // if all the queried aligned sensors does not exist, we will return null boolean exist = false; - for (String measurement : valueMeasurementList) { - TimeseriesMetadata valueColumn = deviceMetadata.get(measurement); + for (TimeseriesMetadata valueColumn : valueTimeSeriesMetadataList) { exist = (exist || (valueColumn != null)); - valueTimeSeriesMetadataList.add(valueColumn); } if (!ignoreAllNullRows || exist) { return setModifications( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index 6d8b235bd1322..47679da161a24 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -1231,10 +1231,9 @@ private int computeTotalFieldColumnCount(DeviceTableScanNode node) { DataNodeTableCache.getInstance() .getTableInCache( qualifiedObjectName.getDatabaseName(), qualifiedObjectName.getObjectName()); - // count every value column (FIELD + ATTRIBUTE) the scan may select, matching the denominator - // used by shouldUseDeviceMetadataBatchScan; getColumnNum() also includes the tag (id) columns - // and the time column, hence the subtraction. - return tsTable == null ? 0 : tsTable.getColumnNum() - tsTable.getTagNum() - 1; + // the numerator only contains FIELD columns, so the denominator must be the field count too; + // getColumnNum() would also count tag, attribute and time columns + return tsTable == null ? 0 : tsTable.getFieldNum(); } private BatchDeviceEntrySource createAggregationDeviceEntrySource(AggregationTableScanNode node) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index c0d1eacee1837..922f2c4e4c959 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -318,26 +318,32 @@ public TimeseriesMetadata get( } /** - * Load all TimeseriesMetadata of a device in a single batch read and populate the per-measurement - * cache. This is used by wide-table scans (a device with many columns) to avoid walking the - * device metadata index once per column / leaf node. + * Load the TimeseriesMetadata of {@code allSensors} of a device, reading the device in a single + * batch when the per-measurement cache does not already hold all of them. This is used by + * wide-table scans (a device with many columns) to avoid walking the device metadata index once + * per column / leaf node. * - *

The returned map holds copies of the cached metadata (or the freshly read objects when the - * cache is disabled / for external tsfiles), so the caller may apply {@code setModified} and - * {@code setChunkMetadataLoader} without corrupting cached entries — matching {@link #get}. + *

The returned map always holds copies of the cached (or freshly read) metadata, so the caller + * may apply {@code setModified} and {@code setChunkMetadataLoader} without corrupting cached + * entries — matching {@link #get}. * * @param filePath path of the tsfile * @param tsFileID id of the tsfile * @param device target device + * @param allSensors measurements the caller needs, including the time column keyed by "" * @param queryContext query context used for metrics and to detect external tsfiles + * @param ignoreNotExists whether a device missing from a file that claims a device-level time + * index should be skipped instead of reported as an inconsistency * @return map from measurementId (including the time column, keyed by "") to TimeseriesMetadata; - * entries whose series is empty (count == 0) are mapped to null. - * @throws IOException if the metadata cannot be read from disk + * entries whose series is empty (count == 0) are mapped to null + * @throws IOException if the metadata cannot be read from disk or the device is unexpectedly + * absent */ public Map getDeviceTimeSeriesMetadata( String filePath, TsFileID tsFileID, IDeviceID device, + Set allSensors, QueryContext queryContext, boolean ignoreNotExists) throws IOException { @@ -345,33 +351,85 @@ public Map getDeviceTimeSeriesMetadata( boolean externalTsFile = queryContext.isExternalTsFileScan(); LongConsumer timeSeriesMetadataIoSizeRecorder = queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; + boolean cacheHit = true; try { - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); if (CACHE_ENABLE && !externalTsFile) { + Map cached = + getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors); + if (cached != null) { + return cached; + } // serialize concurrent reads of the same device+file (same lock key as get) so threads // don't each deserialize the whole device and race on the shared reader synchronized ( devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, WeakReference::new)) { - return readDeviceTimeSeriesMetadata(reader, tsFileID, device, true, ignoreNotExists); + cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors); + if (cached != null) { + return cached; + } + cacheHit = false; + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); + return readDeviceTimeSeriesMetadata( + reader, tsFileID, device, allSensors, true, ignoreNotExists); } } - return readDeviceTimeSeriesMetadata(reader, tsFileID, device, false, ignoreNotExists); + cacheHit = false; + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); + return readDeviceTimeSeriesMetadata( + reader, tsFileID, device, allSensors, false, ignoreNotExists); } finally { - queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromDiskCount().incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskTime() - .getAndAdd(System.nanoTime() - startTime); + if (cacheHit) { + queryContext + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .incrementAndGet(); + queryContext + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheTime() + .getAndAdd(System.nanoTime() - startTime); + } else { + queryContext + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .incrementAndGet(); + queryContext + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskTime() + .getAndAdd(System.nanoTime() - startTime); + } + } + } + + /** + * @return copies of the cached metadata for every measurement in {@code allSensors}, or null as + * soon as one of them is not cached and the device therefore has to be read from disk + */ + private Map getCachedDeviceTimeSeriesMetadata( + TsFileID tsFileID, IDeviceID device, Set allSensors) { + Map result = new HashMap<>(allSensors.size()); + for (String measurement : allSensors) { + TimeseriesMetadata metadata = + lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); + if (metadata == null) { + return null; + } + result.put( + measurement, + metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new TimeseriesMetadata(metadata)); } + return result; } private Map readDeviceTimeSeriesMetadata( TsFileSequenceReader reader, TsFileID tsFileID, IDeviceID device, + Set allSensors, boolean cacheEnabled, boolean ignoreNotExists) throws IOException { @@ -379,10 +437,27 @@ private Map readDeviceTimeSeriesMetadata( // setModifications and then to DiskAlignedChunkMetadataLoader, which dereferences the chunk // metadata list, so readDeviceMetadata (which skips chunk metadata) would NPE at query time. List deviceMetadataList = reader.getDeviceTimeseriesMetadata(device); + if (deviceMetadataList.isEmpty()) { + // Match get(): an absent device is only an inconsistency when ignoreNotExists is false and + // the bloom filter does not already prove that the device is missing from this file + if (ignoreNotExists) { + return Collections.emptyMap(); + } + BloomFilter bloomFilter = reader.readBloomFilter(); + if (bloomFilter != null + && !bloomFilter.contains(device.toString() + IoTDBConstant.PATH_SEPARATOR)) { + return Collections.emptyMap(); + } + throw new IOException( + String.format(StorageEngineMessages.DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT, device)); + } - Map result = new HashMap<>(deviceMetadataList.size()); + Map result = new HashMap<>(allSensors.size()); for (TimeseriesMetadata metadata : deviceMetadataList) { String measurementId = metadata.getMeasurementId(); + if (!allSensors.contains(measurementId)) { + continue; + } if (metadata.getStatistics().getCount() == 0) { result.put(measurementId, null); if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { @@ -397,6 +472,19 @@ private Map readDeviceTimeSeriesMetadata( } result.put(measurementId, new TimeseriesMetadata(metadata)); } + // remember selected measurements that do not exist in this file, so a later lookup of the same + // device can be served from the cache instead of reading the device again + for (String measurement : allSensors) { + if (result.containsKey(measurement)) { + continue; + } + result.put(measurement, null); + if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { + lruCache.put( + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), + NULL_EXISTS_CACHE_PLACE_HOLDER); + } + } return result; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java index 9fab5e782a821..92431d1114b8c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java @@ -19,101 +19,27 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; -import org.apache.iotdb.commons.path.AlignedFullPath; -import org.apache.iotdb.db.exception.CorruptedTsFileException; -import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; -import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; - -import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.IDeviceID.Factory; -import org.apache.tsfile.read.filter.basic.Filter; import org.junit.Test; -import org.mockito.Mockito; - -import java.io.File; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Collections; -import java.util.List; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; public class FileLoaderUtilsTest { @Test - public void testLoadAlignedTimeSeriesMetadataBatchAbsentDevice() throws Exception { - TsFileResource resource = Mockito.mock(TsFileResource.class); - Mockito.when(resource.getTsFileID()).thenReturn(new TsFileID()); - Mockito.when(resource.getTsFile()).thenReturn(new File("target/x.tsfile")); - - TimeSeriesMetadataCache cache = Mockito.mock(TimeSeriesMetadataCache.class); - Mockito.when( - cache.getDeviceTimeSeriesMetadata( - Mockito.anyString(), - Mockito.any(), - Mockito.any(), - Mockito.any(), - Mockito.anyBoolean())) - .thenReturn(Collections.emptyMap()); - - IDeviceID deviceId = Factory.DEFAULT_FACTORY.create("root.device"); + public void testShouldUseDeviceMetadataBatchScan() { + // no field count is known (tree model, external tsfiles, or a table missing from the cache) + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 0)); - Method method = - FileLoaderUtils.class.getDeclaredMethod( - "loadAlignedTimeSeriesMetadataBatch", - TsFileResource.class, - AlignedFullPath.class, - List.class, - FragmentInstanceContext.class, - Filter.class, - boolean.class, - boolean.class, - TimeSeriesMetadataCache.class, - String.class, - IDeviceID.class); - method.setAccessible(true); + // a single-column device gains nothing from a batch read + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(1, 1)); - // a device absent from a device-level time index file is an inconsistency: surface it - try { - method.invoke( - null, - resource, - null, - Collections.emptyList(), - null, - null, - false, - false, - cache, - "target/x.tsfile", - deviceId); - fail("expected CorruptedTsFileException when the device is absent and ignoreNotExists=false"); - } catch (InvocationTargetException e) { - assertTrue(e.getCause() instanceof CorruptedTsFileException); - } + // below, at and above the threshold + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(5, 10)); + assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(6, 10)); + assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10)); - // with a file-level time index (or during recovery) an absent device is expected: skip it - try { - Object result = - method.invoke( - null, - resource, - null, - Collections.emptyList(), - null, - null, - false, - true, - cache, - "target/x.tsfile", - deviceId); - assertNull(result); - } catch (InvocationTargetException e) { - fail("unexpected exception: " + e.getCause()); - } + // a scan can reference measurements that no longer exist, so selected > total is possible + assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(11, 10)); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 67351702f8162..fc2d72d8a393d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -33,6 +33,7 @@ import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.BloomFilter; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; @@ -44,7 +45,9 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -295,6 +298,7 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { try { String filePath = "target/batch-device.tsfile"; IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); timeStatistics.setCount(1); @@ -312,7 +316,7 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { new TimeseriesMetadata( (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS()); - // an empty series (count == 0) must be mapped to null and not cached + // an empty series (count == 0) is mapped to null and cached as the null placeholder TimeseriesMetadata emptyS2 = new TimeseriesMetadata( (byte) 0, @@ -335,7 +339,7 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { Map result = cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, new QueryContext(false, false), false); + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); assertNotNull(result.get("")); assertEquals("", result.get("").getMeasurementId()); @@ -345,7 +349,15 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { assertNotNull(result.get("s1").getChunkMetadataList()); assertNull(result.get("s2")); - // the non-empty entries should now be cached for per-measurement lookups + // a second lookup of cached measurements must be served without touching the reader again + Set cachedSensors = new HashSet<>(Arrays.asList("", "s1")); + Map cachedResult = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, cachedSensors, new QueryContext(false, false), false); + assertNotNull(cachedResult.get("s1")); + Mockito.verify(reader, Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID); + + // the batch read also populated the per-measurement cache TimeseriesMetadata cached = cache.get( filePath, @@ -363,27 +375,66 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { } @Test - public void testGetDeviceTimeSeriesMetadataForAbsentDevice() throws Exception { + public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception { TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); FileReaderManager fileReaderManager = FileReaderManager.getInstance(); TsFileID tsFileID = new TsFileID(); cache.clear(); try { - String filePath = "target/batch-absent-device.tsfile"; + String filePath = "target/batch-absent-device-bloom.tsfile"; IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = Collections.singleton(""); TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) .thenReturn(Collections.emptyList()); - - Map result = - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, new QueryContext(false, false), false); + // the bloom filter proves the device is missing, so the per-column path would skip it too + Mockito.when(reader.readBloomFilter()).thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1)); // an absent device yields an empty map, which loadAlignedTimeSeriesMetadataBatch then // distinguishes from "present but empty time column" - assertTrue(result.isEmpty()); + assertTrue( + cache + .getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false) + .isEmpty()); + } finally { + cache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + try { + String filePath = "target/batch-absent-device-throw.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = Collections.singleton(""); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) + .thenReturn(Collections.emptyList()); + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + bloomFilter.add(deviceID.toString() + "."); + Mockito.when(reader.readBloomFilter()).thenReturn(bloomFilter); + + assertThrows( + IOException.class, + () -> + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false)); + + assertTrue( + cache + .getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true) + .isEmpty()); } finally { cache.clear(); fileReaderManager.closeFileAndRemoveReader(tsFileID); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java index abcfbb7c0b3ef..e4f9b880ab5d5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; import org.apache.iotdb.db.utils.EnvironmentUtils; @@ -62,11 +63,39 @@ public void testBatchScanReadsAlignedData() throws IOException, IllegalPathExcep scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); + TimeSeriesMetadataCache.getInstance().clear(); + long cacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + long diskReadsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get(); + // the batch path must still assemble the same aligned pages: file 1 has 10 points SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); Assert.assertTrue(seriesScanUtil.hasNextPage()); TsBlock tsBlock = seriesScanUtil.nextPage(); Assert.assertEquals(10, tsBlock.getPositionCount()); + + // the device must have been read in one batch: no per-column cache lookups were needed + Assert.assertEquals( + "the batch path must not fall back to per-column cache loads", + cacheHitsBefore, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get()); + // hasNextFile unpacks the first seq file and the first unseq file, one batch read each + Assert.assertEquals( + diskReadsBefore + 2, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get()); } } From 2c9fbe45e3504b14317c8eccdd5ce24a0d0a2465 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 19:04:18 +0800 Subject: [PATCH 12/17] Keep DEBUG queries on the per-column metadata path The batch metadata scan bypasses TimeSeriesMetadataCache.get(..., debug=true) and does not emit equivalent hit/miss diagnostics, so a DEBUG SELECT on a wide table lost its metadata logs. Make the batch gate take the debug and offset-addressed state explicitly so DEBUG queries keep the per-column path, and cover both conditions in the gating unit test. --- .../operator/source/FileLoaderUtils.java | 21 ++++++++++++------- .../operator/source/FileLoaderUtilsTest.java | 18 ++++++++++------ 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index ca34d9f9ba495..7e34046d00634 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -365,9 +365,13 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // For a wide device (a table scan that selects almost all field columns), deserialize the // whole device's TimeSeriesMetadata in a single batch read instead of walking the device // metadata index once per column. External tsfiles address metadata by offset, so they must - // stay on the per-column path. - if (rootMeasurementMetadataIndexNodeOffset == null - && shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColumnCount)) { + // stay on the per-column path; DEBUG queries also stay there so the per-column cache's + // diagnostic logs are preserved. + if (shouldUseDeviceMetadataBatchScan( + valueMeasurementList.size(), + totalFieldColumnCount, + isDebug, + rootMeasurementMetadataIndexNodeOffset != null)) { return loadAlignedTimeSeriesMetadataBatch( resource, alignedPath, @@ -434,10 +438,13 @@ && shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColum } static boolean shouldUseDeviceMetadataBatchScan( - int selectedColumnCount, int totalFieldColumnCount) { - // a single-column device gains nothing from a batch read (one leaf either way), and it would - // bypass the per-column cache's debug logging - return totalFieldColumnCount > 1 + int selectedColumnCount, int totalFieldColumnCount, boolean debug, boolean offsetAddressed) { + // DEBUG queries keep the per-column cache's diagnostic logs; offset-addressed external + // tsfiles cannot be resolved through the table metadata index; a single-column device gains + // nothing from a batch read (one leaf either way) + return !debug + && !offsetAddressed + && totalFieldColumnCount > 1 && (double) selectedColumnCount / totalFieldColumnCount >= DEVICE_METADATA_BATCH_SCAN_RATIO; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java index 92431d1114b8c..2ef0fdd211a4d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java @@ -29,17 +29,23 @@ public class FileLoaderUtilsTest { @Test public void testShouldUseDeviceMetadataBatchScan() { // no field count is known (tree model, external tsfiles, or a table missing from the cache) - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 0)); + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 0, false, false)); // a single-column device gains nothing from a batch read - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(1, 1)); + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(1, 1, false, false)); // below, at and above the threshold - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(5, 10)); - assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(6, 10)); - assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10)); + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(5, 10, false, false)); + assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(6, 10, false, false)); + assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10, false, false)); // a scan can reference measurements that no longer exist, so selected > total is possible - assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(11, 10)); + assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(11, 10, false, false)); + + // DEBUG queries keep the per-column cache's diagnostic logs + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10, true, false)); + + // offset-addressed external tsfiles cannot be resolved through the table metadata index + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10, false, true)); } } From 3fba9804848fd0ce62970eb128e829e992196430 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 19:09:23 +0800 Subject: [PATCH 13/17] Check the bloom filter before the batch device read getDeviceTimeSeriesMetadata read the whole device metadata index and only then discovered the device was absent, while the per-column get() can skip that read by probing the file's bloom filter first. Probe the cached time column entry before the read, keep the same missing-device skip/throw rules, and account the bloom filter cache/disk/time statistics like get(). --- .../buffer/TimeSeriesMetadataCache.java | 57 ++++++++++++++++--- .../buffer/TimeSeriesMetadataCacheTest.java | 18 ++++-- 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 922f2c4e4c959..d1e026a569f2f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -348,9 +348,12 @@ public Map getDeviceTimeSeriesMetadata( boolean ignoreNotExists) throws IOException { long startTime = System.nanoTime(); + long loadBloomFilterTime = 0; boolean externalTsFile = queryContext.isExternalTsFileScan(); LongConsumer timeSeriesMetadataIoSizeRecorder = queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; + LongConsumer bloomFilterIoSizeRecorder = + queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet; boolean cacheHit = true; try { @@ -369,6 +372,14 @@ public Map getDeviceTimeSeriesMetadata( return cached; } cacheHit = false; + long bloomFilterStartTime = System.nanoTime(); + boolean devicePossiblyPresent = + isDevicePossiblyPresent( + filePath, tsFileID, device, queryContext, bloomFilterIoSizeRecorder); + loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; + if (!devicePossiblyPresent) { + return Collections.emptyMap(); + } TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); @@ -377,12 +388,21 @@ public Map getDeviceTimeSeriesMetadata( } } cacheHit = false; + long bloomFilterStartTime = System.nanoTime(); + boolean devicePossiblyPresent = + isDevicePossiblyPresent( + filePath, tsFileID, device, queryContext, bloomFilterIoSizeRecorder); + loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; + if (!devicePossiblyPresent) { + return Collections.emptyMap(); + } TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); return readDeviceTimeSeriesMetadata( reader, tsFileID, device, allSensors, false, ignoreNotExists); } finally { + queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime); if (cacheHit) { queryContext .getQueryStatistics() @@ -391,7 +411,7 @@ public Map getDeviceTimeSeriesMetadata( queryContext .getQueryStatistics() .getLoadTimeSeriesMetadataFromCacheTime() - .getAndAdd(System.nanoTime() - startTime); + .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); } else { queryContext .getQueryStatistics() @@ -400,11 +420,35 @@ public Map getDeviceTimeSeriesMetadata( queryContext .getQueryStatistics() .getLoadTimeSeriesMetadataFromDiskTime() - .getAndAdd(System.nanoTime() - startTime); + .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); } } } + /** + * The time column (keyed by "") is written for every aligned device, so its bloom filter entry is + * a cheap per-file presence probe. A null filter cannot rule the device out. + */ + private boolean isDevicePossiblyPresent( + String filePath, + TsFileID tsFileID, + IDeviceID device, + QueryContext queryContext, + LongConsumer bloomFilterIoSizeRecorder) + throws IOException { + BloomFilter bloomFilter = + BloomFilterCache.getInstance() + .get( + new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID), + false, + bloomFilterIoSizeRecorder, + queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet, + queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet, + false); + return bloomFilter == null + || bloomFilter.contains(device.toString() + IoTDBConstant.PATH_SEPARATOR); + } + /** * @return copies of the cached metadata for every measurement in {@code allSensors}, or null as * soon as one of them is not cached and the device therefore has to be read from disk @@ -438,16 +482,11 @@ private Map readDeviceTimeSeriesMetadata( // metadata list, so readDeviceMetadata (which skips chunk metadata) would NPE at query time. List deviceMetadataList = reader.getDeviceTimeseriesMetadata(device); if (deviceMetadataList.isEmpty()) { - // Match get(): an absent device is only an inconsistency when ignoreNotExists is false and - // the bloom filter does not already prove that the device is missing from this file + // the caller already consulted the bloom filter, so the device could not be ruled out: + // missing metadata is an inconsistency unless ignoreNotExists is set if (ignoreNotExists) { return Collections.emptyMap(); } - BloomFilter bloomFilter = reader.readBloomFilter(); - if (bloomFilter != null - && !bloomFilter.contains(device.toString() + IoTDBConstant.PATH_SEPARATOR)) { - return Collections.emptyMap(); - } throw new IOException( String.format(StorageEngineMessages.DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT, device)); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index fc2d72d8a393d..ef4d2ceb21796 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -292,9 +292,11 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { @Test public void testGetDeviceTimeSeriesMetadata() throws Exception { TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); FileReaderManager fileReaderManager = FileReaderManager.getInstance(); TsFileID tsFileID = new TsFileID(); cache.clear(); + bloomFilterCache.clear(); try { String filePath = "target/batch-device.tsfile"; IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); @@ -370,6 +372,7 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { assertEquals("s1", cached.getMeasurementId()); } finally { cache.clear(); + bloomFilterCache.clear(); fileReaderManager.closeFileAndRemoveReader(tsFileID); } } @@ -377,9 +380,11 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { @Test public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception { TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); FileReaderManager fileReaderManager = FileReaderManager.getInstance(); TsFileID tsFileID = new TsFileID(); cache.clear(); + bloomFilterCache.clear(); try { String filePath = "target/batch-absent-device-bloom.tsfile"; IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); @@ -387,10 +392,9 @@ public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) - .thenReturn(Collections.emptyList()); - // the bloom filter proves the device is missing, so the per-column path would skip it too - Mockito.when(reader.readBloomFilter()).thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1)); + // the bloom filter proves the device is missing, so the device must not be read at all + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))) + .thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1)); // an absent device yields an empty map, which loadAlignedTimeSeriesMetadataBatch then // distinguishes from "present but empty time column" @@ -399,8 +403,10 @@ public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception .getDeviceTimeSeriesMetadata( filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false) .isEmpty()); + Mockito.verify(reader, Mockito.never()).getDeviceTimeseriesMetadata(deviceID); } finally { cache.clear(); + bloomFilterCache.clear(); fileReaderManager.closeFileAndRemoveReader(tsFileID); } } @@ -408,9 +414,11 @@ filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false) @Test public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Exception { TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); FileReaderManager fileReaderManager = FileReaderManager.getInstance(); TsFileID tsFileID = new TsFileID(); cache.clear(); + bloomFilterCache.clear(); try { String filePath = "target/batch-absent-device-throw.tsfile"; IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); @@ -422,7 +430,7 @@ public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Except .thenReturn(Collections.emptyList()); BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); bloomFilter.add(deviceID.toString() + "."); - Mockito.when(reader.readBloomFilter()).thenReturn(bloomFilter); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); assertThrows( IOException.class, From 90a2cb045bfb40ac4d32100a30131642cc3534cb Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 13 Nov 2026 10:37:45 +0800 Subject: [PATCH 14/17] Address wide-scan batch review: partial cache reuse and diagnostics Reuse cached measurements and batch-read only the missing ones, probing the per-measurement bloom filter first so absent columns do not force a whole-device read; the all-or-nothing cache check is gone. Count load statistics per series so they stay comparable with the per-column path, and document the batch read's missing IO-size accounting. Report a dedicated error naming the device and the TsFile when the resource records a device that the file metadata lacks (en + zh), share the load statistics recording between both paths, and pass the external flag through the bloom filter lookup. Only batch when the table has more field columns than the measurement index fan-out, so small devices stay on the per-column path. Add a real 700-column batch-vs-per-column comparison, a 300-column end-to-end batch scan spanning two leaf groups, and gate boundary cases. --- .../iotdb/db/i18n/StorageEngineMessages.java | 3 + .../iotdb/db/i18n/StorageEngineMessages.java | 3 + .../operator/source/FileLoaderUtils.java | 9 +- .../buffer/TimeSeriesMetadataCache.java | 251 +++++++++++------- .../operator/source/FileLoaderUtilsTest.java | 36 ++- .../buffer/TimeSeriesMetadataCacheTest.java | 102 ++++++- .../series/AlignedSeriesScanBatchTest.java | 176 ++++++++---- 7 files changed, 411 insertions(+), 169 deletions(-) diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java index 7133bf09abda5..b78adeed4da52 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1710,6 +1710,9 @@ private StorageEngineMessages() {} public static final String COMPACTION_CROSS_SPACE = "cross"; public static final String DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT = "%s does not exist in the resource file"; + public static final String + EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374 = + "Device %s is recorded in the resource but missing from the metadata of TsFile %s"; public static final String TARGET_FILE_SMALLER_THAN_MAGIC_STRING_AND_VERSION_NUMBER_SIZE_FMT = "target file %s is smaller than magic string and version number size"; public static final String CURRENT_POINT_TIMESTAMP_SHOULD_BE_LATER_FMT = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java index 1603065835bda..a700729990593 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1620,6 +1620,9 @@ private StorageEngineMessages() {} public static final String COMPACTION_CROSS_SPACE = "跨空间"; public static final String DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT = "%s 在资源文件中不存在"; + public static final String + EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374 = + "设备 %s 记录在资源文件中,但在 TsFile %s 的元数据中不存在"; public static final String TARGET_FILE_SMALLER_THAN_MAGIC_STRING_AND_VERSION_NUMBER_SIZE_FMT = "目标文件 %s 小于 magic string 和版本号大小"; public static final String CURRENT_POINT_TIMESTAMP_SHOULD_BE_LATER_FMT = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 7e34046d00634..05e0a681ab2a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -43,6 +43,7 @@ import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.iotdb.db.utils.SchemaUtils; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; @@ -440,11 +441,13 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr static boolean shouldUseDeviceMetadataBatchScan( int selectedColumnCount, int totalFieldColumnCount, boolean debug, boolean offsetAddressed) { // DEBUG queries keep the per-column cache's diagnostic logs; offset-addressed external - // tsfiles cannot be resolved through the table metadata index; a single-column device gains - // nothing from a batch read (one leaf either way) + // tsfiles cannot be resolved through the table metadata index; and when the device's series + // fit in a single measurement leaf group, one per-column lookup already reads them all, so + // batching only pays off once the table has more field columns than the index fan-out return !debug && !offsetAddressed - && totalFieldColumnCount > 1 + && totalFieldColumnCount + > TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode() && (double) selectedColumnCount / totalFieldColumnCount >= DEVICE_METADATA_BATCH_SCAN_RATIO; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index d1e026a569f2f..17193a43d11cd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; +import org.apache.iotdb.db.queryengine.execution.fragment.QueryStatistics; import org.apache.iotdb.db.queryengine.metric.TimeSeriesMetadataCacheMetrics; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; @@ -49,8 +50,11 @@ import java.io.IOException; import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -294,39 +298,35 @@ public TimeseriesMetadata get( return new TimeseriesMetadata(timeseriesMetadata); } } finally { - queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime); - if (cacheHit) { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); - } else { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); - } + long lookupTime = System.nanoTime() - startTime - loadBloomFilterTime; + recordLoadStatistics( + queryContext, + cacheHit ? 1 : 0, + cacheHit ? lookupTime : 0, + cacheHit ? 0 : 1, + cacheHit ? 0 : lookupTime, + loadBloomFilterTime); } } /** - * Load the TimeseriesMetadata of {@code allSensors} of a device, reading the device in a single - * batch when the per-measurement cache does not already hold all of them. This is used by - * wide-table scans (a device with many columns) to avoid walking the device metadata index once - * per column / leaf node. + * Load the TimeseriesMetadata of {@code allSensors} of a device, reusing the per-measurement + * cache and reading the device in a single batch only for the measurements that are still + * missing. This is used by wide-table scans (a device with many columns) to avoid walking the + * device metadata index once per column / leaf group. + * + *

Measurements already cached are returned without touching the file. The remaining ones are + * first checked against the file's cached bloom filter, so columns absent from this file do not + * force a device read; only measurements that cannot be ruled out are loaded by the batch read. * *

The returned map always holds copies of the cached (or freshly read) metadata, so the caller * may apply {@code setModified} and {@code setChunkMetadataLoader} without corrupting cached * entries — matching {@link #get}. * + *

Load statistics are counted per series (not per device read) so that they stay comparable + * with {@link #get}; the bytes read by the batch read cannot be reported because {@link + * TsFileSequenceReader#getDeviceTimeseriesMetadata(IDeviceID)} has no IO-size recorder. + * * @param filePath path of the tsfile * @param tsFileID id of the tsfile * @param device target device @@ -348,129 +348,180 @@ public Map getDeviceTimeSeriesMetadata( boolean ignoreNotExists) throws IOException { long startTime = System.nanoTime(); + long cachePhaseEndTime = 0; long loadBloomFilterTime = 0; + long cacheHitCount = 0; + long diskLookupCount = 0; boolean externalTsFile = queryContext.isExternalTsFileScan(); LongConsumer timeSeriesMetadataIoSizeRecorder = queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; LongConsumer bloomFilterIoSizeRecorder = queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet; - boolean cacheHit = true; + + Map result = new HashMap<>(allSensors.size()); + List missing = new ArrayList<>(allSensors.size()); try { if (CACHE_ENABLE && !externalTsFile) { - Map cached = - getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors); - if (cached != null) { - return cached; + // fast path without locking: reuse everything the per-measurement cache already has + collectCachedMetadata(tsFileID, device, allSensors, result, missing); + cacheHitCount = result.size(); + if (missing.isEmpty()) { + cachePhaseEndTime = System.nanoTime(); + return result; } - // serialize concurrent reads of the same device+file (same lock key as get) so threads - // don't each deserialize the whole device and race on the shared reader + // serialize reads of the same device+file (same lock key as get) so threads don't each + // deserialize the whole device and race on the shared reader synchronized ( devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, WeakReference::new)) { - cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors); - if (cached != null) { - return cached; + List stillMissing = new ArrayList<>(missing.size()); + collectCachedMetadata(tsFileID, device, missing, result, stillMissing); + missing = stillMissing; + cacheHitCount = result.size(); + cachePhaseEndTime = System.nanoTime(); + if (missing.isEmpty()) { + return result; } - cacheHit = false; + + // measurements the cached bloom filter already rejects are known absent in this file, so + // they must not trigger a device read long bloomFilterStartTime = System.nanoTime(); - boolean devicePossiblyPresent = - isDevicePossiblyPresent( - filePath, tsFileID, device, queryContext, bloomFilterIoSizeRecorder); + BloomFilter bloomFilter = + loadBloomFilter(filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, false); + List possiblyPresent = new ArrayList<>(missing.size()); + for (String measurement : missing) { + if (bloomFilterMayContain(bloomFilter, device, measurement)) { + possiblyPresent.add(measurement); + } else { + result.put(measurement, null); + } + } loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; - if (!devicePossiblyPresent) { - return Collections.emptyMap(); + diskLookupCount = missing.size(); + if (possiblyPresent.isEmpty()) { + return result; } + TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); - return readDeviceTimeSeriesMetadata( - reader, tsFileID, device, allSensors, true, ignoreNotExists); + result.putAll( + readDeviceTimeSeriesMetadata( + reader, + filePath, + tsFileID, + device, + new HashSet<>(possiblyPresent), + true, + ignoreNotExists)); + return result; } } - cacheHit = false; + + // cache disabled / external tsfiles: probe the time column first so an absent device does + // not pay for the whole-device read long bloomFilterStartTime = System.nanoTime(); - boolean devicePossiblyPresent = - isDevicePossiblyPresent( - filePath, tsFileID, device, queryContext, bloomFilterIoSizeRecorder); + BloomFilter bloomFilter = + loadBloomFilter( + filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, externalTsFile); loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; - if (!devicePossiblyPresent) { - return Collections.emptyMap(); + diskLookupCount = allSensors.size(); + if (!bloomFilterMayContain(bloomFilter, device, "")) { + for (String measurement : allSensors) { + result.put(measurement, null); + } + return result; } TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); return readDeviceTimeSeriesMetadata( - reader, tsFileID, device, allSensors, false, ignoreNotExists); + reader, filePath, tsFileID, device, allSensors, false, ignoreNotExists); } finally { - queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime); - if (cacheHit) { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); - } else { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); - } + long endTime = System.nanoTime(); + long cacheLookupTime = cachePhaseEndTime == 0 ? 0 : cachePhaseEndTime - startTime; + long diskLookupTime = + cachePhaseEndTime == 0 + ? endTime - startTime - loadBloomFilterTime + : endTime - cachePhaseEndTime - loadBloomFilterTime; + recordLoadStatistics( + queryContext, + cacheHitCount, + cacheLookupTime, + diskLookupCount, + diskLookupTime, + loadBloomFilterTime); } } - /** - * The time column (keyed by "") is written for every aligned device, so its bloom filter entry is - * a cheap per-file presence probe. A null filter cannot rule the device out. - */ - private boolean isDevicePossiblyPresent( + private BloomFilter loadBloomFilter( String filePath, TsFileID tsFileID, - IDeviceID device, QueryContext queryContext, - LongConsumer bloomFilterIoSizeRecorder) + LongConsumer bloomFilterIoSizeRecorder, + boolean externalTsFile) throws IOException { - BloomFilter bloomFilter = - BloomFilterCache.getInstance() - .get( - new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID), - false, - bloomFilterIoSizeRecorder, - queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet, - queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet, - false); + return BloomFilterCache.getInstance() + .get( + new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID), + false, + bloomFilterIoSizeRecorder, + queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet, + queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet, + externalTsFile); + } + + /** A null filter cannot rule the measurement out. */ + private static boolean bloomFilterMayContain( + BloomFilter bloomFilter, IDeviceID device, String measurement) { return bloomFilter == null - || bloomFilter.contains(device.toString() + IoTDBConstant.PATH_SEPARATOR); + || bloomFilter.contains(device.toString() + IoTDBConstant.PATH_SEPARATOR + measurement); + } + + private static void recordLoadStatistics( + QueryContext queryContext, + long cacheHitCount, + long cacheLookupTime, + long diskLookupCount, + long diskLookupTime, + long bloomFilterTime) { + QueryStatistics statistics = queryContext.getQueryStatistics(); + statistics.getLoadBloomFilterTime().getAndAdd(bloomFilterTime); + if (cacheHitCount > 0) { + statistics.getLoadTimeSeriesMetadataFromCacheCount().getAndAdd(cacheHitCount); + statistics.getLoadTimeSeriesMetadataFromCacheTime().getAndAdd(cacheLookupTime); + } + if (diskLookupCount > 0) { + statistics.getLoadTimeSeriesMetadataFromDiskCount().getAndAdd(diskLookupCount); + statistics.getLoadTimeSeriesMetadataFromDiskTime().getAndAdd(diskLookupTime); + } } /** - * @return copies of the cached metadata for every measurement in {@code allSensors}, or null as - * soon as one of them is not cached and the device therefore has to be read from disk + * Resolve cached measurements into {@code resolved} and collect the rest into {@code missing}. */ - private Map getCachedDeviceTimeSeriesMetadata( - TsFileID tsFileID, IDeviceID device, Set allSensors) { - Map result = new HashMap<>(allSensors.size()); - for (String measurement : allSensors) { + private void collectCachedMetadata( + TsFileID tsFileID, + IDeviceID device, + Collection measurements, + Map resolved, + List missing) { + for (String measurement : measurements) { TimeseriesMetadata metadata = lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); if (metadata == null) { - return null; + missing.add(measurement); + } else { + resolved.put( + measurement, + metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new TimeseriesMetadata(metadata)); } - result.put( - measurement, - metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new TimeseriesMetadata(metadata)); } - return result; } private Map readDeviceTimeSeriesMetadata( TsFileSequenceReader reader, + String filePath, TsFileID tsFileID, IDeviceID device, Set allSensors, @@ -488,7 +539,11 @@ private Map readDeviceTimeSeriesMetadata( return Collections.emptyMap(); } throw new IOException( - String.format(StorageEngineMessages.DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT, device)); + String.format( + StorageEngineMessages + .EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374, + device, + filePath)); } Map result = new HashMap<>(allSensors.size()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java index 2ef0fdd211a4d..b62ba8fe7cdc3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.junit.Test; import static org.junit.Assert.assertFalse; @@ -28,24 +29,43 @@ public class FileLoaderUtilsTest { @Test public void testShouldUseDeviceMetadataBatchScan() { + int maxDegreeOfIndexNode = TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode(); + // a table wide enough to span more than one measurement leaf group + int wideTableFieldCount = maxDegreeOfIndexNode * 2; + int threshold = (int) Math.ceil(wideTableFieldCount * 0.6); + // no field count is known (tree model, external tsfiles, or a table missing from the cache) assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 0, false, false)); - // a single-column device gains nothing from a batch read - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(1, 1, false, false)); + // a device whose series fit in a single measurement leaf group gains nothing from batching + assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + maxDegreeOfIndexNode, maxDegreeOfIndexNode, false, false)); // below, at and above the threshold - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(5, 10, false, false)); - assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(6, 10, false, false)); - assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10, false, false)); + assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + threshold - 1, wideTableFieldCount, false, false)); + assertTrue( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + threshold, wideTableFieldCount, false, false)); + assertTrue( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + wideTableFieldCount, wideTableFieldCount, false, false)); // a scan can reference measurements that no longer exist, so selected > total is possible - assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(11, 10, false, false)); + assertTrue( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + wideTableFieldCount + 1, wideTableFieldCount, false, false)); // DEBUG queries keep the per-column cache's diagnostic logs - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10, true, false)); + assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + wideTableFieldCount, wideTableFieldCount, true, false)); // offset-addressed external tsfiles cannot be resolved through the table metadata index - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10, false, true)); + assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + wideTableFieldCount, wideTableFieldCount, false, true)); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index ef4d2ceb21796..90cd54b36b840 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -37,6 +37,7 @@ import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Ignore; import org.junit.Test; @@ -47,6 +48,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -396,13 +398,12 @@ public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))) .thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1)); - // an absent device yields an empty map, which loadAlignedTimeSeriesMetadataBatch then - // distinguishes from "present but empty time column" - assertTrue( - cache - .getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false) - .isEmpty()); + // an absent device must not be read: the filtered-out measurements are returned as null, + // which the caller distinguishes from "present but empty time column" + Map result = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + assertNull(result.get("")); Mockito.verify(reader, Mockito.never()).getDeviceTimeseriesMetadata(deviceID); } finally { cache.clear(); @@ -445,7 +446,94 @@ filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true) .isEmpty()); } finally { cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBatchMetadataMatchesPerColumn() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + int columnCount = 700; + File file = new File("target/batch-compare.tsfile"); + file.delete(); + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.compare.d1"); + List schemas = new ArrayList<>(columnCount); + for (int i = 0; i < columnCount; i++) { + schemas.add(new MeasurementSchema("s" + i, TSDataType.INT32)); + } + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerAlignedTimeseries(deviceID, schemas); + for (long time = 0; time < 3; time++) { + TSRecord record = new TSRecord(deviceID, time); + for (int i = 0; i < columnCount; i++) { + record.addPoint("s" + i, (int) time); + } + writer.writeRecord(record); + } + } + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = file.getAbsolutePath(); + Set allSensors = new HashSet<>(); + for (int i = 0; i < columnCount; i++) { + allSensors.add("s" + i); + } + allSensors.add(""); + allSensors.add("absent"); + + Map perColumn = new HashMap<>(); + for (String measurement : allSensors) { + perColumn.put( + measurement, + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, measurement), + allSensors, + false, + false, + new QueryContext(false, false))); + } + cache.clear(); + + Map batch = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + + assertEquals(allSensors.size(), batch.size()); + for (String measurement : allSensors) { + TimeseriesMetadata expected = perColumn.get(measurement); + TimeseriesMetadata actual = batch.get(measurement); + if (expected == null) { + assertNull(measurement, actual); + continue; + } + assertNotNull(measurement, actual); + assertEquals(measurement, actual.getMeasurementId()); + assertEquals( + measurement, expected.getStatistics().getCount(), actual.getStatistics().getCount()); + assertEquals( + measurement, + expected.getStatistics().getStartTime(), + actual.getStatistics().getStartTime()); + assertEquals( + measurement, + expected.getStatistics().getEndTime(), + actual.getStatistics().getEndTime()); + assertEquals( + measurement, + expected.getChunkMetadataList().size(), + actual.getChunkMetadataList().size()); + } + } finally { + cache.clear(); + bloomFilterCache.clear(); fileReaderManager.closeFileAndRemoveReader(tsFileID); + assertTrue(file.delete()); } } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java index e4f9b880ab5d5..f095bf436af72 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -21,81 +21,151 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.commons.path.AlignedPath; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; +import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; +import org.apache.tsfile.write.chunk.IChunkWriter; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; import org.junit.Assert; import org.junit.Test; +import java.io.File; import java.io.IOException; -import java.util.Arrays; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; +import java.util.List; -/** Verifies the wide-table batch metadata scan (see FileLoaderUtils) reads data correctly. */ -public class AlignedSeriesScanBatchTest extends AbstractAlignedSeriesScanTest { +import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.createChunkWriter; +import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.writeAlignedChunk; + +/** + * Verifies that a real multi-leaf-group device (more field columns than the measurement index + * fan-out) is read correctly through the batch metadata path. + */ +public class AlignedSeriesScanBatchTest { + + private static final String DATABASE = "root.wide_pd"; + private static final IDeviceID DEVICE = + IDeviceID.Factory.DEFAULT_FACTORY.create(DATABASE + ".d1"); @Test - public void testBatchScanReadsAlignedData() throws IOException, IllegalPathException { - AlignedFullPath scanPath = - new AlignedFullPath( - TEST_DEVICE, - Arrays.asList("s1", "s2", "s3"), - Arrays.asList( - new MeasurementSchema("s1", TSDataType.INT32), - new MeasurementSchema("s2", TSDataType.INT32), - new MeasurementSchema("s3", TSDataType.INT32))); + public void testBatchScanReadsWideAlignedDevice() throws IOException, IllegalPathException { + int columnCount = 300; + File file = new File(TestConstant.getTestTsFilePath(DATABASE, 0, 0, 1)); + if (!file.getParentFile().exists()) { + Assert.assertTrue(file.getParentFile().mkdirs()); + } + TsFileResource resource = new TsFileResource(file); + List writtenPaths = new ArrayList<>(columnCount); + List dataTypes = new ArrayList<>(columnCount); + List encodings = new ArrayList<>(columnCount); + List compressionTypes = new ArrayList<>(columnCount); + List measurementNames = new ArrayList<>(columnCount); + List measurementSchemas = new ArrayList<>(columnCount); + + for (int i = 0; i < columnCount; i++) { + String measurement = "s" + i; + measurementNames.add(measurement); + measurementSchemas.add(new MeasurementSchema(measurement, TSDataType.INT32)); + writtenPaths.add(new AlignedPath(DEVICE.toString(), measurement)); + dataTypes.add(TSDataType.INT32); + encodings.add(TSEncoding.PLAIN); + compressionTypes.add(CompressionType.UNCOMPRESSED); + } + + try (TsFileIOWriter writer = new TsFileIOWriter(file)) { + writer.startChunkGroup(DEVICE); + for (IChunkWriter chunkWriter : + createChunkWriter(writtenPaths, dataTypes, encodings, compressionTypes, true)) { + writeAlignedChunk( + (AlignedChunkWriterImpl) chunkWriter, + writer, + Collections.singletonList(new TimeRange(0L, 9L)), + true); + } + writer.endChunkGroup(); + writer.endFile(); + } + resource.updateStartTime(DEVICE, 0); + resource.updateEndTime(DEVICE, 9); + resource.setStatusForTest(TsFileResourceStatus.NORMAL); - SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); - builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); - SeriesScanOptions scanOptions = builder.build(); - // all 3 value columns are selected, so selected/total = 1.0 >= 0.6 and the batch path is taken - scanOptions.setTotalFieldColumnCount(3); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + metadataCache.clear(); + bloomFilterCache.clear(); + try { + long cacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + long diskReadsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get(); - AlignedSeriesScanUtil seriesScanUtil = - new AlignedSeriesScanUtil( - scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); - seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); + AlignedFullPath scanPath = new AlignedFullPath(DEVICE, measurementNames, measurementSchemas); + SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); + builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); + SeriesScanOptions scanOptions = builder.build(); + // all columns are selected and the table is wider than the index fan-out, so batching applies + scanOptions.setTotalFieldColumnCount(columnCount); - TimeSeriesMetadataCache.getInstance().clear(); - long cacheHitsBefore = - EnvironmentUtils.TEST_QUERY_FI_CONTEXT - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .get(); - long diskReadsBefore = - EnvironmentUtils.TEST_QUERY_FI_CONTEXT - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskCount() - .get(); + AlignedSeriesScanUtil seriesScanUtil = + new AlignedSeriesScanUtil( + scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource( + new QueryDataSource(Collections.singletonList(resource), Collections.emptyList())); - // the batch path must still assemble the same aligned pages: file 1 has 10 points - SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); - SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); - Assert.assertTrue(seriesScanUtil.hasNextPage()); - TsBlock tsBlock = seriesScanUtil.nextPage(); - Assert.assertEquals(10, tsBlock.getPositionCount()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); - // the device must have been read in one batch: no per-column cache lookups were needed - Assert.assertEquals( - "the batch path must not fall back to per-column cache loads", - cacheHitsBefore, - EnvironmentUtils.TEST_QUERY_FI_CONTEXT - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .get()); - // hasNextFile unpacks the first seq file and the first unseq file, one batch read each - Assert.assertEquals( - diskReadsBefore + 2, - EnvironmentUtils.TEST_QUERY_FI_CONTEXT - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskCount() - .get()); + // one batch read per device and file: no per-column cache lookups happened + Assert.assertEquals( + "the batch path must not fall back to per-column cache loads", + cacheHitsBefore, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get()); + // statistics are counted per series, like the per-column path (time column + all fields) + Assert.assertEquals( + diskReadsBefore + columnCount + 1, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get()); + } finally { + metadataCache.clear(); + bloomFilterCache.clear(); + FileReaderManager.getInstance().closeFileAndRemoveReader(resource.getTsFileID()); + Assert.assertTrue(file.delete()); + } } } From caf6df1ef1b164b9388a665f1b89874990aded61 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 13 Nov 2026 11:49:41 +0800 Subject: [PATCH 15/17] Refine the batch metadata path after review Skip the device read when the time column is known absent, so a bloom filter false positive on a field cannot force an absent-device read. Fall back to per-column lookups for a small remainder (one or two measurements), which reads a single leaf group instead of the whole device, and count the cold-call cache probe and lock wait in the disk lookup time. Strengthen the wide-scan tests: assert timestamps and every field value, compare a sparse batch scan against the per-column path, cover an all-null projection under ignoreAllNullRows, and add cache regressions for the filtered time column and the single-miss fallback. --- .../buffer/TimeSeriesMetadataCache.java | 52 +++- .../buffer/TimeSeriesMetadataCacheTest.java | 127 +++++++- .../series/AlignedSeriesScanBatchTest.java | 294 ++++++++++++++---- 3 files changed, 403 insertions(+), 70 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 17193a43d11cd..a44bf4016ed01 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -95,6 +95,9 @@ public class TimeSeriesMetadataCache { Collections.synchronizedMap(new WeakHashMap<>()); private static final String SEPARATOR = "$"; + /** A remainder this small is cheaper to load with per-column leaf-group lookups. */ + private static final int MAX_PER_COLUMN_FALLBACK = 2; + private final AtomicLong evictedExistingEntryCount = new AtomicLong(0); private final AtomicLong evictedNonExistingEntryCount = new AtomicLong(0); @@ -352,6 +355,7 @@ public Map getDeviceTimeSeriesMetadata( long loadBloomFilterTime = 0; long cacheHitCount = 0; long diskLookupCount = 0; + boolean delegated = false; boolean externalTsFile = queryContext.isExternalTsFileScan(); LongConsumer timeSeriesMetadataIoSizeRecorder = queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; @@ -402,6 +406,32 @@ public Map getDeviceTimeSeriesMetadata( return result; } + // the time column is always written for an aligned device, so a known-absent time + // column means the file will be skipped anyway; a bloom-filter false positive on a field + // must not force a device read and a possible absent-device error + if (allSensors.contains("") && result.get("") == null && !possiblyPresent.contains("")) { + return result; + } + + // for a small remainder, a per-column lookup reads one leaf group instead of the whole + // device; those lookups record their own statistics + if (possiblyPresent.size() <= MAX_PER_COLUMN_FALLBACK) { + delegated = true; + diskLookupCount = missing.size() - possiblyPresent.size(); + for (String measurement : possiblyPresent) { + result.put( + measurement, + get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), + allSensors, + ignoreNotExists, + false, + queryContext)); + } + return result; + } + TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); @@ -439,11 +469,23 @@ public Map getDeviceTimeSeriesMetadata( reader, filePath, tsFileID, device, allSensors, false, ignoreNotExists); } finally { long endTime = System.nanoTime(); - long cacheLookupTime = cachePhaseEndTime == 0 ? 0 : cachePhaseEndTime - startTime; - long diskLookupTime = - cachePhaseEndTime == 0 - ? endTime - startTime - loadBloomFilterTime - : endTime - cachePhaseEndTime - loadBloomFilterTime; + long cacheLookupTime; + long diskLookupTime; + if (delegated) { + // the per-column fallbacks recorded their own disk time + cacheLookupTime = cachePhaseEndTime == 0 ? 0 : cachePhaseEndTime - startTime; + diskLookupTime = 0; + } else if (cacheHitCount == 0 || cachePhaseEndTime == 0) { + // cold or cache-disabled call: cache probes and the lock wait belong to the disk lookup + cacheLookupTime = 0; + diskLookupTime = endTime - startTime - loadBloomFilterTime; + } else if (diskLookupCount == 0) { + cacheLookupTime = endTime - startTime - loadBloomFilterTime; + diskLookupTime = 0; + } else { + cacheLookupTime = cachePhaseEndTime - startTime; + diskLookupTime = endTime - cachePhaseEndTime - loadBloomFilterTime; + } recordLoadStatistics( queryContext, cacheHitCount, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 90cd54b36b840..32efa93539d31 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -423,7 +423,9 @@ public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Except try { String filePath = "target/batch-absent-device-throw.tsfile"; IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); - Set allSensors = Collections.singleton(""); + // more than MAX_PER_COLUMN_FALLBACK entries so the batch read (not the per-column fallback) + // is used + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); @@ -431,6 +433,8 @@ public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Except .thenReturn(Collections.emptyList()); BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); bloomFilter.add(deviceID.toString() + "."); + bloomFilter.add(deviceID.toString() + ".s1"); + bloomFilter.add(deviceID.toString() + ".s2"); Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); assertThrows( @@ -536,4 +540,125 @@ public void testBatchMetadataMatchesPerColumn() throws Exception { assertTrue(file.delete()); } } + + @Test + public void testAbsentTimeColumnSkipsDeviceRead() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-time-column-filtered.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + // the time column is filtered out, while s1 is a bloom filter false positive + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + bloomFilter.add(deviceID.toString() + ".s1"); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + Map result = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + + assertNull(result.get("")); + assertNull(result.get("s1")); + Mockito.verify(reader, Mockito.never()).getDeviceTimeseriesMetadata(deviceID); + Mockito.verify(reader, Mockito.never()) + .readTimeseriesMetadata( + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.anyBoolean(), + Mockito.any()); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testSingleMissingMeasurementUsesPerColumnFallback() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-single-miss.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + Statistics statistics = Statistics.getStatsByType(TSDataType.INT32); + statistics.setCount(1); + statistics.setStartTime(0); + statistics.setEndTime(10); + TimeseriesMetadata s2 = + new TimeseriesMetadata((byte) 0, 0, "s2", TSDataType.INT32, statistics, new PublicBAOS()); + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + + List deviceMetadata = new ArrayList<>(); + deviceMetadata.add( + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS())); + deviceMetadata.add( + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS())); + deviceMetadata.add(s2); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata); + + Map first = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + assertNotNull(first.get("s2")); + + // evict one measurement: the next call must load only that one, not the whole device + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s2")); + Mockito.when( + reader.readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.eq("s2"), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenReturn(Collections.singletonList(s2)); + + Map second = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + + assertNotNull(second.get("s2")); + Mockito.verify(reader, Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.eq("s2"), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java index f095bf436af72..017180dff82fb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -35,6 +35,7 @@ import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.db.utils.constant.TestConstant; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; @@ -55,62 +56,28 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.createChunkWriter; import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.writeAlignedChunk; /** - * Verifies that a real multi-leaf-group device (more field columns than the measurement index - * fan-out) is read correctly through the batch metadata path. + * Verifies the batch metadata path on a real device wider than one measurement leaf group, and that + * it matches the per-column path for dense, sparse and all-null projections. */ public class AlignedSeriesScanBatchTest { private static final String DATABASE = "root.wide_pd"; private static final IDeviceID DEVICE = IDeviceID.Factory.DEFAULT_FACTORY.create(DATABASE + ".d1"); + private static final int COLUMN_COUNT = 300; + private static final int PRESENT_COLUMN_COUNT = 30; + private static final int ROW_COUNT = 10; @Test - public void testBatchScanReadsWideAlignedDevice() throws IOException, IllegalPathException { - int columnCount = 300; - File file = new File(TestConstant.getTestTsFilePath(DATABASE, 0, 0, 1)); - if (!file.getParentFile().exists()) { - Assert.assertTrue(file.getParentFile().mkdirs()); - } - TsFileResource resource = new TsFileResource(file); - List writtenPaths = new ArrayList<>(columnCount); - List dataTypes = new ArrayList<>(columnCount); - List encodings = new ArrayList<>(columnCount); - List compressionTypes = new ArrayList<>(columnCount); - List measurementNames = new ArrayList<>(columnCount); - List measurementSchemas = new ArrayList<>(columnCount); - - for (int i = 0; i < columnCount; i++) { - String measurement = "s" + i; - measurementNames.add(measurement); - measurementSchemas.add(new MeasurementSchema(measurement, TSDataType.INT32)); - writtenPaths.add(new AlignedPath(DEVICE.toString(), measurement)); - dataTypes.add(TSDataType.INT32); - encodings.add(TSEncoding.PLAIN); - compressionTypes.add(CompressionType.UNCOMPRESSED); - } - - try (TsFileIOWriter writer = new TsFileIOWriter(file)) { - writer.startChunkGroup(DEVICE); - for (IChunkWriter chunkWriter : - createChunkWriter(writtenPaths, dataTypes, encodings, compressionTypes, true)) { - writeAlignedChunk( - (AlignedChunkWriterImpl) chunkWriter, - writer, - Collections.singletonList(new TimeRange(0L, 9L)), - true); - } - writer.endChunkGroup(); - writer.endFile(); - } - resource.updateStartTime(DEVICE, 0); - resource.updateEndTime(DEVICE, 9); - resource.setStatusForTest(TsFileResourceStatus.NORMAL); - + public void testBatchScanReadsWideAlignedDevice() throws Exception { + File file = newFile(1); + TsFileResource resource = writeWideDevice(file, COLUMN_COUNT); TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); metadataCache.clear(); @@ -127,24 +94,20 @@ public void testBatchScanReadsWideAlignedDevice() throws IOException, IllegalPat .getLoadTimeSeriesMetadataFromDiskCount() .get(); - AlignedFullPath scanPath = new AlignedFullPath(DEVICE, measurementNames, measurementSchemas); - SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); - builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); - SeriesScanOptions scanOptions = builder.build(); - // all columns are selected and the table is wider than the index fan-out, so batching applies - scanOptions.setTotalFieldColumnCount(columnCount); - - AlignedSeriesScanUtil seriesScanUtil = - new AlignedSeriesScanUtil( - scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); - seriesScanUtil.initQueryDataSource( - new QueryDataSource(Collections.singletonList(resource), Collections.emptyList())); - - SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); - SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); - Assert.assertTrue(seriesScanUtil.hasNextPage()); - TsBlock tsBlock = seriesScanUtil.nextPage(); - Assert.assertEquals(10, tsBlock.getPositionCount()); + TsBlock block = scan(alignedPath(columnNames(COLUMN_COUNT)), COLUMN_COUNT, resource); + + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals(row, block.getTimeByIndex(row)); + } + for (int column = 0; column < COLUMN_COUNT; column++) { + Column valueColumn = block.getColumn(column); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertFalse("s" + column + " row " + row, valueColumn.isNull(row)); + Assert.assertEquals("s" + column + " row " + row, row, valueColumn.getInt(row)); + } + } // one batch read per device and file: no per-column cache lookups happened Assert.assertEquals( @@ -156,16 +119,219 @@ public void testBatchScanReadsWideAlignedDevice() throws IOException, IllegalPat .get()); // statistics are counted per series, like the per-column path (time column + all fields) Assert.assertEquals( - diskReadsBefore + columnCount + 1, + diskReadsBefore + COLUMN_COUNT + 1, EnvironmentUtils.TEST_QUERY_FI_CONTEXT .getQueryStatistics() .getLoadTimeSeriesMetadataFromDiskCount() .get()); } finally { + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + @Test + public void testSparseBatchScanMatchesPerColumn() throws Exception { + File file = newFile(2); + TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + AlignedFullPath scanPath = alignedPath(columnNames(COLUMN_COUNT)); + try { metadataCache.clear(); bloomFilterCache.clear(); - FileReaderManager.getInstance().closeFileAndRemoveReader(resource.getTsFileID()); - Assert.assertTrue(file.delete()); + long cacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + TsBlock batched = scan(scanPath, COLUMN_COUNT, resource); + long batchedCacheHits = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get() + - cacheHitsBefore; + + metadataCache.clear(); + long perColumnCacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + TsBlock perColumn = scan(scanPath, 0, resource); + long perColumnCacheHits = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get() + - perColumnCacheHitsBefore; + + // the sparse projection is identical to the per-column scan, including every null field + Assert.assertEquals(perColumn.getPositionCount(), batched.getPositionCount()); + Assert.assertEquals(perColumn.getValueColumnCount(), batched.getValueColumnCount()); + for (int row = 0; row < batched.getPositionCount(); row++) { + Assert.assertEquals(perColumn.getTimeByIndex(row), batched.getTimeByIndex(row)); + } + for (int column = 0; column < batched.getValueColumnCount(); column++) { + Column batchedColumn = batched.getColumn(column); + Column perColumnColumn = perColumn.getColumn(column); + boolean present = column < PRESENT_COLUMN_COUNT; + for (int row = 0; row < batched.getPositionCount(); row++) { + Assert.assertEquals( + "s" + column + " row " + row, perColumnColumn.isNull(row), batchedColumn.isNull(row)); + if (present) { + Assert.assertFalse(batchedColumn.isNull(row)); + Assert.assertEquals(row, batchedColumn.getInt(row)); + Assert.assertEquals(perColumnColumn.getInt(row), batchedColumn.getInt(row)); + } else { + Assert.assertTrue(batchedColumn.isNull(row)); + } + } + } + + // the batch scan resolved everything from disk, the per-column scan reused leaf groups + Assert.assertEquals(0, batchedCacheHits); + Assert.assertTrue(perColumnCacheHits > 0); + } finally { + cleanup(file, resource, metadataCache, bloomFilterCache); } } + + @Test + public void testBatchScanSkipsAllNullRows() throws Exception { + File file = newFile(3); + TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + // every selected field is absent from the file: with ignoreAllNullRows the file must be + // skipped by both paths + AlignedFullPath absentPath = alignedPath(absentColumnNames(COLUMN_COUNT)); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(true); + try { + metadataCache.clear(); + bloomFilterCache.clear(); + Assert.assertFalse(hasFile(absentPath, COLUMN_COUNT, resource)); + + metadataCache.clear(); + Assert.assertFalse(hasFile(absentPath, 0, resource)); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + private static TsBlock scan( + AlignedFullPath scanPath, int totalFieldColumnCount, TsFileResource resource) + throws IOException { + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, totalFieldColumnCount, resource); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + return seriesScanUtil.nextPage(); + } + + private static boolean hasFile( + AlignedFullPath scanPath, int totalFieldColumnCount, TsFileResource resource) + throws IOException { + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, totalFieldColumnCount, resource); + while (true) { + Optional hasNextFile = seriesScanUtil.hasNextFile(); + if (hasNextFile.isPresent()) { + return hasNextFile.get(); + } + } + } + + private static AlignedSeriesScanUtil newScanUtil( + AlignedFullPath scanPath, int totalFieldColumnCount, TsFileResource resource) + throws IOException { + SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); + builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); + SeriesScanOptions scanOptions = builder.build(); + scanOptions.setTotalFieldColumnCount(totalFieldColumnCount); + AlignedSeriesScanUtil seriesScanUtil = + new AlignedSeriesScanUtil( + scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource( + new QueryDataSource(Collections.singletonList(resource), Collections.emptyList())); + return seriesScanUtil; + } + + private static AlignedFullPath alignedPath(List measurementNames) { + List schemas = new ArrayList<>(measurementNames.size()); + for (String measurement : measurementNames) { + schemas.add(new MeasurementSchema(measurement, TSDataType.INT32)); + } + return new AlignedFullPath(DEVICE, measurementNames, schemas); + } + + private static List columnNames(int count) { + List names = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + names.add("s" + i); + } + return names; + } + + private static List absentColumnNames(int count) { + List names = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + names.add("absent" + i); + } + return names; + } + + private static File newFile(int version) { + File file = new File(TestConstant.getTestTsFilePath(DATABASE, 0, 0, version)); + if (!file.getParentFile().exists()) { + Assert.assertTrue(file.getParentFile().mkdirs()); + } + return file; + } + + private static TsFileResource writeWideDevice(File file, int presentColumnCount) + throws IOException, IllegalPathException { + TsFileResource resource = new TsFileResource(file); + List writtenPaths = new ArrayList<>(presentColumnCount); + List dataTypes = new ArrayList<>(presentColumnCount); + List encodings = new ArrayList<>(presentColumnCount); + List compressionTypes = new ArrayList<>(presentColumnCount); + for (int i = 0; i < presentColumnCount; i++) { + writtenPaths.add(new AlignedPath(DEVICE.toString(), "s" + i)); + dataTypes.add(TSDataType.INT32); + encodings.add(TSEncoding.PLAIN); + compressionTypes.add(CompressionType.UNCOMPRESSED); + } + try (TsFileIOWriter writer = new TsFileIOWriter(file)) { + writer.startChunkGroup(DEVICE); + for (IChunkWriter chunkWriter : + createChunkWriter(writtenPaths, dataTypes, encodings, compressionTypes, true)) { + writeAlignedChunk( + (AlignedChunkWriterImpl) chunkWriter, + writer, + Collections.singletonList(new TimeRange(0L, ROW_COUNT - 1L)), + true); + } + writer.endChunkGroup(); + writer.endFile(); + } + resource.updateStartTime(DEVICE, 0); + resource.updateEndTime(DEVICE, ROW_COUNT - 1L); + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + return resource; + } + + private static void cleanup( + File file, + TsFileResource resource, + TimeSeriesMetadataCache metadataCache, + BloomFilterCache bloomFilterCache) + throws IOException { + metadataCache.clear(); + bloomFilterCache.clear(); + FileReaderManager.getInstance().closeFileAndRemoveReader(resource.getTsFileID()); + Assert.assertTrue(file.delete()); + } } From e9275076bab9c6478ce02a54f1f66127aa147a1d Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 13 Nov 2026 11:55:39 +0800 Subject: [PATCH 16/17] Document the selective-read limitation of the batch path TsFileSequenceReader is an external apache/tsfile dependency and has no batch API that accepts a measurement set, so the batch path still reads the whole device and filters afterwards. Note the constraint and the existing mitigations (bloom filter, per-column fallback) at the call site; a truly selective reader needs an upstream change. --- .../db/storageengine/buffer/TimeSeriesMetadataCache.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index a44bf4016ed01..f2bdddf890192 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -432,6 +432,12 @@ public Map getDeviceTimeSeriesMetadata( return result; } + // TsFileSequenceReader has no selective batch-read API that accepts a measurement set, + // so this call loads every measurement of the device and filters afterwards. A truly + // selective read requires an upstream apache/tsfile change; the bloom filter and the + // per-column fallback above keep this branch to measurements that are plausibly present. + // The measurement-level readTimeseriesMetadata API is group-selective already and is what + // the per-column path and the fallback use. TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); From f8b9c831371d2abeed6580803c4cad22b0144af9 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 11:35:16 +0800 Subject: [PATCH 17/17] Address the follow-up review of the wide-scan batch path Load the remaining misses through the per-column get() when the cache already holds most of the request: it touches only their leaf groups instead of re-reading the whole device. Replace the GC-sensitive WeakHashMap monitor with fixed lock stripes, and make the returned map complete for allSensors on every early return. Give aggregation table scans a totalFieldColumnCount so they can use the batch path, and make the fan-out gate span two leaf groups (>=), fixing its off-by-one and comment. Tests: production ignoreAllNullRows=false coverage including all-null projections, batch negative-caching parity, copy-on-return, a three-miss eviction case, the multi-leaf premise, and per-chunk offset/statistics comparison. --- .../operator/source/FileLoaderUtils.java | 9 +- .../DataNodeTableOperatorGenerator.java | 4 + .../buffer/TimeSeriesMetadataCache.java | 55 +++++-- .../operator/source/FileLoaderUtilsTest.java | 9 +- .../buffer/TimeSeriesMetadataCacheTest.java | 150 +++++++++++++++++- .../series/AlignedSeriesScanBatchTest.java | 49 +++++- 6 files changed, 246 insertions(+), 30 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 05e0a681ab2a7..aa6df29f1ba67 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -441,13 +441,14 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr static boolean shouldUseDeviceMetadataBatchScan( int selectedColumnCount, int totalFieldColumnCount, boolean debug, boolean offsetAddressed) { // DEBUG queries keep the per-column cache's diagnostic logs; offset-addressed external - // tsfiles cannot be resolved through the table metadata index; and when the device's series - // fit in a single measurement leaf group, one per-column lookup already reads them all, so - // batching only pays off once the table has more field columns than the index fan-out + // tsfiles cannot be resolved through the table metadata index; and a table with at least + // max_degree_of_index_node field columns has a dense device of max_degree+1 series (time column + // included), which spans more than one measurement leaf group. A sparse device may hold fewer + // in-file series; the bloom filter and the cache-aware load keep that case cheap. return !debug && !offsetAddressed && totalFieldColumnCount - > TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode() + >= TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode() && (double) selectedColumnCount / totalFieldColumnCount >= DEVICE_METADATA_BATCH_SCAN_RATIO; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index 47679da161a24..b6a763fd9f0e0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -1783,6 +1783,10 @@ private boolean ensureFieldColumnForTreeNonAlignedDeviceViewScan( node.getPushDownPredicate()); seriesScanOptions.setTTLForTableView(tableViewTTL); seriesScanOptions.setIsTableViewForTreeModel(node instanceof AggregationTreeDeviceViewScanNode); + if (!(node instanceof AggregationTreeDeviceViewScanNode) + && !(node instanceof ExternalTsFileAggregationScanNode)) { + seriesScanOptions.setTotalFieldColumnCount(computeTotalFieldColumnCount(node)); + } Set allSensors = new HashSet<>(measurementColumnNames); allSensors.add(""); // for time column diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index f2bdddf890192..b20d3dd5286dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -49,7 +49,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -59,7 +58,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.WeakHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongConsumer; @@ -91,11 +89,18 @@ public class TimeSeriesMetadataCache { private final AtomicLong entryAverageSize = new AtomicLong(0); - private final Map> devices = - Collections.synchronizedMap(new WeakHashMap<>()); private static final String SEPARATOR = "$"; - /** A remainder this small is cheaper to load with per-column leaf-group lookups. */ + private static final int DEVICE_LOCK_STRIPES = 1024; + private static final Object[] DEVICE_LOCKS = new Object[DEVICE_LOCK_STRIPES]; + + static { + for (int i = 0; i < DEVICE_LOCK_STRIPES; i++) { + DEVICE_LOCKS[i] = new Object(); + } + } + + /** A cold-cache remainder this small is cheaper to load with per-column leaf-group lookups. */ private static final int MAX_PER_COLUMN_FALLBACK = 2; private final AtomicLong evictedExistingEntryCount = new AtomicLong(0); @@ -218,9 +223,7 @@ public TimeseriesMetadata get( } String deviceStringFormat = key.device.toString(); // allow for the parallelism of different devices - synchronized ( - devices.computeIfAbsent( - deviceStringFormat + SEPARATOR + filePath, WeakReference::new)) { + synchronized (deviceLock(deviceStringFormat, filePath)) { // double check timeseriesMetadata = lruCache.getIfPresent(key); if (timeseriesMetadata == null) { @@ -376,8 +379,7 @@ public Map getDeviceTimeSeriesMetadata( } // serialize reads of the same device+file (same lock key as get) so threads don't each // deserialize the whole device and race on the shared reader - synchronized ( - devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, WeakReference::new)) { + synchronized (deviceLock(device.toString(), filePath)) { List stillMissing = new ArrayList<>(missing.size()); collectCachedMetadata(tsFileID, device, missing, result, stillMissing); missing = stillMissing; @@ -410,12 +412,18 @@ public Map getDeviceTimeSeriesMetadata( // column means the file will be skipped anyway; a bloom-filter false positive on a field // must not force a device read and a possible absent-device error if (allSensors.contains("") && result.get("") == null && !possiblyPresent.contains("")) { + for (String measurement : possiblyPresent) { + result.put(measurement, null); + } return result; } - // for a small remainder, a per-column lookup reads one leaf group instead of the whole - // device; those lookups record their own statistics - if (possiblyPresent.size() <= MAX_PER_COLUMN_FALLBACK) { + // when the cache already holds most of the request, the remaining misses are likely + // evicted present columns; loading them through get() touches only their leaf groups + // (later misses in the same group hit the cache) instead of re-reading the whole device. + // Those lookups record their own statistics. + if (possiblyPresent.size() <= MAX_PER_COLUMN_FALLBACK + || (cacheHitCount > 0 && possiblyPresent.size() <= cacheHitCount)) { delegated = true; diskLookupCount = missing.size() - possiblyPresent.size(); for (String measurement : possiblyPresent) { @@ -450,6 +458,9 @@ public Map getDeviceTimeSeriesMetadata( new HashSet<>(possiblyPresent), true, ignoreNotExists)); + for (String measurement : possiblyPresent) { + result.putIfAbsent(measurement, null); + } return result; } } @@ -471,8 +482,13 @@ public Map getDeviceTimeSeriesMetadata( TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); - return readDeviceTimeSeriesMetadata( - reader, filePath, tsFileID, device, allSensors, false, ignoreNotExists); + result.putAll( + readDeviceTimeSeriesMetadata( + reader, filePath, tsFileID, device, allSensors, false, ignoreNotExists)); + for (String measurement : allSensors) { + result.putIfAbsent(measurement, null); + } + return result; } finally { long endTime = System.nanoTime(); long cacheLookupTime; @@ -630,6 +646,15 @@ private Map readDeviceTimeSeriesMetadata( return result; } + /** + * Striped, strongly referenced monitor for one device+file. A WeakHashMap entry can be reclaimed + * by GC between calls, which would let two threads hold "the same" lock simultaneously. + */ + private static Object deviceLock(String deviceString, String filePath) { + return DEVICE_LOCKS[ + Math.floorMod((deviceString + SEPARATOR + filePath).hashCode(), DEVICE_LOCK_STRIPES)]; + } + public double calculateTimeSeriesMetadataHitRatio() { return lruCache.stats().hitRate(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java index b62ba8fe7cdc3..2db7cafcbc6db 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java @@ -37,12 +37,17 @@ public void testShouldUseDeviceMetadataBatchScan() { // no field count is known (tree model, external tsfiles, or a table missing from the cache) assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 0, false, false)); - // a device whose series fit in a single measurement leaf group gains nothing from batching + // a table with maxDegree - 1 fields has at most maxDegree device series (time included), which + // still fit in a single measurement leaf group assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + maxDegreeOfIndexNode - 1, maxDegreeOfIndexNode - 1, false, false)); + // maxDegree fields span maxDegree + 1 device series (time included), i.e. two leaf groups + assertTrue( FileLoaderUtils.shouldUseDeviceMetadataBatchScan( maxDegreeOfIndexNode, maxDegreeOfIndexNode, false, false)); - // below, at and above the threshold + // below, at and above the ratio threshold on a wider table assertFalse( FileLoaderUtils.shouldUseDeviceMetadataBatchScan( threshold - 1, wideTableFieldCount, false, false)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 32efa93539d31..2ebdf4375b35c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -60,6 +60,7 @@ import java.util.function.LongConsumer; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; @@ -372,6 +373,19 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { new QueryContext(false, false)); assertNotNull(cached); assertEquals("s1", cached.getMeasurementId()); + + // the cache hands out copies: mutating a returned entry must not poison later reads + result.get("s1").setModified(true); + TimeseriesMetadata cachedAgain = + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"), + Collections.singleton("s1"), + true, + false, + new QueryContext(false, false)); + assertNotNull(cachedAgain); + assertFalse(cachedAgain.isModified()); } finally { cache.clear(); bloomFilterCache.clear(); @@ -443,11 +457,12 @@ public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Except cache.getDeviceTimeSeriesMetadata( filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false)); - assertTrue( - cache - .getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true) - .isEmpty()); + Map ignored = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true); + assertNull(ignored.get("")); + assertNull(ignored.get("s1")); + assertNull(ignored.get("s2")); } finally { cache.clear(); bloomFilterCache.clear(); @@ -532,6 +547,22 @@ public void testBatchMetadataMatchesPerColumn() throws Exception { measurement, expected.getChunkMetadataList().size(), actual.getChunkMetadataList().size()); + for (int i = 0; i < expected.getChunkMetadataList().size(); i++) { + ChunkMetadata expectedChunk = (ChunkMetadata) expected.getChunkMetadataList().get(i); + ChunkMetadata actualChunk = (ChunkMetadata) actual.getChunkMetadataList().get(i); + assertEquals( + measurement, + expectedChunk.getOffsetOfChunkHeader(), + actualChunk.getOffsetOfChunkHeader()); + assertEquals( + measurement, + expectedChunk.getStatistics().getStartTime(), + actualChunk.getStatistics().getStartTime()); + assertEquals( + measurement, + expectedChunk.getStatistics().getEndTime(), + actualChunk.getStatistics().getEndTime()); + } } } finally { cache.clear(); @@ -661,4 +692,113 @@ public void testSingleMissingMeasurementUsesPerColumnFallback() throws Exception fileReaderManager.closeFileAndRemoveReader(tsFileID); } } + + @Test + public void testBatchIgnoreNotExistsDoesNotCachePlaceholders() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-ignore-not-exists.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) + .thenReturn(Collections.emptyList()); + + Map ignored = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true); + assertNull(ignored.get("")); + assertNull(ignored.get("s1")); + + // ignoreNotExists must not write NULL placeholders: a normal query still has to read + assertThrows( + IOException.class, + () -> + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false)); + Mockito.verify(reader, Mockito.times(2)).getDeviceTimeseriesMetadata(deviceID); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testFewMissingMeasurementsUsePerColumnFallbackAfterEviction() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-few-missing.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2", "s3", "s4", "s5")); + + List deviceMetadata = new ArrayList<>(); + for (String measurement : allSensors) { + TSDataType dataType = measurement.isEmpty() ? TSDataType.INT64 : TSDataType.INT32; + Statistics statistics = Statistics.getStatsByType(dataType); + statistics.setCount(1); + statistics.setStartTime(0); + statistics.setEndTime(10); + deviceMetadata.add( + new TimeseriesMetadata( + (byte) 0, 0, measurement, dataType, statistics, new PublicBAOS())); + } + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata); + + Map first = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + assertNotNull(first.get("s5")); + + // evict three measurements: the cache still holds the majority, so the remaining misses + // must be loaded per column instead of re-reading the whole device + for (String measurement : Arrays.asList("s2", "s3", "s4")) { + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, measurement)); + } + Mockito.when( + reader.readTimeseriesMetadata( + Mockito.any(), + Mockito.isNull(), + Mockito.anyString(), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenReturn(deviceMetadata); + + Map second = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + assertNotNull(second.get("s2")); + assertNotNull(second.get("s3")); + assertNotNull(second.get("s4")); + Mockito.verify(reader, Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadata( + Mockito.any(), + Mockito.isNull(), + Mockito.anyString(), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java index 017180dff82fb..fed93e643a639 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; @@ -76,12 +77,19 @@ public class AlignedSeriesScanBatchTest { @Test public void testBatchScanReadsWideAlignedDevice() throws Exception { + Assert.assertTrue( + "the fixture must span more than one measurement leaf group", + COLUMN_COUNT + 1 > TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode()); File file = newFile(1); TsFileResource resource = writeWideDevice(file, COLUMN_COUNT); TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); metadataCache.clear(); bloomFilterCache.clear(); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + // the batch path is only reachable from table dialect, which forces this flag to false + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); try { long cacheHitsBefore = EnvironmentUtils.TEST_QUERY_FI_CONTEXT @@ -125,6 +133,7 @@ public void testBatchScanReadsWideAlignedDevice() throws Exception { .getLoadTimeSeriesMetadataFromDiskCount() .get()); } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); cleanup(file, resource, metadataCache, bloomFilterCache); } } @@ -136,6 +145,9 @@ public void testSparseBatchScanMatchesPerColumn() throws Exception { TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); AlignedFullPath scanPath = alignedPath(columnNames(COLUMN_COUNT)); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); try { metadataCache.clear(); bloomFilterCache.clear(); @@ -193,23 +205,52 @@ public void testSparseBatchScanMatchesPerColumn() throws Exception { Assert.assertEquals(0, batchedCacheHits); Assert.assertTrue(perColumnCacheHits > 0); } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); cleanup(file, resource, metadataCache, bloomFilterCache); } } @Test - public void testBatchScanSkipsAllNullRows() throws Exception { + public void testAllNullProjectionMatchesPerColumn() throws Exception { File file = newFile(3); TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); - // every selected field is absent from the file: with ignoreAllNullRows the file must be - // skipped by both paths + // every selected field is absent from the file AlignedFullPath absentPath = alignedPath(absentColumnNames(COLUMN_COUNT)); boolean previousIgnoreAllNullRows = EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); - EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(true); try { + // table dialect (the only dialect that reaches the batch path): all-null rows are kept + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + metadataCache.clear(); + bloomFilterCache.clear(); + TsBlock batched = scan(absentPath, COLUMN_COUNT, resource); + Assert.assertEquals(ROW_COUNT, batched.getPositionCount()); + Assert.assertEquals(COLUMN_COUNT, batched.getValueColumnCount()); + for (int column = 0; column < COLUMN_COUNT; column++) { + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertTrue( + "absent" + column + " row " + row, batched.getColumn(column).isNull(row)); + } + } + + metadataCache.clear(); + TsBlock perColumn = scan(absentPath, 0, resource); + Assert.assertEquals(perColumn.getPositionCount(), batched.getPositionCount()); + Assert.assertEquals(perColumn.getValueColumnCount(), batched.getValueColumnCount()); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals(perColumn.getTimeByIndex(row), batched.getTimeByIndex(row)); + } + for (int column = 0; column < COLUMN_COUNT; column++) { + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals( + perColumn.getColumn(column).isNull(row), batched.getColumn(column).isNull(row)); + } + } + + // tree dialect: an all-null projection skips the file in both paths + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(true); metadataCache.clear(); bloomFilterCache.clear(); Assert.assertFalse(hasFile(absentPath, COLUMN_COUNT, resource));