diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java index 7133bf09abda5..b78adeed4da52 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1710,6 +1710,9 @@ private StorageEngineMessages() {} public static final String COMPACTION_CROSS_SPACE = "cross"; public static final String DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT = "%s does not exist in the resource file"; + public static final String + EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374 = + "Device %s is recorded in the resource but missing from the metadata of TsFile %s"; public static final String TARGET_FILE_SMALLER_THAN_MAGIC_STRING_AND_VERSION_NUMBER_SIZE_FMT = "target file %s is smaller than magic string and version number size"; public static final String CURRENT_POINT_TIMESTAMP_SHOULD_BE_LATER_FMT = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java index 1603065835bda..a700729990593 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1620,6 +1620,9 @@ private StorageEngineMessages() {} public static final String COMPACTION_CROSS_SPACE = "跨空间"; public static final String DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT = "%s 在资源文件中不存在"; + public static final String + EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374 = + "设备 %s 记录在资源文件中,但在 TsFile %s 的元数据中不存在"; public static final String TARGET_FILE_SMALLER_THAN_MAGIC_STRING_AND_VERSION_NUMBER_SIZE_FMT = "目标文件 %s 小于 magic string 和版本号大小"; public static final String CURRENT_POINT_TIMESTAMP_SHOULD_BE_LATER_FMT = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java index 331a5a5b267d1..517a5e0718bfc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java @@ -101,7 +101,8 @@ protected AbstractAlignedTimeSeriesMetadata loadTimeSeriesMetadata( scanOptions.getGlobalTimeFilter(), isSeq, ignoreAllNullRows, - null); + null, + scanOptions.getTotalFieldColumnCount()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 4374e974db5bc..aa6df29f1ba67 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -43,6 +43,7 @@ import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.iotdb.db.utils.SchemaUtils; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; @@ -62,6 +63,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -72,6 +74,12 @@ public class FileLoaderUtils { private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET = SeriesScanCostMetricSet.getInstance(); + /** + * Fraction of a table's field columns that a scan must select for it to use the batch + * device-metadata read instead of the per-column path. + */ + private static final double DEVICE_METADATA_BATCH_SCAN_RATIO = 0.6; + private FileLoaderUtils() { // util class } @@ -222,7 +230,8 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata( Filter globalTimeFilter, boolean isSeq, boolean ignoreAllNullRows, - long[] rootMeasurementMetadataIndexNodeOffset) + long[] rootMeasurementMetadataIndexNodeOffset, + int totalFieldColumnCount) throws IOException { final long t1 = System.nanoTime(); boolean loadFromMem = false; @@ -241,7 +250,8 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata( context, globalTimeFilter, ignoreAllNullRows, - rootMeasurementMetadataIndexNodeOffset); + rootMeasurementMetadataIndexNodeOffset, + totalFieldColumnCount); } else { // if the tsfile is unclosed, we just get it directly from TsFileResource loadFromMem = true; alignedTimeSeriesMetadata = @@ -333,7 +343,8 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr FragmentInstanceContext context, Filter globalTimeFilter, boolean ignoreAllNullRows, - long[] rootMeasurementMetadataIndexNodeOffset) + long[] rootMeasurementMetadataIndexNodeOffset, + int totalFieldColumnCount) throws IOException { AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null; // load all the TimeseriesMetadata of vector, the first one is for time column and the @@ -348,13 +359,40 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex // we should not ignore the non-exist of device in TsFileMetadata + boolean ignoreNotExists = + context.ignoreNotExistsDevice() + || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE; + + // For a wide device (a table scan that selects almost all field columns), deserialize the + // whole device's TimeSeriesMetadata in a single batch read instead of walking the device + // metadata index once per column. External tsfiles address metadata by offset, so they must + // stay on the per-column path; DEBUG queries also stay there so the per-column cache's + // diagnostic logs are preserved. + if (shouldUseDeviceMetadataBatchScan( + valueMeasurementList.size(), + totalFieldColumnCount, + isDebug, + rootMeasurementMetadataIndexNodeOffset != null)) { + return loadAlignedTimeSeriesMetadataBatch( + resource, + alignedPath, + valueMeasurementList, + allSensors, + context, + globalTimeFilter, + ignoreAllNullRows, + ignoreNotExists, + cache, + filePath, + deviceId); + } + TimeseriesMetadata timeColumn = cache.get( filePath, new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""), allSensors, - context.ignoreNotExistsDevice() - || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, + ignoreNotExists, isDebug, context, rootMeasurementMetadataIndexNodeOffset); @@ -375,38 +413,111 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr } else { List valueTimeSeriesMetadataList = new ArrayList<>(valueMeasurementList.size()); - // if all the queried aligned sensors does not exist, we will return null - boolean exist = false; for (String measurement : valueMeasurementList) { - TimeseriesMetadata valueColumn = + valueTimeSeriesMetadataList.add( cache.get( filePath, new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement), allSensors, - context.ignoreNotExistsDevice() - || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, + ignoreNotExists, isDebug, context, - rootMeasurementMetadataIndexNodeOffset); - exist = (exist || (valueColumn != null)); - valueTimeSeriesMetadataList.add(valueColumn); - } - if (!ignoreAllNullRows || exist) { - alignedTimeSeriesMetadata = - setModifications( - resource, - timeColumn, - valueTimeSeriesMetadataList, - alignedPath, - context, - globalTimeFilter, - ignoreAllNullRows); + rootMeasurementMetadataIndexNodeOffset)); } + alignedTimeSeriesMetadata = + buildAlignedTimeSeriesMetadata( + resource, + alignedPath, + timeColumn, + valueTimeSeriesMetadataList, + context, + globalTimeFilter, + ignoreAllNullRows); } } return alignedTimeSeriesMetadata; } + static boolean shouldUseDeviceMetadataBatchScan( + int selectedColumnCount, int totalFieldColumnCount, boolean debug, boolean offsetAddressed) { + // DEBUG queries keep the per-column cache's diagnostic logs; offset-addressed external + // tsfiles cannot be resolved through the table metadata index; and a table with at least + // max_degree_of_index_node field columns has a dense device of max_degree+1 series (time column + // included), which spans more than one measurement leaf group. A sparse device may hold fewer + // in-file series; the bloom filter and the cache-aware load keep that case cheap. + return !debug + && !offsetAddressed + && totalFieldColumnCount + >= TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode() + && (double) selectedColumnCount / totalFieldColumnCount >= DEVICE_METADATA_BATCH_SCAN_RATIO; + } + + private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBatch( + TsFileResource resource, + AlignedFullPath alignedPath, + List valueMeasurementList, + Set allSensors, + FragmentInstanceContext context, + Filter globalTimeFilter, + boolean ignoreAllNullRows, + boolean ignoreNotExists, + TimeSeriesMetadataCache cache, + String filePath, + IDeviceID deviceId) + throws IOException { + Map deviceMetadata = + cache.getDeviceTimeSeriesMetadata( + filePath, resource.getTsFileID(), deviceId, allSensors, context, ignoreNotExists); + if (deviceMetadata.isEmpty()) { + // the cache applies the same bloom-filter / ignoreNotExists rules as the per-column get(), + // so an empty map means the device is expected to be absent from this file + return null; + } + TimeseriesMetadata timeColumn = deviceMetadata.get(""); + if (timeColumn == null) { + return null; + } + List valueTimeSeriesMetadataList = + new ArrayList<>(valueMeasurementList.size()); + for (String measurement : valueMeasurementList) { + valueTimeSeriesMetadataList.add(deviceMetadata.get(measurement)); + } + return buildAlignedTimeSeriesMetadata( + resource, + alignedPath, + timeColumn, + valueTimeSeriesMetadataList, + context, + globalTimeFilter, + ignoreAllNullRows); + } + + private static AbstractAlignedTimeSeriesMetadata buildAlignedTimeSeriesMetadata( + TsFileResource resource, + AlignedFullPath alignedPath, + TimeseriesMetadata timeColumn, + List valueTimeSeriesMetadataList, + FragmentInstanceContext context, + Filter globalTimeFilter, + boolean ignoreAllNullRows) { + // if all the queried aligned sensors does not exist, we will return null + boolean exist = false; + for (TimeseriesMetadata valueColumn : valueTimeSeriesMetadataList) { + exist = (exist || (valueColumn != null)); + } + if (!ignoreAllNullRows || exist) { + return setModifications( + resource, + timeColumn, + valueTimeSeriesMetadataList, + alignedPath, + context, + globalTimeFilter, + ignoreAllNullRows); + } + return null; + } + private static AbstractAlignedTimeSeriesMetadata setModifications( TsFileResource resource, TimeseriesMetadata timeColumnMetadata, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java index e907218fe8a7b..61a2832e66b40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java @@ -83,7 +83,8 @@ static AbstractAlignedTimeSeriesMetadata loadTimeSeriesMetadata( globalTimeFilter, resource.isSeq(), context.isIgnoreAllNullRows(), - new long[] {currentDeviceOffset.getStartOffset(), currentDeviceOffset.getEndOffset()}); + new long[] {currentDeviceOffset.getStartOffset(), currentDeviceOffset.getEndOffset()}, + 0); } @FunctionalInterface diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index 01584fe230776..b6a763fd9f0e0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -1185,6 +1185,9 @@ private void addSource( topKRuntimeFilter); seriesScanOptions.setTTLForTableView(viewTTL); seriesScanOptions.setIsTableViewForTreeModel(node instanceof TreeDeviceViewScanNode); + if (!(node instanceof TreeDeviceViewScanNode) && !(node instanceof ExternalTsFileScanNode)) { + seriesScanOptions.setTotalFieldColumnCount(computeTotalFieldColumnCount(node)); + } OperatorContext operatorContext = addOperatorContext(context, node.getPlanNodeId(), className); @@ -1222,6 +1225,17 @@ private BatchDeviceEntrySource createDeviceEntrySource(DeviceTableScanNode node) .orElseGet(() -> new InMemoryDeviceEntrySource(node.getDeviceEntries())); } + private int computeTotalFieldColumnCount(DeviceTableScanNode node) { + QualifiedObjectName qualifiedObjectName = node.getQualifiedObjectName(); + TsTable tsTable = + DataNodeTableCache.getInstance() + .getTableInCache( + qualifiedObjectName.getDatabaseName(), qualifiedObjectName.getObjectName()); + // the numerator only contains FIELD columns, so the denominator must be the field count too; + // getColumnNum() would also count tag, attribute and time columns + return tsTable == null ? 0 : tsTable.getFieldNum(); + } + private BatchDeviceEntrySource createAggregationDeviceEntrySource(AggregationTableScanNode node) { if (node.getDeviceEntryCount() == 0 && node.getGroupingKeys().isEmpty()) { return new InMemoryDeviceEntrySource(Collections.singletonList(null)); @@ -1769,6 +1783,10 @@ private boolean ensureFieldColumnForTreeNonAlignedDeviceViewScan( node.getPushDownPredicate()); seriesScanOptions.setTTLForTableView(tableViewTTL); seriesScanOptions.setIsTableViewForTreeModel(node instanceof AggregationTreeDeviceViewScanNode); + if (!(node instanceof AggregationTreeDeviceViewScanNode) + && !(node instanceof ExternalTsFileAggregationScanNode)) { + seriesScanOptions.setTotalFieldColumnCount(computeTotalFieldColumnCount(node)); + } Set allSensors = new HashSet<>(measurementColumnNames); allSensors.add(""); // for time column diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java index 4d40bbd32c4d4..2a173e3b09e0f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java @@ -56,6 +56,7 @@ public class SeriesScanOptions implements Accountable { private PaginationController paginationController; private boolean isTableViewForTreeModel; private long ttlForTableView = Long.MAX_VALUE; + private int totalFieldColumnCount = 0; private final TopKRuntimeFilter topKRuntimeFilter; private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance( @@ -178,6 +179,14 @@ public void setIsTableViewForTreeModel(boolean isTableViewForTreeModel) { this.isTableViewForTreeModel = isTableViewForTreeModel; } + public int getTotalFieldColumnCount() { + return totalFieldColumnCount; + } + + public void setTotalFieldColumnCount(int totalFieldColumnCount) { + this.totalFieldColumnCount = totalFieldColumnCount; + } + /** * pushLimitToEachDevice==false means that all devices return total limit rows. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 68457df0ebbcd..b20d3dd5286dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; +import org.apache.iotdb.db.queryengine.execution.fragment.QueryStatistics; import org.apache.iotdb.db.queryengine.metric.TimeSeriesMetadataCacheMetrics; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; @@ -48,13 +49,15 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.WeakHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongConsumer; @@ -86,10 +89,20 @@ public class TimeSeriesMetadataCache { private final AtomicLong entryAverageSize = new AtomicLong(0); - private final Map> devices = - Collections.synchronizedMap(new WeakHashMap<>()); private static final String SEPARATOR = "$"; + private static final int DEVICE_LOCK_STRIPES = 1024; + private static final Object[] DEVICE_LOCKS = new Object[DEVICE_LOCK_STRIPES]; + + static { + for (int i = 0; i < DEVICE_LOCK_STRIPES; i++) { + DEVICE_LOCKS[i] = new Object(); + } + } + + /** A cold-cache remainder this small is cheaper to load with per-column leaf-group lookups. */ + private static final int MAX_PER_COLUMN_FALLBACK = 2; + private final AtomicLong evictedExistingEntryCount = new AtomicLong(0); private final AtomicLong evictedNonExistingEntryCount = new AtomicLong(0); @@ -210,9 +223,7 @@ public TimeseriesMetadata get( } String deviceStringFormat = key.device.toString(); // allow for the parallelism of different devices - synchronized ( - devices.computeIfAbsent( - deviceStringFormat + SEPARATOR + filePath, WeakReference::new)) { + synchronized (deviceLock(deviceStringFormat, filePath)) { // double check timeseriesMetadata = lruCache.getIfPresent(key); if (timeseriesMetadata == null) { @@ -293,27 +304,355 @@ public TimeseriesMetadata get( return new TimeseriesMetadata(timeseriesMetadata); } } finally { - queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime); - if (cacheHit) { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); + long lookupTime = System.nanoTime() - startTime - loadBloomFilterTime; + recordLoadStatistics( + queryContext, + cacheHit ? 1 : 0, + cacheHit ? lookupTime : 0, + cacheHit ? 0 : 1, + cacheHit ? 0 : lookupTime, + loadBloomFilterTime); + } + } + + /** + * Load the TimeseriesMetadata of {@code allSensors} of a device, reusing the per-measurement + * cache and reading the device in a single batch only for the measurements that are still + * missing. This is used by wide-table scans (a device with many columns) to avoid walking the + * device metadata index once per column / leaf group. + * + *

Measurements already cached are returned without touching the file. The remaining ones are + * first checked against the file's cached bloom filter, so columns absent from this file do not + * force a device read; only measurements that cannot be ruled out are loaded by the batch read. + * + *

The returned map always holds copies of the cached (or freshly read) metadata, so the caller + * may apply {@code setModified} and {@code setChunkMetadataLoader} without corrupting cached + * entries — matching {@link #get}. + * + *

Load statistics are counted per series (not per device read) so that they stay comparable + * with {@link #get}; the bytes read by the batch read cannot be reported because {@link + * TsFileSequenceReader#getDeviceTimeseriesMetadata(IDeviceID)} has no IO-size recorder. + * + * @param filePath path of the tsfile + * @param tsFileID id of the tsfile + * @param device target device + * @param allSensors measurements the caller needs, including the time column keyed by "" + * @param queryContext query context used for metrics and to detect external tsfiles + * @param ignoreNotExists whether a device missing from a file that claims a device-level time + * index should be skipped instead of reported as an inconsistency + * @return map from measurementId (including the time column, keyed by "") to TimeseriesMetadata; + * entries whose series is empty (count == 0) are mapped to null + * @throws IOException if the metadata cannot be read from disk or the device is unexpectedly + * absent + */ + public Map getDeviceTimeSeriesMetadata( + String filePath, + TsFileID tsFileID, + IDeviceID device, + Set allSensors, + QueryContext queryContext, + boolean ignoreNotExists) + throws IOException { + long startTime = System.nanoTime(); + long cachePhaseEndTime = 0; + long loadBloomFilterTime = 0; + long cacheHitCount = 0; + long diskLookupCount = 0; + boolean delegated = false; + boolean externalTsFile = queryContext.isExternalTsFileScan(); + LongConsumer timeSeriesMetadataIoSizeRecorder = + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; + LongConsumer bloomFilterIoSizeRecorder = + queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet; + + Map result = new HashMap<>(allSensors.size()); + List missing = new ArrayList<>(allSensors.size()); + + try { + if (CACHE_ENABLE && !externalTsFile) { + // fast path without locking: reuse everything the per-measurement cache already has + collectCachedMetadata(tsFileID, device, allSensors, result, missing); + cacheHitCount = result.size(); + if (missing.isEmpty()) { + cachePhaseEndTime = System.nanoTime(); + return result; + } + // serialize reads of the same device+file (same lock key as get) so threads don't each + // deserialize the whole device and race on the shared reader + synchronized (deviceLock(device.toString(), filePath)) { + List stillMissing = new ArrayList<>(missing.size()); + collectCachedMetadata(tsFileID, device, missing, result, stillMissing); + missing = stillMissing; + cacheHitCount = result.size(); + cachePhaseEndTime = System.nanoTime(); + if (missing.isEmpty()) { + return result; + } + + // measurements the cached bloom filter already rejects are known absent in this file, so + // they must not trigger a device read + long bloomFilterStartTime = System.nanoTime(); + BloomFilter bloomFilter = + loadBloomFilter(filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, false); + List possiblyPresent = new ArrayList<>(missing.size()); + for (String measurement : missing) { + if (bloomFilterMayContain(bloomFilter, device, measurement)) { + possiblyPresent.add(measurement); + } else { + result.put(measurement, null); + } + } + loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; + diskLookupCount = missing.size(); + if (possiblyPresent.isEmpty()) { + return result; + } + + // the time column is always written for an aligned device, so a known-absent time + // column means the file will be skipped anyway; a bloom-filter false positive on a field + // must not force a device read and a possible absent-device error + if (allSensors.contains("") && result.get("") == null && !possiblyPresent.contains("")) { + for (String measurement : possiblyPresent) { + result.put(measurement, null); + } + return result; + } + + // when the cache already holds most of the request, the remaining misses are likely + // evicted present columns; loading them through get() touches only their leaf groups + // (later misses in the same group hit the cache) instead of re-reading the whole device. + // Those lookups record their own statistics. + if (possiblyPresent.size() <= MAX_PER_COLUMN_FALLBACK + || (cacheHitCount > 0 && possiblyPresent.size() <= cacheHitCount)) { + delegated = true; + diskLookupCount = missing.size() - possiblyPresent.size(); + for (String measurement : possiblyPresent) { + result.put( + measurement, + get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), + allSensors, + ignoreNotExists, + false, + queryContext)); + } + return result; + } + + // TsFileSequenceReader has no selective batch-read API that accepts a measurement set, + // so this call loads every measurement of the device and filters afterwards. A truly + // selective read requires an upstream apache/tsfile change; the bloom filter and the + // per-column fallback above keep this branch to measurements that are plausibly present. + // The measurement-level readTimeseriesMetadata API is group-selective already and is what + // the per-column path and the fallback use. + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); + result.putAll( + readDeviceTimeSeriesMetadata( + reader, + filePath, + tsFileID, + device, + new HashSet<>(possiblyPresent), + true, + ignoreNotExists)); + for (String measurement : possiblyPresent) { + result.putIfAbsent(measurement, null); + } + return result; + } + } + + // cache disabled / external tsfiles: probe the time column first so an absent device does + // not pay for the whole-device read + long bloomFilterStartTime = System.nanoTime(); + BloomFilter bloomFilter = + loadBloomFilter( + filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, externalTsFile); + loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; + diskLookupCount = allSensors.size(); + if (!bloomFilterMayContain(bloomFilter, device, "")) { + for (String measurement : allSensors) { + result.put(measurement, null); + } + return result; + } + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); + result.putAll( + readDeviceTimeSeriesMetadata( + reader, filePath, tsFileID, device, allSensors, false, ignoreNotExists)); + for (String measurement : allSensors) { + result.putIfAbsent(measurement, null); + } + return result; + } finally { + long endTime = System.nanoTime(); + long cacheLookupTime; + long diskLookupTime; + if (delegated) { + // the per-column fallbacks recorded their own disk time + cacheLookupTime = cachePhaseEndTime == 0 ? 0 : cachePhaseEndTime - startTime; + diskLookupTime = 0; + } else if (cacheHitCount == 0 || cachePhaseEndTime == 0) { + // cold or cache-disabled call: cache probes and the lock wait belong to the disk lookup + cacheLookupTime = 0; + diskLookupTime = endTime - startTime - loadBloomFilterTime; + } else if (diskLookupCount == 0) { + cacheLookupTime = endTime - startTime - loadBloomFilterTime; + diskLookupTime = 0; } else { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); + cacheLookupTime = cachePhaseEndTime - startTime; + diskLookupTime = endTime - cachePhaseEndTime - loadBloomFilterTime; } + recordLoadStatistics( + queryContext, + cacheHitCount, + cacheLookupTime, + diskLookupCount, + diskLookupTime, + loadBloomFilterTime); + } + } + + private BloomFilter loadBloomFilter( + String filePath, + TsFileID tsFileID, + QueryContext queryContext, + LongConsumer bloomFilterIoSizeRecorder, + boolean externalTsFile) + throws IOException { + return BloomFilterCache.getInstance() + .get( + new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID), + false, + bloomFilterIoSizeRecorder, + queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet, + queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet, + externalTsFile); + } + + /** A null filter cannot rule the measurement out. */ + private static boolean bloomFilterMayContain( + BloomFilter bloomFilter, IDeviceID device, String measurement) { + return bloomFilter == null + || bloomFilter.contains(device.toString() + IoTDBConstant.PATH_SEPARATOR + measurement); + } + + private static void recordLoadStatistics( + QueryContext queryContext, + long cacheHitCount, + long cacheLookupTime, + long diskLookupCount, + long diskLookupTime, + long bloomFilterTime) { + QueryStatistics statistics = queryContext.getQueryStatistics(); + statistics.getLoadBloomFilterTime().getAndAdd(bloomFilterTime); + if (cacheHitCount > 0) { + statistics.getLoadTimeSeriesMetadataFromCacheCount().getAndAdd(cacheHitCount); + statistics.getLoadTimeSeriesMetadataFromCacheTime().getAndAdd(cacheLookupTime); } + if (diskLookupCount > 0) { + statistics.getLoadTimeSeriesMetadataFromDiskCount().getAndAdd(diskLookupCount); + statistics.getLoadTimeSeriesMetadataFromDiskTime().getAndAdd(diskLookupTime); + } + } + + /** + * Resolve cached measurements into {@code resolved} and collect the rest into {@code missing}. + */ + private void collectCachedMetadata( + TsFileID tsFileID, + IDeviceID device, + Collection measurements, + Map resolved, + List missing) { + for (String measurement : measurements) { + TimeseriesMetadata metadata = + lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); + if (metadata == null) { + missing.add(measurement); + } else { + resolved.put( + measurement, + metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new TimeseriesMetadata(metadata)); + } + } + } + + private Map readDeviceTimeSeriesMetadata( + TsFileSequenceReader reader, + String filePath, + TsFileID tsFileID, + IDeviceID device, + Set allSensors, + boolean cacheEnabled, + boolean ignoreNotExists) + throws IOException { + // Use the chunk-metadata-including variant: the returned TimeseriesMetadata are passed to + // setModifications and then to DiskAlignedChunkMetadataLoader, which dereferences the chunk + // metadata list, so readDeviceMetadata (which skips chunk metadata) would NPE at query time. + List deviceMetadataList = reader.getDeviceTimeseriesMetadata(device); + if (deviceMetadataList.isEmpty()) { + // the caller already consulted the bloom filter, so the device could not be ruled out: + // missing metadata is an inconsistency unless ignoreNotExists is set + if (ignoreNotExists) { + return Collections.emptyMap(); + } + throw new IOException( + String.format( + StorageEngineMessages + .EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374, + device, + filePath)); + } + + Map result = new HashMap<>(allSensors.size()); + for (TimeseriesMetadata metadata : deviceMetadataList) { + String measurementId = metadata.getMeasurementId(); + if (!allSensors.contains(measurementId)) { + continue; + } + if (metadata.getStatistics().getCount() == 0) { + result.put(measurementId, null); + if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { + lruCache.put( + new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), + NULL_EXISTS_CACHE_PLACE_HOLDER); + } + continue; + } + if (cacheEnabled) { + lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), metadata); + } + result.put(measurementId, new TimeseriesMetadata(metadata)); + } + // remember selected measurements that do not exist in this file, so a later lookup of the same + // device can be served from the cache instead of reading the device again + for (String measurement : allSensors) { + if (result.containsKey(measurement)) { + continue; + } + result.put(measurement, null); + if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { + lruCache.put( + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), + NULL_EXISTS_CACHE_PLACE_HOLDER); + } + } + return result; + } + + /** + * Striped, strongly referenced monitor for one device+file. A WeakHashMap entry can be reclaimed + * by GC between calls, which would let two threads hold "the same" lock simultaneously. + */ + private static Object deviceLock(String deviceString, String filePath) { + return DEVICE_LOCKS[ + Math.floorMod((deviceString + SEPARATOR + filePath).hashCode(), DEVICE_LOCK_STRIPES)]; } public double calculateTimeSeriesMetadataHitRatio() { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java new file mode 100644 index 0000000000000..2db7cafcbc6db --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source; + +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class FileLoaderUtilsTest { + + @Test + public void testShouldUseDeviceMetadataBatchScan() { + int maxDegreeOfIndexNode = TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode(); + // a table wide enough to span more than one measurement leaf group + int wideTableFieldCount = maxDegreeOfIndexNode * 2; + int threshold = (int) Math.ceil(wideTableFieldCount * 0.6); + + // no field count is known (tree model, external tsfiles, or a table missing from the cache) + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 0, false, false)); + + // a table with maxDegree - 1 fields has at most maxDegree device series (time included), which + // still fit in a single measurement leaf group + assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + maxDegreeOfIndexNode - 1, maxDegreeOfIndexNode - 1, false, false)); + // maxDegree fields span maxDegree + 1 device series (time included), i.e. two leaf groups + assertTrue( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + maxDegreeOfIndexNode, maxDegreeOfIndexNode, false, false)); + + // below, at and above the ratio threshold on a wider table + assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + threshold - 1, wideTableFieldCount, false, false)); + assertTrue( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + threshold, wideTableFieldCount, false, false)); + assertTrue( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + wideTableFieldCount, wideTableFieldCount, false, false)); + + // a scan can reference measurements that no longer exist, so selected > total is possible + assertTrue( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + wideTableFieldCount + 1, wideTableFieldCount, false, false)); + + // DEBUG queries keep the per-column cache's diagnostic logs + assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + wideTableFieldCount, wideTableFieldCount, true, false)); + + // offset-addressed external tsfiles cannot be resolved through the table metadata index + assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + wideTableFieldCount, wideTableFieldCount, false, true)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 87526eb594b9f..2ebdf4375b35c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -27,12 +27,17 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.BloomFilter; +import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Ignore; import org.junit.Test; @@ -41,8 +46,12 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -51,9 +60,11 @@ import java.util.function.LongConsumer; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; public class TimeSeriesMetadataCacheTest { @@ -280,4 +291,514 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { .setMayCacheNonExistSeries(mayCacheNonExistSeries); } } + + @Test + public void testGetDeviceTimeSeriesMetadata() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-device.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + TimeseriesMetadata timeColumn = + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS()); + + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + TimeseriesMetadata s1 = + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS()); + + // an empty series (count == 0) is mapped to null and cached as the null placeholder + TimeseriesMetadata emptyS2 = + new TimeseriesMetadata( + (byte) 0, + 0, + "s2", + TSDataType.INT32, + Statistics.getStatsByType(TSDataType.INT32), + new PublicBAOS()); + + List deviceMetadata = new ArrayList<>(); + deviceMetadata.add(timeColumn); + deviceMetadata.add(s1); + deviceMetadata.add(emptyS2); + // the batch read must carry chunk metadata, otherwise the downstream chunk loader NPEs + s1.setChunkMetadataList(Collections.singletonList(new ChunkMetadata())); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata); + + Map result = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + + assertNotNull(result.get("")); + assertEquals("", result.get("").getMeasurementId()); + assertNotNull(result.get("s1")); + assertEquals("s1", result.get("s1").getMeasurementId()); + // chunk metadata must flow through to the result (and the cached copy) unchanged + assertNotNull(result.get("s1").getChunkMetadataList()); + assertNull(result.get("s2")); + + // a second lookup of cached measurements must be served without touching the reader again + Set cachedSensors = new HashSet<>(Arrays.asList("", "s1")); + Map cachedResult = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, cachedSensors, new QueryContext(false, false), false); + assertNotNull(cachedResult.get("s1")); + Mockito.verify(reader, Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID); + + // the batch read also populated the per-measurement cache + TimeseriesMetadata cached = + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"), + Collections.singleton("s1"), + true, + false, + new QueryContext(false, false)); + assertNotNull(cached); + assertEquals("s1", cached.getMeasurementId()); + + // the cache hands out copies: mutating a returned entry must not poison later reads + result.get("s1").setModified(true); + TimeseriesMetadata cachedAgain = + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"), + Collections.singleton("s1"), + true, + false, + new QueryContext(false, false)); + assertNotNull(cachedAgain); + assertFalse(cachedAgain.isModified()); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-absent-device-bloom.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = Collections.singleton(""); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + // the bloom filter proves the device is missing, so the device must not be read at all + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))) + .thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1)); + + // an absent device must not be read: the filtered-out measurements are returned as null, + // which the caller distinguishes from "present but empty time column" + Map result = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + assertNull(result.get("")); + Mockito.verify(reader, Mockito.never()).getDeviceTimeseriesMetadata(deviceID); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-absent-device-throw.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + // more than MAX_PER_COLUMN_FALLBACK entries so the batch read (not the per-column fallback) + // is used + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) + .thenReturn(Collections.emptyList()); + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + bloomFilter.add(deviceID.toString() + "."); + bloomFilter.add(deviceID.toString() + ".s1"); + bloomFilter.add(deviceID.toString() + ".s2"); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + assertThrows( + IOException.class, + () -> + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false)); + + Map ignored = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true); + assertNull(ignored.get("")); + assertNull(ignored.get("s1")); + assertNull(ignored.get("s2")); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBatchMetadataMatchesPerColumn() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + int columnCount = 700; + File file = new File("target/batch-compare.tsfile"); + file.delete(); + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.compare.d1"); + List schemas = new ArrayList<>(columnCount); + for (int i = 0; i < columnCount; i++) { + schemas.add(new MeasurementSchema("s" + i, TSDataType.INT32)); + } + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerAlignedTimeseries(deviceID, schemas); + for (long time = 0; time < 3; time++) { + TSRecord record = new TSRecord(deviceID, time); + for (int i = 0; i < columnCount; i++) { + record.addPoint("s" + i, (int) time); + } + writer.writeRecord(record); + } + } + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = file.getAbsolutePath(); + Set allSensors = new HashSet<>(); + for (int i = 0; i < columnCount; i++) { + allSensors.add("s" + i); + } + allSensors.add(""); + allSensors.add("absent"); + + Map perColumn = new HashMap<>(); + for (String measurement : allSensors) { + perColumn.put( + measurement, + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, measurement), + allSensors, + false, + false, + new QueryContext(false, false))); + } + cache.clear(); + + Map batch = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + + assertEquals(allSensors.size(), batch.size()); + for (String measurement : allSensors) { + TimeseriesMetadata expected = perColumn.get(measurement); + TimeseriesMetadata actual = batch.get(measurement); + if (expected == null) { + assertNull(measurement, actual); + continue; + } + assertNotNull(measurement, actual); + assertEquals(measurement, actual.getMeasurementId()); + assertEquals( + measurement, expected.getStatistics().getCount(), actual.getStatistics().getCount()); + assertEquals( + measurement, + expected.getStatistics().getStartTime(), + actual.getStatistics().getStartTime()); + assertEquals( + measurement, + expected.getStatistics().getEndTime(), + actual.getStatistics().getEndTime()); + assertEquals( + measurement, + expected.getChunkMetadataList().size(), + actual.getChunkMetadataList().size()); + for (int i = 0; i < expected.getChunkMetadataList().size(); i++) { + ChunkMetadata expectedChunk = (ChunkMetadata) expected.getChunkMetadataList().get(i); + ChunkMetadata actualChunk = (ChunkMetadata) actual.getChunkMetadataList().get(i); + assertEquals( + measurement, + expectedChunk.getOffsetOfChunkHeader(), + actualChunk.getOffsetOfChunkHeader()); + assertEquals( + measurement, + expectedChunk.getStatistics().getStartTime(), + actualChunk.getStatistics().getStartTime()); + assertEquals( + measurement, + expectedChunk.getStatistics().getEndTime(), + actualChunk.getStatistics().getEndTime()); + } + } + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + assertTrue(file.delete()); + } + } + + @Test + public void testAbsentTimeColumnSkipsDeviceRead() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-time-column-filtered.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + // the time column is filtered out, while s1 is a bloom filter false positive + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + bloomFilter.add(deviceID.toString() + ".s1"); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + Map result = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + + assertNull(result.get("")); + assertNull(result.get("s1")); + Mockito.verify(reader, Mockito.never()).getDeviceTimeseriesMetadata(deviceID); + Mockito.verify(reader, Mockito.never()) + .readTimeseriesMetadata( + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.anyBoolean(), + Mockito.any()); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testSingleMissingMeasurementUsesPerColumnFallback() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-single-miss.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + Statistics statistics = Statistics.getStatsByType(TSDataType.INT32); + statistics.setCount(1); + statistics.setStartTime(0); + statistics.setEndTime(10); + TimeseriesMetadata s2 = + new TimeseriesMetadata((byte) 0, 0, "s2", TSDataType.INT32, statistics, new PublicBAOS()); + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + + List deviceMetadata = new ArrayList<>(); + deviceMetadata.add( + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS())); + deviceMetadata.add( + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS())); + deviceMetadata.add(s2); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata); + + Map first = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + assertNotNull(first.get("s2")); + + // evict one measurement: the next call must load only that one, not the whole device + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s2")); + Mockito.when( + reader.readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.eq("s2"), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenReturn(Collections.singletonList(s2)); + + Map second = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + + assertNotNull(second.get("s2")); + Mockito.verify(reader, Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.eq("s2"), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBatchIgnoreNotExistsDoesNotCachePlaceholders() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-ignore-not-exists.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) + .thenReturn(Collections.emptyList()); + + Map ignored = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true); + assertNull(ignored.get("")); + assertNull(ignored.get("s1")); + + // ignoreNotExists must not write NULL placeholders: a normal query still has to read + assertThrows( + IOException.class, + () -> + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false)); + Mockito.verify(reader, Mockito.times(2)).getDeviceTimeseriesMetadata(deviceID); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testFewMissingMeasurementsUsePerColumnFallbackAfterEviction() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-few-missing.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2", "s3", "s4", "s5")); + + List deviceMetadata = new ArrayList<>(); + for (String measurement : allSensors) { + TSDataType dataType = measurement.isEmpty() ? TSDataType.INT64 : TSDataType.INT32; + Statistics statistics = Statistics.getStatsByType(dataType); + statistics.setCount(1); + statistics.setStartTime(0); + statistics.setEndTime(10); + deviceMetadata.add( + new TimeseriesMetadata( + (byte) 0, 0, measurement, dataType, statistics, new PublicBAOS())); + } + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata); + + Map first = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + assertNotNull(first.get("s5")); + + // evict three measurements: the cache still holds the majority, so the remaining misses + // must be loaded per column instead of re-reading the whole device + for (String measurement : Arrays.asList("s2", "s3", "s4")) { + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, measurement)); + } + Mockito.when( + reader.readTimeseriesMetadata( + Mockito.any(), + Mockito.isNull(), + Mockito.anyString(), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenReturn(deviceMetadata); + + Map second = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + assertNotNull(second.get("s2")); + assertNotNull(second.get("s3")); + assertNotNull(second.get("s4")); + Mockito.verify(reader, Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadata( + Mockito.any(), + Mockito.isNull(), + Mockito.anyString(), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java new file mode 100644 index 0000000000000..fed93e643a639 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -0,0 +1,378 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.commons.path.AlignedPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache; +import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; +import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.db.utils.constant.TestConstant; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; +import org.apache.tsfile.write.chunk.IChunkWriter; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; + +import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.createChunkWriter; +import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.writeAlignedChunk; + +/** + * Verifies the batch metadata path on a real device wider than one measurement leaf group, and that + * it matches the per-column path for dense, sparse and all-null projections. + */ +public class AlignedSeriesScanBatchTest { + + private static final String DATABASE = "root.wide_pd"; + private static final IDeviceID DEVICE = + IDeviceID.Factory.DEFAULT_FACTORY.create(DATABASE + ".d1"); + private static final int COLUMN_COUNT = 300; + private static final int PRESENT_COLUMN_COUNT = 30; + private static final int ROW_COUNT = 10; + + @Test + public void testBatchScanReadsWideAlignedDevice() throws Exception { + Assert.assertTrue( + "the fixture must span more than one measurement leaf group", + COLUMN_COUNT + 1 > TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode()); + File file = newFile(1); + TsFileResource resource = writeWideDevice(file, COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + metadataCache.clear(); + bloomFilterCache.clear(); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + // the batch path is only reachable from table dialect, which forces this flag to false + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + try { + long cacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + long diskReadsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get(); + + TsBlock block = scan(alignedPath(columnNames(COLUMN_COUNT)), COLUMN_COUNT, resource); + + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals(row, block.getTimeByIndex(row)); + } + for (int column = 0; column < COLUMN_COUNT; column++) { + Column valueColumn = block.getColumn(column); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertFalse("s" + column + " row " + row, valueColumn.isNull(row)); + Assert.assertEquals("s" + column + " row " + row, row, valueColumn.getInt(row)); + } + } + + // one batch read per device and file: no per-column cache lookups happened + Assert.assertEquals( + "the batch path must not fall back to per-column cache loads", + cacheHitsBefore, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get()); + // statistics are counted per series, like the per-column path (time column + all fields) + Assert.assertEquals( + diskReadsBefore + COLUMN_COUNT + 1, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get()); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + @Test + public void testSparseBatchScanMatchesPerColumn() throws Exception { + File file = newFile(2); + TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + AlignedFullPath scanPath = alignedPath(columnNames(COLUMN_COUNT)); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + try { + metadataCache.clear(); + bloomFilterCache.clear(); + long cacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + TsBlock batched = scan(scanPath, COLUMN_COUNT, resource); + long batchedCacheHits = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get() + - cacheHitsBefore; + + metadataCache.clear(); + long perColumnCacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + TsBlock perColumn = scan(scanPath, 0, resource); + long perColumnCacheHits = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get() + - perColumnCacheHitsBefore; + + // the sparse projection is identical to the per-column scan, including every null field + Assert.assertEquals(perColumn.getPositionCount(), batched.getPositionCount()); + Assert.assertEquals(perColumn.getValueColumnCount(), batched.getValueColumnCount()); + for (int row = 0; row < batched.getPositionCount(); row++) { + Assert.assertEquals(perColumn.getTimeByIndex(row), batched.getTimeByIndex(row)); + } + for (int column = 0; column < batched.getValueColumnCount(); column++) { + Column batchedColumn = batched.getColumn(column); + Column perColumnColumn = perColumn.getColumn(column); + boolean present = column < PRESENT_COLUMN_COUNT; + for (int row = 0; row < batched.getPositionCount(); row++) { + Assert.assertEquals( + "s" + column + " row " + row, perColumnColumn.isNull(row), batchedColumn.isNull(row)); + if (present) { + Assert.assertFalse(batchedColumn.isNull(row)); + Assert.assertEquals(row, batchedColumn.getInt(row)); + Assert.assertEquals(perColumnColumn.getInt(row), batchedColumn.getInt(row)); + } else { + Assert.assertTrue(batchedColumn.isNull(row)); + } + } + } + + // the batch scan resolved everything from disk, the per-column scan reused leaf groups + Assert.assertEquals(0, batchedCacheHits); + Assert.assertTrue(perColumnCacheHits > 0); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + @Test + public void testAllNullProjectionMatchesPerColumn() throws Exception { + File file = newFile(3); + TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + // every selected field is absent from the file + AlignedFullPath absentPath = alignedPath(absentColumnNames(COLUMN_COUNT)); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + try { + // table dialect (the only dialect that reaches the batch path): all-null rows are kept + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + metadataCache.clear(); + bloomFilterCache.clear(); + TsBlock batched = scan(absentPath, COLUMN_COUNT, resource); + Assert.assertEquals(ROW_COUNT, batched.getPositionCount()); + Assert.assertEquals(COLUMN_COUNT, batched.getValueColumnCount()); + for (int column = 0; column < COLUMN_COUNT; column++) { + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertTrue( + "absent" + column + " row " + row, batched.getColumn(column).isNull(row)); + } + } + + metadataCache.clear(); + TsBlock perColumn = scan(absentPath, 0, resource); + Assert.assertEquals(perColumn.getPositionCount(), batched.getPositionCount()); + Assert.assertEquals(perColumn.getValueColumnCount(), batched.getValueColumnCount()); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals(perColumn.getTimeByIndex(row), batched.getTimeByIndex(row)); + } + for (int column = 0; column < COLUMN_COUNT; column++) { + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals( + perColumn.getColumn(column).isNull(row), batched.getColumn(column).isNull(row)); + } + } + + // tree dialect: an all-null projection skips the file in both paths + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(true); + metadataCache.clear(); + bloomFilterCache.clear(); + Assert.assertFalse(hasFile(absentPath, COLUMN_COUNT, resource)); + + metadataCache.clear(); + Assert.assertFalse(hasFile(absentPath, 0, resource)); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + private static TsBlock scan( + AlignedFullPath scanPath, int totalFieldColumnCount, TsFileResource resource) + throws IOException { + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, totalFieldColumnCount, resource); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + return seriesScanUtil.nextPage(); + } + + private static boolean hasFile( + AlignedFullPath scanPath, int totalFieldColumnCount, TsFileResource resource) + throws IOException { + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, totalFieldColumnCount, resource); + while (true) { + Optional hasNextFile = seriesScanUtil.hasNextFile(); + if (hasNextFile.isPresent()) { + return hasNextFile.get(); + } + } + } + + private static AlignedSeriesScanUtil newScanUtil( + AlignedFullPath scanPath, int totalFieldColumnCount, TsFileResource resource) + throws IOException { + SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); + builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); + SeriesScanOptions scanOptions = builder.build(); + scanOptions.setTotalFieldColumnCount(totalFieldColumnCount); + AlignedSeriesScanUtil seriesScanUtil = + new AlignedSeriesScanUtil( + scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource( + new QueryDataSource(Collections.singletonList(resource), Collections.emptyList())); + return seriesScanUtil; + } + + private static AlignedFullPath alignedPath(List measurementNames) { + List schemas = new ArrayList<>(measurementNames.size()); + for (String measurement : measurementNames) { + schemas.add(new MeasurementSchema(measurement, TSDataType.INT32)); + } + return new AlignedFullPath(DEVICE, measurementNames, schemas); + } + + private static List columnNames(int count) { + List names = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + names.add("s" + i); + } + return names; + } + + private static List absentColumnNames(int count) { + List names = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + names.add("absent" + i); + } + return names; + } + + private static File newFile(int version) { + File file = new File(TestConstant.getTestTsFilePath(DATABASE, 0, 0, version)); + if (!file.getParentFile().exists()) { + Assert.assertTrue(file.getParentFile().mkdirs()); + } + return file; + } + + private static TsFileResource writeWideDevice(File file, int presentColumnCount) + throws IOException, IllegalPathException { + TsFileResource resource = new TsFileResource(file); + List writtenPaths = new ArrayList<>(presentColumnCount); + List dataTypes = new ArrayList<>(presentColumnCount); + List encodings = new ArrayList<>(presentColumnCount); + List compressionTypes = new ArrayList<>(presentColumnCount); + for (int i = 0; i < presentColumnCount; i++) { + writtenPaths.add(new AlignedPath(DEVICE.toString(), "s" + i)); + dataTypes.add(TSDataType.INT32); + encodings.add(TSEncoding.PLAIN); + compressionTypes.add(CompressionType.UNCOMPRESSED); + } + try (TsFileIOWriter writer = new TsFileIOWriter(file)) { + writer.startChunkGroup(DEVICE); + for (IChunkWriter chunkWriter : + createChunkWriter(writtenPaths, dataTypes, encodings, compressionTypes, true)) { + writeAlignedChunk( + (AlignedChunkWriterImpl) chunkWriter, + writer, + Collections.singletonList(new TimeRange(0L, ROW_COUNT - 1L)), + true); + } + writer.endChunkGroup(); + writer.endFile(); + } + resource.updateStartTime(DEVICE, 0); + resource.updateEndTime(DEVICE, ROW_COUNT - 1L); + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + return resource; + } + + private static void cleanup( + File file, + TsFileResource resource, + TimeSeriesMetadataCache metadataCache, + BloomFilterCache bloomFilterCache) + throws IOException { + metadataCache.clear(); + bloomFilterCache.clear(); + FileReaderManager.getInstance().closeFileAndRemoveReader(resource.getTsFileID()); + Assert.assertTrue(file.delete()); + } +}