Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1710,6 +1710,9 @@ private StorageEngineMessages() {}
public static final String COMPACTION_CROSS_SPACE = "cross";
public static final String DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT =
"%s does not exist in the resource file";
public static final String
EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374 =
"Device %s is recorded in the resource but missing from the metadata of TsFile %s";
public static final String TARGET_FILE_SMALLER_THAN_MAGIC_STRING_AND_VERSION_NUMBER_SIZE_FMT =
"target file %s is smaller than magic string and version number size";
public static final String CURRENT_POINT_TIMESTAMP_SHOULD_BE_LATER_FMT =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1620,6 +1620,9 @@ private StorageEngineMessages() {}
public static final String COMPACTION_CROSS_SPACE = "跨空间";
public static final String DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT =
"%s 在资源文件中不存在";
public static final String
EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374 =
"设备 %s 记录在资源文件中,但在 TsFile %s 的元数据中不存在";
public static final String TARGET_FILE_SMALLER_THAN_MAGIC_STRING_AND_VERSION_NUMBER_SIZE_FMT =
"目标文件 %s 小于 magic string 和版本号大小";
public static final String CURRENT_POINT_TIMESTAMP_SHOULD_BE_LATER_FMT =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@ protected AbstractAlignedTimeSeriesMetadata loadTimeSeriesMetadata(
scanOptions.getGlobalTimeFilter(),
isSeq,
ignoreAllNullRows,
null);
null,
scanOptions.getTotalFieldColumnCount());
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import org.apache.iotdb.db.utils.ModificationUtils;
import org.apache.iotdb.db.utils.SchemaUtils;

import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata;
Expand All @@ -62,6 +63,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

Expand All @@ -72,6 +74,12 @@ public class FileLoaderUtils {
private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET =
SeriesScanCostMetricSet.getInstance();

/**
* Fraction of a table's field columns that a scan must select for it to use the batch
* device-metadata read instead of the per-column path.
*/
private static final double DEVICE_METADATA_BATCH_SCAN_RATIO = 0.6;
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved

private FileLoaderUtils() {
// util class
}
Expand Down Expand Up @@ -222,7 +230,8 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata(
Filter globalTimeFilter,
boolean isSeq,
boolean ignoreAllNullRows,
long[] rootMeasurementMetadataIndexNodeOffset)
long[] rootMeasurementMetadataIndexNodeOffset,
int totalFieldColumnCount)
throws IOException {
final long t1 = System.nanoTime();
boolean loadFromMem = false;
Expand All @@ -241,7 +250,8 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata(
context,
globalTimeFilter,
ignoreAllNullRows,
rootMeasurementMetadataIndexNodeOffset);
rootMeasurementMetadataIndexNodeOffset,
totalFieldColumnCount);
} else { // if the tsfile is unclosed, we just get it directly from TsFileResource
loadFromMem = true;
alignedTimeSeriesMetadata =
Expand Down Expand Up @@ -333,7 +343,8 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr
FragmentInstanceContext context,
Filter globalTimeFilter,
boolean ignoreAllNullRows,
long[] rootMeasurementMetadataIndexNodeOffset)
long[] rootMeasurementMetadataIndexNodeOffset,
int totalFieldColumnCount)
throws IOException {
AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null;
// load all the TimeseriesMetadata of vector, the first one is for time column and the
Expand All @@ -348,13 +359,40 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr

// when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
// we should not ignore the non-exist of device in TsFileMetadata
boolean ignoreNotExists =
context.ignoreNotExistsDevice()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE;

// For a wide device (a table scan that selects almost all field columns), deserialize the
// whole device's TimeSeriesMetadata in a single batch read instead of walking the device
// metadata index once per column. External tsfiles address metadata by offset, so they must
// stay on the per-column path; DEBUG queries also stay there so the per-column cache's
// diagnostic logs are preserved.
if (shouldUseDeviceMetadataBatchScan(
valueMeasurementList.size(),
totalFieldColumnCount,
isDebug,
rootMeasurementMetadataIndexNodeOffset != null)) {
return loadAlignedTimeSeriesMetadataBatch(
resource,
alignedPath,
valueMeasurementList,
allSensors,
context,
globalTimeFilter,
ignoreAllNullRows,
ignoreNotExists,
cache,
filePath,
deviceId);
}

TimeseriesMetadata timeColumn =
cache.get(
filePath,
new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""),
allSensors,
context.ignoreNotExistsDevice()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
ignoreNotExists,
isDebug,
context,
rootMeasurementMetadataIndexNodeOffset);
Expand All @@ -375,38 +413,111 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr
} else {
List<TimeseriesMetadata> valueTimeSeriesMetadataList =
new ArrayList<>(valueMeasurementList.size());
// if all the queried aligned sensors does not exist, we will return null
boolean exist = false;
for (String measurement : valueMeasurementList) {
TimeseriesMetadata valueColumn =
valueTimeSeriesMetadataList.add(
cache.get(
filePath,
new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement),
allSensors,
context.ignoreNotExistsDevice()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
ignoreNotExists,
isDebug,
context,
rootMeasurementMetadataIndexNodeOffset);
exist = (exist || (valueColumn != null));
valueTimeSeriesMetadataList.add(valueColumn);
}
if (!ignoreAllNullRows || exist) {
alignedTimeSeriesMetadata =
setModifications(
resource,
timeColumn,
valueTimeSeriesMetadataList,
alignedPath,
context,
globalTimeFilter,
ignoreAllNullRows);
rootMeasurementMetadataIndexNodeOffset));
}
alignedTimeSeriesMetadata =
buildAlignedTimeSeriesMetadata(
resource,
alignedPath,
timeColumn,
valueTimeSeriesMetadataList,
context,
globalTimeFilter,
ignoreAllNullRows);
}
}
return alignedTimeSeriesMetadata;
}

static boolean shouldUseDeviceMetadataBatchScan(
int selectedColumnCount, int totalFieldColumnCount, boolean debug, boolean offsetAddressed) {
// DEBUG queries keep the per-column cache's diagnostic logs; offset-addressed external
// tsfiles cannot be resolved through the table metadata index; and a table with at least
// max_degree_of_index_node field columns has a dense device of max_degree+1 series (time column
// included), which spans more than one measurement leaf group. A sparse device may hold fewer
// in-file series; the bloom filter and the cache-aware load keep that case cheap.
return !debug
&& !offsetAddressed
&& totalFieldColumnCount
>= TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode()
&& (double) selectedColumnCount / totalFieldColumnCount >= DEVICE_METADATA_BATCH_SCAN_RATIO;
}

private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBatch(
TsFileResource resource,
AlignedFullPath alignedPath,
List<String> valueMeasurementList,
Set<String> allSensors,
FragmentInstanceContext context,
Filter globalTimeFilter,
boolean ignoreAllNullRows,
boolean ignoreNotExists,
TimeSeriesMetadataCache cache,
String filePath,
IDeviceID deviceId)
throws IOException {
Map<String, TimeseriesMetadata> deviceMetadata =
cache.getDeviceTimeSeriesMetadata(
filePath, resource.getTsFileID(), deviceId, allSensors, context, ignoreNotExists);
if (deviceMetadata.isEmpty()) {
// the cache applies the same bloom-filter / ignoreNotExists rules as the per-column get(),
// so an empty map means the device is expected to be absent from this file
return null;
}
TimeseriesMetadata timeColumn = deviceMetadata.get("");
if (timeColumn == null) {
return null;
}
List<TimeseriesMetadata> valueTimeSeriesMetadataList =
new ArrayList<>(valueMeasurementList.size());
for (String measurement : valueMeasurementList) {
valueTimeSeriesMetadataList.add(deviceMetadata.get(measurement));
}
return buildAlignedTimeSeriesMetadata(
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved
resource,
alignedPath,
timeColumn,
valueTimeSeriesMetadataList,
context,
globalTimeFilter,
ignoreAllNullRows);
}

private static AbstractAlignedTimeSeriesMetadata buildAlignedTimeSeriesMetadata(
TsFileResource resource,
AlignedFullPath alignedPath,
TimeseriesMetadata timeColumn,
List<TimeseriesMetadata> valueTimeSeriesMetadataList,
FragmentInstanceContext context,
Filter globalTimeFilter,
boolean ignoreAllNullRows) {
// if all the queried aligned sensors does not exist, we will return null
boolean exist = false;
for (TimeseriesMetadata valueColumn : valueTimeSeriesMetadataList) {
exist = (exist || (valueColumn != null));
}
if (!ignoreAllNullRows || exist) {
return setModifications(
resource,
timeColumn,
valueTimeSeriesMetadataList,
alignedPath,
context,
globalTimeFilter,
ignoreAllNullRows);
}
return null;
}

private static AbstractAlignedTimeSeriesMetadata setModifications(
TsFileResource resource,
TimeseriesMetadata timeColumnMetadata,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,8 @@ static AbstractAlignedTimeSeriesMetadata loadTimeSeriesMetadata(
globalTimeFilter,
resource.isSeq(),
context.isIgnoreAllNullRows(),
new long[] {currentDeviceOffset.getStartOffset(), currentDeviceOffset.getEndOffset()});
new long[] {currentDeviceOffset.getStartOffset(), currentDeviceOffset.getEndOffset()},
0);
}

@FunctionalInterface
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1185,6 +1185,9 @@ private void addSource(
topKRuntimeFilter);
seriesScanOptions.setTTLForTableView(viewTTL);
seriesScanOptions.setIsTableViewForTreeModel(node instanceof TreeDeviceViewScanNode);
if (!(node instanceof TreeDeviceViewScanNode) && !(node instanceof ExternalTsFileScanNode)) {
seriesScanOptions.setTotalFieldColumnCount(computeTotalFieldColumnCount(node));
}

OperatorContext operatorContext = addOperatorContext(context, node.getPlanNodeId(), className);

Expand Down Expand Up @@ -1222,6 +1225,17 @@ private BatchDeviceEntrySource createDeviceEntrySource(DeviceTableScanNode node)
.orElseGet(() -> new InMemoryDeviceEntrySource(node.getDeviceEntries()));
}

private int computeTotalFieldColumnCount(DeviceTableScanNode node) {
QualifiedObjectName qualifiedObjectName = node.getQualifiedObjectName();
TsTable tsTable =
DataNodeTableCache.getInstance()
.getTableInCache(
qualifiedObjectName.getDatabaseName(), qualifiedObjectName.getObjectName());
// the numerator only contains FIELD columns, so the denominator must be the field count too;
// getColumnNum() would also count tag, attribute and time columns
return tsTable == null ? 0 : tsTable.getFieldNum();
}

private BatchDeviceEntrySource createAggregationDeviceEntrySource(AggregationTableScanNode node) {
if (node.getDeviceEntryCount() == 0 && node.getGroupingKeys().isEmpty()) {
return new InMemoryDeviceEntrySource(Collections.singletonList(null));
Expand Down Expand Up @@ -1769,6 +1783,10 @@ private boolean ensureFieldColumnForTreeNonAlignedDeviceViewScan(
node.getPushDownPredicate());
seriesScanOptions.setTTLForTableView(tableViewTTL);
seriesScanOptions.setIsTableViewForTreeModel(node instanceof AggregationTreeDeviceViewScanNode);
if (!(node instanceof AggregationTreeDeviceViewScanNode)
&& !(node instanceof ExternalTsFileAggregationScanNode)) {
seriesScanOptions.setTotalFieldColumnCount(computeTotalFieldColumnCount(node));
}

Set<String> allSensors = new HashSet<>(measurementColumnNames);
allSensors.add(""); // for time column
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ public class SeriesScanOptions implements Accountable {
private PaginationController paginationController;
private boolean isTableViewForTreeModel;
private long ttlForTableView = Long.MAX_VALUE;
private int totalFieldColumnCount = 0;
private final TopKRuntimeFilter topKRuntimeFilter;
private static final long INSTANCE_SIZE =
RamUsageEstimator.shallowSizeOfInstance(
Expand Down Expand Up @@ -178,6 +179,14 @@ public void setIsTableViewForTreeModel(boolean isTableViewForTreeModel) {
this.isTableViewForTreeModel = isTableViewForTreeModel;
}

public int getTotalFieldColumnCount() {
return totalFieldColumnCount;
}

public void setTotalFieldColumnCount(int totalFieldColumnCount) {
this.totalFieldColumnCount = totalFieldColumnCount;
}

/**
* pushLimitToEachDevice==false means that all devices return total limit rows.
*
Expand Down
Loading
Loading