Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
1284723
feat(table): add LTTB table function
DaZuiZui Sep 24, 2026
472494e
fix(table): bound LTTB overlapping window buffer memory
DaZuiZui Oct 8, 2026
b2f9b77
fix(table): correct LTTB memory bounds and sparse sampling
DaZuiZui Oct 8, 2026
4074d36
fix(table): preserve LTTB integer precision and guard window bounds
DaZuiZui Oct 8, 2026
edbb76a
fix(table): handle LTTB sampling and count-window overflow
DaZuiZui Oct 9, 2026
98730f3
fix(lttb): reject duplicate timestamps with 701 and keep non-finite v…
DaZuiZui Oct 12, 2026
1e3e8a9
Merge remote-tracking branch 'origin/feat/lttb-table-function' into f…
DaZuiZui Oct 12, 2026
cb52833
test(lttb): source duplicate timestamps from a real table in the 701 IT
DaZuiZui Oct 12, 2026
5cdb336
fix(lttb): address review findings on the LTTB table function
DaZuiZui Oct 13, 2026
213393e
docs(lttb): state the settled SIZE-mode semantics and cover the bound…
DaZuiZui Oct 13, 2026
b7c5a83
fix(lttb): move the memory budget out of the UDF API and run LTTB ove…
DaZuiZui Oct 21, 2026
578a2bf
Merge remote-tracking branch 'upstream/master' into feat/lttb-table-f…
DaZuiZui Oct 21, 2026
9a10836
test(lttb): reject duplicates that span distributed fragments
DaZuiZui Oct 21, 2026
979cd06
fix(lttb): use one lookahead rule and cover the review gaps
DaZuiZui Oct 23, 2026
87b58e8
test(lttb): cover the non-finite lookahead case for N => 4
DaZuiZui Oct 23, 2026
3660c70
fix(lttb): stream windows when a column has a single eligible point
DaZuiZui Nov 2, 2026
640d402
fix(lttb): release closed windows that can never emit a row
DaZuiZui Nov 2, 2026
160bc71
fix(lttb): keep the anchor when a window bucket has no finite candidate
DaZuiZui Nov 12, 2026
5f96d40
refactor(lttb): merge hasExactValue into isFiniteValue
DaZuiZui Nov 12, 2026
0e173d9
fix(lttb): validate ORIGIN in analyze() and pin the budget accounting
DaZuiZui Nov 12, 2026
5ed7032
fix(lttb): tell an explicit ORIGIN from the default by presence, not …
DaZuiZui Nov 12, 2026
41b7abf
Merge remote-tracking branch 'origin/master' into feat/lttb-table-fun…
DaZuiZui Nov 16, 2026
db7fa8c
fix(lttb): count the buffer once so 1C1D fragments fit
DaZuiZui Nov 16, 2026
ed14769
fix(lttb): keep both memory sides and shrink the default budget
DaZuiZui Nov 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -119,13 +119,19 @@ public void showConfigurationDisplaysEffectiveValue() throws Exception {

// Case 2: a valid value is not rewritten, so display must equal the file value (regression).
Map<String, String> afterValidReload =
appendLinesLoadAndShow(dataNodeId, confPath, "cte_buffer_size_in_bytes=262144");
appendLinesLoadAndShow(
dataNodeId,
confPath,
"cte_buffer_size_in_bytes=262144",
"lttb_buffer_size_in_bytes=262144");
Assert.assertEquals("262144", afterValidReload.get("cte_buffer_size_in_bytes"));
Assert.assertEquals("262144", afterValidReload.get("lttb_buffer_size_in_bytes"));

// Case 3: params whose template default is 0 are always rewritten to a computed default,
// so they must never display 0 (this was always wrong, even without a bad file value).
Map<String, String> defaultsReload = appendLinesLoadAndShow(dataNodeId, confPath);
assertPositiveNonZero(defaultsReload, "sort_buffer_size_in_bytes");
assertPositiveNonZero(defaultsReload, "lttb_buffer_size_in_bytes");
assertPositiveNonZero(defaultsReload, "mods_cache_size_limit_per_fi_in_bytes");
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,13 @@ public class TableFunctionOperator extends AbstractOperator implements ProcessOp
private final boolean isDeclaredAsPassThrough;
private final IoTDBLocal ioTDBLocal;

/**
* Bytes the data processors may buffer internally while a partition is processed. It is resolved
* once from the provider, which also gives the same value to every processor it creates, so the
* reservation and the enforced limit cannot diverge.
*/
private final long estimatedMaxBufferedBytes;

private TableFunctionDataProcessor processor;
private PartitionState partitionState;
private ListenableFuture<?> isBlocked;
Expand All @@ -83,6 +90,7 @@ public class TableFunctionOperator extends AbstractOperator implements ProcessOp
public TableFunctionOperator(
CommonOperatorContext operatorContext,
TableFunctionProcessorProvider processorProvider,
long estimatedMaxBufferedBytes,
Operator inputOperator,
List<TSDataType> inputDataTypes,
List<TSDataType> outputDataTypes,
Expand All @@ -104,6 +112,7 @@ public TableFunctionOperator(
new PartitionRecognizer(
partitionChannels, requiredChannels, passThroughChannels, inputDataTypes);
this.isDeclaredAsPassThrough = isDeclaredAsPassThrough;
this.estimatedMaxBufferedBytes = estimatedMaxBufferedBytes;
this.needPassThrough = properChannelCount != outputDataTypes.size();
this.partitionState = null;
this.properBlockBuilder = new TsBlockBuilder(outputDataTypes.subList(0, properChannelCount));
Expand Down Expand Up @@ -294,7 +303,10 @@ public boolean isFinished() throws Exception {

@Override
public long calculateMaxPeekMemory() {
// A processor that buffers its input (LTTB) holds up to estimatedMaxBufferedBytes while the
// partition is processed, in addition to the input it reads and the block it builds.
return inputOperator.calculateMaxPeekMemory()
+ estimatedMaxBufferedBytes
+ Math.max(maxReturnSize, properBlockBuilder.getRetainedSizeInBytes());
}

Expand All @@ -305,7 +317,12 @@ public long calculateMaxReturnSize() {

@Override
public long calculateRetainedSizeAfterCallingNext() {
return inputOperator.calculateRetainedSizeAfterCallingNext();
// The buffers survive across next() calls until the partition completes, so they belong to the
// retained size as well as the peak. Parents read one or the other (FilterAndProjectOperator
// adds the child's retained size and never reads the child's peak; sinks and LimitOperator
// take the child's peak), so keep both. Sort adds the two together; the LTTB default budget is
// sized so that sum still fits one query thread.
return inputOperator.calculateRetainedSizeAfterCallingNext() + estimatedMaxBufferedBytes;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@
import org.apache.iotdb.commons.queryengine.plan.relational.function.BoundSignature;
import org.apache.iotdb.commons.queryengine.plan.relational.function.FunctionKind;
import org.apache.iotdb.commons.queryengine.plan.relational.function.ITableFunctionFactory;
import org.apache.iotdb.commons.queryengine.plan.relational.function.TableFunctionProcessorMemoryBudget;
import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ResolvedFunction;
import org.apache.iotdb.commons.queryengine.plan.relational.planner.OrderingScheme;
import org.apache.iotdb.commons.queryengine.plan.relational.planner.SortOrder;
Expand Down Expand Up @@ -1684,6 +1685,12 @@ public Operator visitTableFunctionProcessor(TableFunctionProcessorNode node, C c
return new TableFunctionOperator(
operatorContext,
processorProvider,
// The budget is taken from the provider itself, so the operator reserves exactly what the
// processors of this provider enforce. A user-defined function reports nothing.
processorProvider instanceof TableFunctionProcessorMemoryBudget
? ((TableFunctionProcessorMemoryBudget) processorProvider)
.getEstimatedMaxBufferedBytes()
: 0L,
operator,
inputDataTypes,
outputDataTypes,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1122,6 +1122,15 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException
commonConfig.setCteBufferSize(cteBufferSizeInBytes);
}

// The per-processor buffer budget of the LTTB table function. The budget is reserved in both
// the peak and the retained size, and a sort above LTTB adds those two on top of its own
// buffer. The non-positive default therefore takes one eighth of one query thread's operator
// memory, which keeps 2 * lttb + sort within that thread. Half, which is what
// sort_buffer_size_in_bytes uses, makes the sum larger than the free operator memory on the
// 1C1D integration-test heap and the fragment is rejected with error 719.
loadFixedSizeLimitForQuery(
properties, "lttb_buffer_size_in_bytes", commonConfig::setLttbBufferSize, 8);

// max number of rows for cte materialization
int maxRowsInCteBuffer =
Integer.parseInt(
Expand Down Expand Up @@ -1298,12 +1307,23 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException

private void loadFixedSizeLimitForQuery(
TrimProperties properties, String name, LongConsumer setFunction) {
loadFixedSizeLimitForQuery(properties, name, setFunction, 2);
}

/**
* Resolves a per-fragment byte budget. A non-positive configured value becomes {@code min(32 MiB,
* operator memory / query_thread_count / threadDivisor)}. Sort and the mods cache split one query
* thread's operator memory in two. LTTB splits it in eight: its budget is counted in the peak
* and again in the retained size, and the sort buffer is reserved beside those two.
*/
private void loadFixedSizeLimitForQuery(
TrimProperties properties, String name, LongConsumer setFunction, int threadDivisor) {
long defaultValue =
Math.min(
32 * 1024 * 1024L,
memoryConfig.getOperatorsMemoryManager().getTotalMemorySizeInBytes()
/ memoryConfig.getQueryThreadCount()
/ 2);
/ threadDivisor);
long size = Long.parseLong(properties.getProperty(name, Long.toString(defaultValue)));
if (size <= 0) {
size = defaultValue;
Expand Down Expand Up @@ -2405,6 +2425,14 @@ public synchronized void loadHotModifiedProps(TrimProperties properties)
if (cteBufferSizeInBytes > 0) {
commonConfig.setCteBufferSize(cteBufferSizeInBytes);
}
// The per-processor buffer budget of the LTTB table function. The budget is reserved in both
// the peak and the retained size, and a sort above LTTB adds those two on top of its own
// buffer. The non-positive default therefore takes one eighth of one query thread's operator
// memory, which keeps 2 * lttb + sort within that thread. Half, which is what
// sort_buffer_size_in_bytes uses, makes the sum larger than the free operator memory on the
// 1C1D integration-test heap and the fragment is rejected with error 719.
loadFixedSizeLimitForQuery(
properties, "lttb_buffer_size_in_bytes", commonConfig::setLttbBufferSize, 8);
// max number of rows for cte materialization
int maxRowsInCteBuffer =
Integer.parseInt(
Expand Down Expand Up @@ -2452,6 +2480,8 @@ public synchronized void loadHotModifiedProps(TrimProperties properties)
private void overlayEffectiveConfigurationValues() {
ConfigurationFileUtils.updateAppliedProperties(
"cte_buffer_size_in_bytes", Long.toString(commonConfig.getCteBufferSize()));
ConfigurationFileUtils.updateAppliedProperties(
"lttb_buffer_size_in_bytes", Long.toString(commonConfig.getLttbBufferSize()));
ConfigurationFileUtils.updateAppliedProperties(
"max_rows_in_cte_buffer", Integer.toString(commonConfig.getMaxRowsInCteBuffer()));
ConfigurationFileUtils.updateAppliedProperties(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.commons.exception.IoTDBRuntimeException;
import org.apache.iotdb.commons.exception.SemanticException;
import org.apache.iotdb.commons.i18n.CommonMessages;
import org.apache.iotdb.commons.i18n.QueryMessages;
import org.apache.iotdb.commons.queryengine.common.SessionInfo;
import org.apache.iotdb.commons.queryengine.plan.relational.analyzer.NodeRef;
Expand Down Expand Up @@ -125,6 +126,7 @@
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
import org.apache.iotdb.commons.udf.builtin.relational.tvf.FFTTableFunction;
import org.apache.iotdb.commons.udf.builtin.relational.tvf.LTTBTableFunction;
import org.apache.iotdb.commons.udf.builtin.relational.tvf.M4TableFunction;
import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer;
import org.apache.iotdb.commons.utils.FileUtils;
Expand Down Expand Up @@ -6105,13 +6107,15 @@ private ArgumentsAnalysis analyzeArguments(
}
}
tryAppendM4ModeArgument(functionName, arguments, parameterSpecifications, passedArguments);
tryAppendLTTBModeArgument(functionName, arguments, parameterSpecifications, passedArguments);
tryAppendFFTInternalArguments(
functionName, arguments, parameterSpecifications, passedArguments);
return new ArgumentsAnalysis(passedArguments.buildOrThrow(), tableArgumentAnalyses.build());
}

private boolean isPartitionColumnsProvidedByProperSchema(String functionName) {
return TableBuiltinTableFunction.M4.getFunctionName().equalsIgnoreCase(functionName)
|| TableBuiltinTableFunction.LTTB.getFunctionName().equalsIgnoreCase(functionName)
|| TableBuiltinTableFunction.LOWPASS.getFunctionName().equalsIgnoreCase(functionName)
|| TableBuiltinTableFunction.HIGHPASS.getFunctionName().equalsIgnoreCase(functionName)
|| TableBuiltinTableFunction.XCORR.getFunctionName().equalsIgnoreCase(functionName)
Expand Down Expand Up @@ -6223,6 +6227,92 @@ && containsTableFunctionArgument(
new ScalarArgument(org.apache.iotdb.udf.api.type.Type.BOOLEAN, isTimeWindow));
}

/**
* Resolves the LTTB execution mode from the SQL literals. The mode (target-count vs. time
* window vs. count window) depends on whether {@code SIZE} is a duration literal, which is only
* visible at this level, so it is injected as the internal boolean argument {@link
* LTTBTableFunction#WINDOW_MODE_PARAMETER_NAME}. Parameter-combination checks that only need
* the presence of arguments are also performed here so they fail before analysis.
*/
private void tryAppendLTTBModeArgument(
String functionName,
List<TableFunctionArgument> arguments,
List<ParameterSpecification> parameterSpecifications,
ImmutableMap.Builder<String, Argument> passedArguments) {
if (!TableBuiltinTableFunction.LTTB.getFunctionName().equalsIgnoreCase(functionName)) {
return;
}

Optional<TableFunctionArgument> nArgument =
findOptionalTableFunctionArgument(
arguments, parameterSpecifications, LTTBTableFunction.N_PARAMETER_NAME);
Optional<TableFunctionArgument> sizeArgument =
findOptionalTableFunctionArgument(
arguments, parameterSpecifications, LTTBTableFunction.SIZE_PARAMETER_NAME);
Optional<TableFunctionArgument> slideArgument =
findOptionalTableFunctionArgument(
arguments, parameterSpecifications, LTTBTableFunction.SLIDE_PARAMETER_NAME);
boolean originSpecified =
containsTableFunctionArgument(
arguments, parameterSpecifications, LTTBTableFunction.ORIGIN_PARAMETER_NAME);

if (nArgument.isPresent() == sizeArgument.isPresent()) {
throw new SemanticException(
CommonMessages
.EXCEPTION_EXACTLY_ONE_OF_THE_N_AND_SIZE_ARGUMENTS_MUST_BE_SPECIFIED_FOR_LTTB_54AF0733);
}

boolean isTimeWindow = false;
if (nArgument.isPresent()) {
if (slideArgument.isPresent() || originSpecified) {
throw new SemanticException(
CommonMessages
.EXCEPTION_THE_N_ARGUMENT_OF_LTTB_CANNOT_BE_COMBINED_WITH_THE_SLIDE_OR_ORIGIN_ARGUMENTS_C2FF1FE1);
}
if (!(nArgument.get().getValue() instanceof Expression)
|| nArgument.get().getValue() instanceof TimeDurationLiteral) {
throw new SemanticException(
CommonMessages.EXCEPTION_THE_N_ARGUMENT_OF_LTTB_MUST_BE_A_POSITIVE_INTEGER_21795D8E);
}
} else {
if (!(sizeArgument.get().getValue() instanceof Expression)) {
throw new SemanticException(
String.format(
QueryMessages.EXCEPTION_INVALID_ARGUMENT_ARG_EXPECTED_SCALAR_ARGUMENT_4AD4E32F,
LTTBTableFunction.SIZE_PARAMETER_NAME));
}
isTimeWindow = sizeArgument.get().getValue() instanceof TimeDurationLiteral;
if (slideArgument.isPresent()) {
if (!(slideArgument.get().getValue() instanceof Expression)) {
throw new SemanticException(
String.format(
QueryMessages.EXCEPTION_INVALID_ARGUMENT_ARG_EXPECTED_SCALAR_ARGUMENT_4AD4E32F,
LTTBTableFunction.SLIDE_PARAMETER_NAME));
}
boolean isTimeSlide = slideArgument.get().getValue() instanceof TimeDurationLiteral;
if (isTimeWindow != isTimeSlide) {
throw new SemanticException(
QueryMessages
.EXCEPTION_THE_SLIDE_ARGUMENT_MUST_HAVE_THE_SAME_WINDOW_MODE_AS_THE_SIZE_ARGUMENT_419C9844);
}
}
if (!isTimeWindow && originSpecified) {
throw new SemanticException(
QueryMessages
.EXCEPTION_THE_ORIGIN_ARGUMENT_IS_ONLY_SUPPORTED_IN_TIME_WINDOW_MODE_B3F358B9);
}
}
validateM4OrderBySortOrder(arguments, parameterSpecifications);
passedArguments.put(
LTTBTableFunction.WINDOW_MODE_PARAMETER_NAME,
new ScalarArgument(org.apache.iotdb.udf.api.type.Type.BOOLEAN, isTimeWindow));
// Presence, not value: Long.MIN_VALUE is a legal timestamp, so the value alone cannot tell
// an explicit ORIGIN from the injected default.
passedArguments.put(
LTTBTableFunction.ORIGIN_SPECIFIED_PARAMETER_NAME,
new ScalarArgument(org.apache.iotdb.udf.api.type.Type.BOOLEAN, originSpecified));
}

private void validateM4OrderBySortOrder(
List<TableFunctionArgument> arguments,
List<ParameterSpecification> parameterSpecifications) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1610,6 +1610,7 @@ public RelationPlan visitTableFunctionInvocation(TableFunctionInvocation node, V

private boolean needAddPartitionColumn(String functionName) {
return !TableBuiltinTableFunction.M4.getFunctionName().equalsIgnoreCase(functionName)
&& !TableBuiltinTableFunction.LTTB.getFunctionName().equalsIgnoreCase(functionName)
&& !TableBuiltinTableFunction.LOWPASS.getFunctionName().equalsIgnoreCase(functionName)
&& !TableBuiltinTableFunction.HIGHPASS.getFunctionName().equalsIgnoreCase(functionName)
&& !TableBuiltinTableFunction.XCORR.getFunctionName().equalsIgnoreCase(functionName)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3403,7 +3403,13 @@ private boolean canSplitTableFunctionProcessor(TableFunctionProcessorNode node)
if (!isPartitionedGroup(node.getChild())) {
return false;
}
return !TableBuiltinTableFunction.FFT.getFunctionName().equalsIgnoreCase(node.getName());
// LTTB samples a whole partition and FFT transforms a whole partition, so neither may run once
// per fragment: a partitioned GroupNode does not make a partition complete within one branch.
// The function therefore has to run once, over the partitions merged in order. The split rule
// itself is a separate planner issue; the exclusion can be revisited once that is fixed.
String functionName = node.getName();
return !TableBuiltinTableFunction.FFT.getFunctionName().equalsIgnoreCase(functionName)
&& !TableBuiltinTableFunction.LTTB.getFunctionName().equalsIgnoreCase(functionName);
}

private boolean isPartitionedGroup(PlanNode node) {
Expand Down
Loading