From 1284723ce512c32158a997c3e28d2ba737ca2bed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:32:31 +0800 Subject: [PATCH 01/21] feat(table): add LTTB table function --- .../it/db/it/IoTDBLTTBTableFunctionIT.java | 141 +++ .../analyzer/StatementAnalyzer.java | 85 ++ .../relational/planner/RelationPlanner.java | 1 + .../TableDistributedPlanGenerator.java | 5 +- .../analyzer/TableFunctionTest.java | 158 +++ .../iotdb/commons/i18n/CommonMessages.java | 19 + .../iotdb/commons/i18n/CommonMessages.java | 19 + .../function/TableBuiltinTableFunction.java | 4 + .../relational/tvf/LTTBTableFunction.java | 930 ++++++++++++++++++ .../relational/tvf/LTTBTableFunctionTest.java | 140 +++ 10 files changed, 1501 insertions(+), 1 deletion(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java new file mode 100644 index 0000000000000..63ab046927233 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBLTTBTableFunctionIT { + + private static final String DATABASE_NAME = "lttb"; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE metrics(device STRING TAG, s1 DOUBLE FIELD, s2 DOUBLE FIELD)"); + statement.execute( + "INSERT INTO metrics(time, device, s1, s2) VALUES " + + "(0, 'd1', 0.0, 0.0)," + + "(1, 'd1', 1.0, 0.0)," + + "(2, 'd1', 8.0, null)," + + "(3, 'd1', 2.0, 9.0)," + + "(4, 'd1', 0.0, 1.0)," + + "(5, 'd1', 0.0, 0.0)," + + "(0, 'd2', null, null)," + + "(1, 'd2', null, null)"); + statement.execute("FLUSH"); + } catch (Exception e) { + fail("insertData failed: " + e.getMessage()); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testTargetCountPreservesEndpointsAndAlignsColumns() { + tableResultSetEqualTest( + "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time, " + + "TIMECOL => 'time', N => 3) ORDER BY device, s1_time", + new String[] {"window_index", "device", "s1_time", "s1", "s2_time", "s2"}, + new String[] { + "0,d1,1970-01-01T00:00:00.000Z,0.0,1970-01-01T00:00:00.000Z,0.0,", + "0,d1,1970-01-01T00:00:00.002Z,8.0,1970-01-01T00:00:00.003Z,9.0,", + "0,d1,1970-01-01T00:00:00.005Z,0.0,1970-01-01T00:00:00.005Z,0.0,", + "0,d2,null,null,null,null," + }, + DATABASE_NAME); + } + + @Test + public void testCountWindowAndTimeWindowOutput() { + tableResultSetEqualTest( + "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') " + + "PARTITION BY device ORDER BY time, TIMECOL => 'time', SIZE => 2) " + + "ORDER BY window_index", + new String[] {"window_index", "device", "s1_time", "s1"}, + new String[] { + "0,d1,1970-01-01T00:00:00.001Z,1.0,", + "1,d1,1970-01-01T00:00:00.002Z,8.0,", + "2,d1,1970-01-01T00:00:00.004Z,0.0," + }, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') " + + "PARTITION BY device ORDER BY time, TIMECOL => 'time', SIZE => 2ms, " + + "ORIGIN => 1970-01-01T00:00:00.000+00:00) ORDER BY window_start", + new String[] {"window_start", "window_end", "device", "s1_time", "s1"}, + new String[] { + "1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.002Z,d1,1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.004Z,d1,1970-01-01T00:00:00.002Z,8.0,", + "1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.004Z,0.0," + }, + DATABASE_NAME); + } + + @Test + public void testInvalidParameterCombinations() { + String prefix = + "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time, TIMECOL => 'time', "; + tableAssertTestFail( + "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time, TIMECOL => 'time')", + "701: Exactly one of the N and SIZE arguments must be specified for LTTB.", + DATABASE_NAME); + tableAssertTestFail( + prefix + "N => 3, SIZE => 2)", + "701: Exactly one of the N and SIZE arguments must be specified for LTTB.", + DATABASE_NAME); + tableAssertTestFail( + prefix + "N => 2)", "701: The N argument of LTTB must be at least 3.", DATABASE_NAME); + tableAssertTestFail( + prefix + "N => 3, SLIDE => 2)", + "701: The N argument of LTTB cannot be combined with the SLIDE or ORIGIN arguments.", + DATABASE_NAME); + tableAssertTestFail( + prefix + "SIZE => 2, ORIGIN => 1970-01-01T00:00:00.000+00:00)", + "701: The ORIGIN argument is only supported in time window mode.", + DATABASE_NAME); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index b5c394539f217..fc5cec448bd8a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -22,6 +22,7 @@ import org.apache.iotdb.calc.plan.relational.metadata.CommonMetadataUtils; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.i18n.CommonMessages; import org.apache.iotdb.commons.i18n.QueryMessages; import org.apache.iotdb.commons.queryengine.common.SessionInfo; import org.apache.iotdb.commons.queryengine.plan.relational.analyzer.NodeRef; @@ -122,6 +123,7 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.udf.builtin.relational.tvf.FFTTableFunction; +import org.apache.iotdb.commons.udf.builtin.relational.tvf.LTTBTableFunction; import org.apache.iotdb.commons.udf.builtin.relational.tvf.M4TableFunction; import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; @@ -5916,6 +5918,7 @@ private ArgumentsAnalysis analyzeArguments( } } tryAppendM4ModeArgument(functionName, arguments, parameterSpecifications, passedArguments); + tryAppendLTTBModeArgument(functionName, arguments, parameterSpecifications, passedArguments); tryAppendFFTInternalArguments( functionName, arguments, parameterSpecifications, passedArguments); return new ArgumentsAnalysis(passedArguments.buildOrThrow(), tableArgumentAnalyses.build()); @@ -5923,6 +5926,7 @@ private ArgumentsAnalysis analyzeArguments( private boolean isPartitionColumnsProvidedByProperSchema(String functionName) { return TableBuiltinTableFunction.M4.getFunctionName().equalsIgnoreCase(functionName) + || TableBuiltinTableFunction.LTTB.getFunctionName().equalsIgnoreCase(functionName) || TableBuiltinTableFunction.LOWPASS.getFunctionName().equalsIgnoreCase(functionName) || TableBuiltinTableFunction.HIGHPASS.getFunctionName().equalsIgnoreCase(functionName) || TableBuiltinTableFunction.XCORR.getFunctionName().equalsIgnoreCase(functionName) @@ -6034,6 +6038,87 @@ && containsTableFunctionArgument( new ScalarArgument(org.apache.iotdb.udf.api.type.Type.BOOLEAN, isTimeWindow)); } + /** + * Resolves the LTTB execution mode from the SQL literals. The mode (target-count vs. time + * window vs. count window) depends on whether {@code SIZE} is a duration literal, which is only + * visible at this level, so it is injected as the internal boolean argument {@link + * LTTBTableFunction#WINDOW_MODE_PARAMETER_NAME}. Parameter-combination checks that only need + * the presence of arguments are also performed here so they fail before analysis. + */ + private void tryAppendLTTBModeArgument( + String functionName, + List arguments, + List parameterSpecifications, + ImmutableMap.Builder passedArguments) { + if (!TableBuiltinTableFunction.LTTB.getFunctionName().equalsIgnoreCase(functionName)) { + return; + } + + Optional nArgument = + findOptionalTableFunctionArgument( + arguments, parameterSpecifications, LTTBTableFunction.N_PARAMETER_NAME); + Optional sizeArgument = + findOptionalTableFunctionArgument( + arguments, parameterSpecifications, LTTBTableFunction.SIZE_PARAMETER_NAME); + Optional slideArgument = + findOptionalTableFunctionArgument( + arguments, parameterSpecifications, LTTBTableFunction.SLIDE_PARAMETER_NAME); + boolean originSpecified = + containsTableFunctionArgument( + arguments, parameterSpecifications, LTTBTableFunction.ORIGIN_PARAMETER_NAME); + + if (nArgument.isPresent() == sizeArgument.isPresent()) { + throw new SemanticException( + CommonMessages + .EXCEPTION_EXACTLY_ONE_OF_THE_N_AND_SIZE_ARGUMENTS_MUST_BE_SPECIFIED_FOR_LTTB_54AF0733); + } + + boolean isTimeWindow = false; + if (nArgument.isPresent()) { + if (slideArgument.isPresent() || originSpecified) { + throw new SemanticException( + CommonMessages + .EXCEPTION_THE_N_ARGUMENT_OF_LTTB_CANNOT_BE_COMBINED_WITH_THE_SLIDE_OR_ORIGIN_ARGUMENTS_C2FF1FE1); + } + if (!(nArgument.get().getValue() instanceof Expression) + || nArgument.get().getValue() instanceof TimeDurationLiteral) { + throw new SemanticException( + CommonMessages.EXCEPTION_THE_N_ARGUMENT_OF_LTTB_MUST_BE_A_POSITIVE_INTEGER_21795D8E); + } + } else { + if (!(sizeArgument.get().getValue() instanceof Expression)) { + throw new SemanticException( + String.format( + QueryMessages.EXCEPTION_INVALID_ARGUMENT_ARG_EXPECTED_SCALAR_ARGUMENT_4AD4E32F, + LTTBTableFunction.SIZE_PARAMETER_NAME)); + } + isTimeWindow = sizeArgument.get().getValue() instanceof TimeDurationLiteral; + if (slideArgument.isPresent()) { + if (!(slideArgument.get().getValue() instanceof Expression)) { + throw new SemanticException( + String.format( + QueryMessages.EXCEPTION_INVALID_ARGUMENT_ARG_EXPECTED_SCALAR_ARGUMENT_4AD4E32F, + LTTBTableFunction.SLIDE_PARAMETER_NAME)); + } + boolean isTimeSlide = slideArgument.get().getValue() instanceof TimeDurationLiteral; + if (isTimeWindow != isTimeSlide) { + throw new SemanticException( + QueryMessages + .EXCEPTION_THE_SLIDE_ARGUMENT_MUST_HAVE_THE_SAME_WINDOW_MODE_AS_THE_SIZE_ARGUMENT_419C9844); + } + } + if (!isTimeWindow && originSpecified) { + throw new SemanticException( + QueryMessages + .EXCEPTION_THE_ORIGIN_ARGUMENT_IS_ONLY_SUPPORTED_IN_TIME_WINDOW_MODE_B3F358B9); + } + } + validateM4OrderBySortOrder(arguments, parameterSpecifications); + passedArguments.put( + LTTBTableFunction.WINDOW_MODE_PARAMETER_NAME, + new ScalarArgument(org.apache.iotdb.udf.api.type.Type.BOOLEAN, isTimeWindow)); + } + private void validateM4OrderBySortOrder( List arguments, List parameterSpecifications) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 6f5f3c278ed3d..f17d5d9a24e66 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -1610,6 +1610,7 @@ public RelationPlan visitTableFunctionInvocation(TableFunctionInvocation node, V private boolean needAddPartitionColumn(String functionName) { return !TableBuiltinTableFunction.M4.getFunctionName().equalsIgnoreCase(functionName) + && !TableBuiltinTableFunction.LTTB.getFunctionName().equalsIgnoreCase(functionName) && !TableBuiltinTableFunction.LOWPASS.getFunctionName().equalsIgnoreCase(functionName) && !TableBuiltinTableFunction.HIGHPASS.getFunctionName().equalsIgnoreCase(functionName) && !TableBuiltinTableFunction.XCORR.getFunctionName().equalsIgnoreCase(functionName) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 00db7b95e4924..a67c1c8f01c30 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -2899,7 +2899,10 @@ private boolean canSplitTableFunctionProcessor(TableFunctionProcessorNode node) if (!isPartitionedGroup(node.getChild())) { return false; } - return !TableBuiltinTableFunction.FFT.getFunctionName().equalsIgnoreCase(node.getName()); + // LTTB is not mergeable: fragment-local samples cannot be combined into a globally correct + // result, so the whole partition must be gathered and ordered before the function runs. + return !TableBuiltinTableFunction.FFT.getFunctionName().equalsIgnoreCase(node.getName()) + && !TableBuiltinTableFunction.LTTB.getFunctionName().equalsIgnoreCase(node.getName()); } private boolean isPartitionedGroup(PlanNode node) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java index 637113026f4f0..13a579a403adc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.queryengine.plan.relational.function.tvf.ForecastTableFunction; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.commons.udf.builtin.relational.tvf.FFTTableFunction; +import org.apache.iotdb.commons.udf.builtin.relational.tvf.LTTBTableFunction; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; import org.apache.iotdb.db.queryengine.plan.relational.planner.PlanTester; import org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern; @@ -842,4 +843,161 @@ public void testM4RejectsDescendingOrderBy() { e.getMessage()); } } + + // Aliases use the planner's actual symbol names so the same matcher also resolves them through + // the ExchangeNodes of the distributed plan (which bind symbols to their own names). + private static final ImmutableMap TABLE1_COLUMNS = + ImmutableMap.builder() + .put("time", "time") + .put("tag1_0", "tag1") + .put("tag2_1", "tag2") + .put("tag3_2", "tag3") + .put("attr1_3", "attr1") + .put("attr2_4", "attr2") + .put("s1_5", "s1") + .put("s2_6", "s2") + .put("s3_7", "s3") + .buildOrThrow(); + + private static final String LTTB_DATA_ARGUMENT = + "DATA => table1 PARTITION BY (tag1, tag2, tag3, attr1, attr2) ORDER BY time"; + + private static MapTableFunctionHandle.Builder lttbHandle(String mode) { + return new MapTableFunctionHandle.Builder() + .addProperty(LTTBTableFunction.MODE_PROPERTY, mode) + .addProperty( + LTTBTableFunction.PARTITION_TYPES_PROPERTY, "STRING,STRING,STRING,STRING,STRING") + .addProperty(LTTBTableFunction.PARTICIPANT_TYPES_PROPERTY, "INT64,INT64,DOUBLE"); + } + + private static TableFunctionProcessorMatcher.Builder lttbMatcher( + TableFunctionProcessorMatcher.Builder builder, String... windowColumns) { + ImmutableList.Builder properOutputs = ImmutableList.builder(); + properOutputs.add(windowColumns); + properOutputs.add( + "lttb_tag1", + "lttb_tag2", + "lttb_tag3", + "lttb_attr1", + "lttb_attr2", + "lttb_s1_time", + "lttb_s1", + "lttb_s2_time", + "lttb_s2", + "lttb_s3_time", + "lttb_s3"); + return builder + .name("lttb") + .properOutputs(properOutputs.build().toArray(new String[0])) + .requiredSymbols( + "time", "tag1_0", "tag2_1", "tag3_2", "attr1_3", "attr2_4", "s1_5", "s2_6", "s3_7"); + } + + @Test + public void testLTTBTargetCountMode() { + PlanTester planTester = new PlanTester(); + String sql = "SELECT * FROM LTTB(" + LTTB_DATA_ARGUMENT + ", TIMECOL => 'time', N => 100)"; + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + PlanMatchPattern tableScan = tableScan("testdb.table1", TABLE1_COLUMNS); + + Consumer tableFunctionMatcher = + builder -> + lttbMatcher(builder, "window_index") + .handle( + lttbHandle("TARGET_COUNT") + .addProperty(LTTBTableFunction.N_PARAMETER_NAME, 100L) + .build()); + + // PARTITION BY several columns plans a GroupNode (a SortNode subclass) below the function + assertPlan( + logicalQueryPlan, anyTree(tableFunctionProcessor(tableFunctionMatcher, group(tableScan)))); + // LTTB is not mergeable: the whole partition is gathered and ordered before the function runs + assertPlan( + planTester.getFragmentPlan(0), + output( + tableFunctionProcessor( + tableFunctionMatcher, mergeSort(exchange(), exchange(), exchange())))); + } + + @Test + public void testLTTBTimeWindowMode() { + PlanTester planTester = new PlanTester(); + String sql = + "SELECT * FROM LTTB(" + + LTTB_DATA_ARGUMENT + + ", TIMECOL => 'time', SIZE => 1h, SLIDE => 30m, " + + "ORIGIN => 1970-01-01T00:00:00.000+00:00)"; + Consumer tableFunctionMatcher = + builder -> + lttbMatcher(builder, "window_start", "window_end") + .handle( + lttbHandle("TIME_WINDOW") + .addProperty(LTTBTableFunction.SIZE_PARAMETER_NAME, 3600000L) + .addProperty(LTTBTableFunction.SLIDE_PARAMETER_NAME, 1800000L) + .addProperty(LTTBTableFunction.ORIGIN_PARAMETER_NAME, 0L) + .build()); + assertPlan( + planTester.createPlan(sql), + anyTree( + tableFunctionProcessor( + tableFunctionMatcher, group(tableScan("testdb.table1", TABLE1_COLUMNS))))); + } + + @Test + public void testLTTBCountWindowMode() { + PlanTester planTester = new PlanTester(); + String sql = "SELECT * FROM LTTB(" + LTTB_DATA_ARGUMENT + ", TIMECOL => 'time', SIZE => 5)"; + Consumer tableFunctionMatcher = + builder -> + lttbMatcher(builder, "window_index") + .handle( + lttbHandle("COUNT_WINDOW") + .addProperty(LTTBTableFunction.SIZE_PARAMETER_NAME, 5L) + .addProperty(LTTBTableFunction.SLIDE_PARAMETER_NAME, 5L) + .build()); + assertPlan( + planTester.createPlan(sql), + anyTree( + tableFunctionProcessor( + tableFunctionMatcher, group(tableScan("testdb.table1", TABLE1_COLUMNS))))); + } + + @Test + public void testLTTBRejectsInvalidArguments() { + String data = "DATA => (SELECT time, tag1, s3 FROM table1) PARTITION BY tag1 ORDER BY time"; + assertAnalyzeFails( + "SELECT * FROM LTTB(" + data + ", TIMECOL => 'time')", + "Exactly one of the N and SIZE arguments must be specified for LTTB."); + assertAnalyzeFails( + "SELECT * FROM LTTB(" + data + ", TIMECOL => 'time', N => 10, SIZE => 1h)", + "Exactly one of the N and SIZE arguments must be specified for LTTB."); + assertAnalyzeFails( + "SELECT * FROM LTTB(" + data + ", TIMECOL => 'time', N => 2)", + "The N argument of LTTB must be at least 3."); + assertAnalyzeFails( + "SELECT * FROM LTTB(" + data + ", TIMECOL => 'time', N => 1h)", + "The N argument of LTTB must be a positive integer."); + assertAnalyzeFails( + "SELECT * FROM LTTB(" + + data + + ", TIMECOL => 'time', N => 10, ORIGIN => 1970-01-01T00:00:00.000+00:00)", + "The N argument of LTTB cannot be combined with the SLIDE or ORIGIN arguments."); + assertAnalyzeFails( + "SELECT * FROM LTTB(" + data + ", TIMECOL => 'time', SLIDE => 1h)", + "Exactly one of the N and SIZE arguments must be specified for LTTB."); + assertAnalyzeFails( + "SELECT * FROM LTTB(" + + data + + ", TIMECOL => 'time', SIZE => 5, ORIGIN => 1970-01-01T00:00:00.000+00:00)", + "The ORIGIN argument is only supported in time window mode."); + assertAnalyzeFails( + "SELECT * FROM LTTB(" + data + ", TIMECOL => 'time', SIZE => 1h, SLIDE => 5)", + "The SLIDE argument must have the same window mode as the SIZE argument."); + assertAnalyzeFails( + "SELECT * FROM LTTB(DATA => (SELECT time, tag1, s3 FROM table1) PARTITION BY tag1 ORDER BY time DESC, TIMECOL => 'time', N => 10)", + "The ORDER BY clause of the DATA argument must sort the time column in ascending order."); + assertAnalyzeFails( + "SELECT * FROM LTTB(DATA => (SELECT time, tag1, attr1, s3 FROM table1) PARTITION BY tag1 ORDER BY time, TIMECOL => 'time', N => 10)", + "Only column with double, float, int32, int64 can be calculated by the function, attr1 is the STRING."); + } } diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java index a6b958fd1420f..5200e2da0481f 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -342,6 +342,25 @@ private CommonMessages() {} EXCEPTION_XCORR_REQUIRES_EXACTLY_TWO_CALCULATION_COLUMNS_BUT_FOUND_ARG_2FF8EB0C = "XCorr requires exactly two calculation columns, but found %d."; public static final String EXCEPTION_COLUMN_LACK_OF_NAME = "the column in table lack of the name"; + public static final String + EXCEPTION_EXACTLY_ONE_OF_THE_N_AND_SIZE_ARGUMENTS_MUST_BE_SPECIFIED_FOR_LTTB_54AF0733 = + "Exactly one of the N and SIZE arguments must be specified for LTTB."; + public static final String EXCEPTION_THE_N_ARGUMENT_OF_LTTB_MUST_BE_AT_LEAST_3_29CE2B87 = + "The N argument of LTTB must be at least 3."; + public static final String + EXCEPTION_THE_N_ARGUMENT_OF_LTTB_MUST_BE_A_POSITIVE_INTEGER_21795D8E = + "The N argument of LTTB must be a positive integer."; + public static final String + EXCEPTION_THE_N_ARGUMENT_OF_LTTB_CANNOT_BE_COMBINED_WITH_THE_SLIDE_OR_ORIGIN_ARGUMENTS_C2FF1FE1 = + "The N argument of LTTB cannot be combined with the SLIDE or ORIGIN arguments."; + public static final String + EXCEPTION_INVALID_SCALAR_ARGUMENT_SIZE_SHOULD_BE_A_POSITIVE_VALUE_3ECF76E0 = + "Invalid scalar argument SIZE, should be a positive value"; + public static final String + EXCEPTION_LTTB_BUFFERS_AT_MOST_ARG_ELIGIBLE_POINTS_PER_COLUMN_IN_ONE_PARTITION_OR_WINDOW_4C746D82 = + "LTTB buffers at most %d eligible points per column in one partition or window."; + public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = + "Unsupported LTTB value type: %s"; public static final String EXCEPTION_SNAPSHOT_BUFFER_SIZE_MUST_NOT_EXCEED_ARG_BYTES_BUT_WAS_ARG_D1DA6F7E = "Snapshot buffer size must not exceed %d bytes, but was %d."; diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java index c3af2dbe7e24a..9b44c9b590cc0 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -238,6 +238,25 @@ private CommonMessages() {} EXCEPTION_XCORR_REQUIRES_EXACTLY_TWO_CALCULATION_COLUMNS_BUT_FOUND_ARG_2FF8EB0C = "XCorr 要求必须正好有两列计算列,但实际找到 %d 列。"; public static final String EXCEPTION_COLUMN_LACK_OF_NAME = "表参数中列缺少名字"; + public static final String + EXCEPTION_EXACTLY_ONE_OF_THE_N_AND_SIZE_ARGUMENTS_MUST_BE_SPECIFIED_FOR_LTTB_54AF0733 = + "LTTB 必须且只能指定 N 和 SIZE 参数中的一个。"; + public static final String EXCEPTION_THE_N_ARGUMENT_OF_LTTB_MUST_BE_AT_LEAST_3_29CE2B87 = + "LTTB 的 N 参数必须至少为 3。"; + public static final String + EXCEPTION_THE_N_ARGUMENT_OF_LTTB_MUST_BE_A_POSITIVE_INTEGER_21795D8E = + "LTTB 的 N 参数必须是正整数。"; + public static final String + EXCEPTION_THE_N_ARGUMENT_OF_LTTB_CANNOT_BE_COMBINED_WITH_THE_SLIDE_OR_ORIGIN_ARGUMENTS_C2FF1FE1 = + "LTTB 的 N 参数不能与 SLIDE 或 ORIGIN 参数同时使用。"; + public static final String + EXCEPTION_INVALID_SCALAR_ARGUMENT_SIZE_SHOULD_BE_A_POSITIVE_VALUE_3ECF76E0 = + "无效的标量参数 SIZE,应为正数"; + public static final String + EXCEPTION_LTTB_BUFFERS_AT_MOST_ARG_ELIGIBLE_POINTS_PER_COLUMN_IN_ONE_PARTITION_OR_WINDOW_4C746D82 = + "LTTB 在单个分区或窗口内每列最多缓存 %d 个有效点。"; + public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = + "不支持的 LTTB 值类型:%s"; public static final String EXCEPTION_SNAPSHOT_BUFFER_SIZE_MUST_NOT_EXCEED_ARG_BYTES_BUT_WAS_ARG_D1DA6F7E = "快照缓冲区大小不得超过 %d 字节,但实际为 %d。"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/function/TableBuiltinTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/function/TableBuiltinTableFunction.java index 818747bb71494..69e7b38d53757 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/function/TableBuiltinTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/function/TableBuiltinTableFunction.java @@ -28,6 +28,7 @@ import org.apache.iotdb.commons.udf.builtin.relational.tvf.FFTTableFunction; import org.apache.iotdb.commons.udf.builtin.relational.tvf.HOPTableFunction; import org.apache.iotdb.commons.udf.builtin.relational.tvf.HighPassTableFunction; +import org.apache.iotdb.commons.udf.builtin.relational.tvf.LTTBTableFunction; import org.apache.iotdb.commons.udf.builtin.relational.tvf.LowPassTableFunction; import org.apache.iotdb.commons.udf.builtin.relational.tvf.M4TableFunction; import org.apache.iotdb.commons.udf.builtin.relational.tvf.SessionTableFunction; @@ -49,6 +50,7 @@ public enum TableBuiltinTableFunction { VARIATION("variation"), CAPACITY("capacity"), M4("m4"), + LTTB("lttb"), FFT("fft"), FORECAST("forecast"), PATTERN_MATCH("pattern_match"), @@ -99,6 +101,8 @@ public static TableFunction getBuiltinTableFunction(String functionName) { return new CapacityTableFunction(); case "m4": return new M4TableFunction(); + case "lttb": + return new LTTBTableFunction(); case "fft": return new FFTTableFunction(); case "forecast": diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java new file mode 100644 index 0000000000000..0a161fc236128 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -0,0 +1,930 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.udf.builtin.relational.tvf; + +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.i18n.CommonMessages; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; +import org.apache.iotdb.udf.api.relational.table.TableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider; +import org.apache.iotdb.udf.api.relational.table.argument.Argument; +import org.apache.iotdb.udf.api.relational.table.argument.DescribedSchema; +import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; +import org.apache.iotdb.udf.api.relational.table.argument.TableArgument; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; +import org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.TableParameterSpecification; +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.block.column.ColumnBuilder; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Deque; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.iotdb.udf.api.relational.table.argument.ScalarArgumentChecker.POSITIVE_LONG_CHECKER; + +/** + * Largest-Triangle-Three-Buckets (LTTB) downsampling as a built-in Table Model table function. + * + *

The function shares its table-argument conventions with {@link M4TableFunction}: {@code DATA} + * has set semantics, {@code TIMECOL} names the {@code TIMESTAMP} ordering column, partition columns + * are echoed into the proper output, and every other numeric column is a participant column that is + * downsampled independently. Two mutually exclusive execution modes exist: + * + *

    + *
  • Target-count mode ({@code N}): the whole partition is buffered and the standard LTTB + * algorithm reduces each participant column to exactly {@code N} points (or all points when + * the column has no more than {@code N} eligible points). The output uses a fixed {@code + * window_index = 0}. + *
  • Window/bucket mode ({@code SIZE}, optional {@code SLIDE}/{@code ORIGIN}): windows + * are built exactly like M4 (time windows for duration literals, count windows for integers). + * Each window is one LTTB bucket that selects at most one point per participant column: the + * anchor is the previously selected point of that column, the candidates are the eligible + * points in the window, and the third vertex is the average point of the next window that + * carries data for that column (or the window's own last eligible point for the final + * window). + *
+ * + *

{@code NULL} and non-finite participant values are ignored per column. Ties in the triangle + * area are broken deterministically by the earliest timestamp (stable input order). + */ +public class LTTBTableFunction implements TableFunction { + + public static final String DATA_PARAMETER_NAME = "DATA"; + public static final String TIMECOL_PARAMETER_NAME = "TIMECOL"; + public static final String N_PARAMETER_NAME = "N"; + public static final String SIZE_PARAMETER_NAME = "SIZE"; + public static final String SLIDE_PARAMETER_NAME = "SLIDE"; + public static final String ORIGIN_PARAMETER_NAME = "ORIGIN"; + + /** Internal boolean argument injected by the analyzer: {@code true} if SIZE is a duration. */ + public static final String WINDOW_MODE_PARAMETER_NAME = "__LTTB_WINDOW_MODE"; + + public static final long UNSPECIFIED_N = Long.MIN_VALUE; + public static final long UNSPECIFIED_SIZE = Long.MIN_VALUE; + public static final long UNSPECIFIED_SLIDE = Long.MIN_VALUE; + public static final long MIN_TARGET_COUNT = 3L; + + /** Hard cap on eligible points buffered per participant column in one partition or window. */ + public static final int MAX_BUFFERED_POINTS = 1 << 22; + + public static final String MODE_PROPERTY = "__LTTB_MODE"; + public static final String PARTITION_TYPES_PROPERTY = "__LTTB_PARTITION_TYPES"; + public static final String PARTICIPANT_TYPES_PROPERTY = "__LTTB_PARTICIPANT_TYPES"; + + private static final String OUTPUT_WINDOW_START_COLUMN = "window_start"; + private static final String OUTPUT_WINDOW_END_COLUMN = "window_end"; + private static final String OUTPUT_WINDOW_INDEX_COLUMN = "window_index"; + private static final String TIME_SUFFIX = "_time"; + private static final int INITIAL_CAPACITY = 64; + private static final int NOT_SELECTED = -1; + + /** Execution mode persisted in the {@link MapTableFunctionHandle}. */ + public enum Mode { + TARGET_COUNT, + TIME_WINDOW, + COUNT_WINDOW + } + + @Override + public List getArgumentsSpecifications() { + return Arrays.asList( + TableParameterSpecification.builder().name(DATA_PARAMETER_NAME).setSemantics().build(), + ScalarParameterSpecification.builder() + .name(TIMECOL_PARAMETER_NAME) + .type(Type.STRING) + .build(), + ScalarParameterSpecification.builder() + .name(N_PARAMETER_NAME) + .type(Type.INT64) + .defaultValue(UNSPECIFIED_N) + .build(), + ScalarParameterSpecification.builder() + .name(SIZE_PARAMETER_NAME) + .type(Type.INT64) + .defaultValue(UNSPECIFIED_SIZE) + .addChecker(POSITIVE_LONG_CHECKER) + .build(), + ScalarParameterSpecification.builder() + .name(SLIDE_PARAMETER_NAME) + .type(Type.INT64) + .defaultValue(UNSPECIFIED_SLIDE) + .addChecker(POSITIVE_LONG_CHECKER) + .build(), + ScalarParameterSpecification.builder() + .name(ORIGIN_PARAMETER_NAME) + .type(Type.TIMESTAMP) + .defaultValue(0L) + .build()); + } + + @Override + public TableFunctionAnalysis analyze(Map arguments) throws UDFException { + int timeColumnIndex = + WindowTVFUtils.checkOrderByColumn(arguments, DATA_PARAMETER_NAME, TIMECOL_PARAMETER_NAME); + TableArgument tableArgument = (TableArgument) arguments.get(DATA_PARAMETER_NAME); + List partitionIndexes = WindowTVFUtils.getPartitionIndexes(tableArgument); + Set excludedIndexes = new HashSet<>(partitionIndexes); + excludedIndexes.add(timeColumnIndex); + + long n = getLong(arguments, N_PARAMETER_NAME); + long size = getLong(arguments, SIZE_PARAMETER_NAME); + long slide = getLong(arguments, SLIDE_PARAMETER_NAME); + boolean nSpecified = n != UNSPECIFIED_N; + boolean sizeSpecified = size != UNSPECIFIED_SIZE; + boolean slideSpecified = slide != UNSPECIFIED_SLIDE; + if (nSpecified == sizeSpecified) { + throw new SemanticException( + CommonMessages + .EXCEPTION_EXACTLY_ONE_OF_THE_N_AND_SIZE_ARGUMENTS_MUST_BE_SPECIFIED_FOR_LTTB_54AF0733); + } + + Mode mode; + if (nSpecified) { + if (slideSpecified) { + throw new SemanticException( + CommonMessages + .EXCEPTION_THE_N_ARGUMENT_OF_LTTB_CANNOT_BE_COMBINED_WITH_THE_SLIDE_OR_ORIGIN_ARGUMENTS_C2FF1FE1); + } + if (n < MIN_TARGET_COUNT) { + throw new SemanticException( + CommonMessages.EXCEPTION_THE_N_ARGUMENT_OF_LTTB_MUST_BE_AT_LEAST_3_29CE2B87); + } + mode = Mode.TARGET_COUNT; + } else { + if (size <= 0) { + throw new UDFException( + CommonMessages + .EXCEPTION_INVALID_SCALAR_ARGUMENT_SIZE_SHOULD_BE_A_POSITIVE_VALUE_3ECF76E0); + } + if (!slideSpecified) { + slide = size; + } else if (slide <= 0) { + throw new UDFException( + CommonMessages + .EXCEPTION_INVALID_SCALAR_ARGUMENT_SLIDE_SHOULD_BE_A_POSITIVE_VALUE_F019E091); + } + boolean isTimeWindow = + arguments.containsKey(WINDOW_MODE_PARAMETER_NAME) + && (boolean) ((ScalarArgument) arguments.get(WINDOW_MODE_PARAMETER_NAME)).getValue(); + mode = isTimeWindow ? Mode.TIME_WINDOW : Mode.COUNT_WINDOW; + } + + DescribedSchema.Builder schemaBuilder = new DescribedSchema.Builder(); + if (mode == Mode.TIME_WINDOW) { + schemaBuilder + .addField(OUTPUT_WINDOW_START_COLUMN, Type.TIMESTAMP) + .addField(OUTPUT_WINDOW_END_COLUMN, Type.TIMESTAMP); + } else { + schemaBuilder.addField(OUTPUT_WINDOW_INDEX_COLUMN, Type.INT64); + } + + List partitionTypes = new ArrayList<>(); + for (int partitionIndex : partitionIndexes) { + Type type = tableArgument.getFieldTypes().get(partitionIndex); + partitionTypes.add(type); + schemaBuilder.addField(tableArgument.getFieldNames().get(partitionIndex).get(), type); + } + + // Rejects every non-numeric participant column with the shared "not allowed columns" message. + List participantIndexes = + WindowTVFUtils.getCalculationIndexes(tableArgument, excludedIndexes, null); + if (participantIndexes.isEmpty()) { + throw new SemanticException(CommonMessages.EXCEPTION_NO_CALCULATE_COLUMNS); + } + List participantTypes = new ArrayList<>(); + for (int participantIndex : participantIndexes) { + Type type = tableArgument.getFieldTypes().get(participantIndex); + String columnName = tableArgument.getFieldNames().get(participantIndex).get(); + participantTypes.add(type); + schemaBuilder.addField(columnName + TIME_SUFFIX, Type.TIMESTAMP); + schemaBuilder.addField(columnName, type); + } + + MapTableFunctionHandle.Builder handleBuilder = + new MapTableFunctionHandle.Builder() + .addProperty(MODE_PROPERTY, mode.name()) + .addProperty(PARTITION_TYPES_PROPERTY, WindowTVFUtils.joinTypes(partitionTypes)) + .addProperty(PARTICIPANT_TYPES_PROPERTY, WindowTVFUtils.joinTypes(participantTypes)); + if (mode == Mode.TARGET_COUNT) { + handleBuilder.addProperty(N_PARAMETER_NAME, n); + } else { + handleBuilder.addProperty(SIZE_PARAMETER_NAME, size).addProperty(SLIDE_PARAMETER_NAME, slide); + if (mode == Mode.TIME_WINDOW) { + handleBuilder.addProperty(ORIGIN_PARAMETER_NAME, getLong(arguments, ORIGIN_PARAMETER_NAME)); + } + } + + List requiredColumns = new ArrayList<>(); + requiredColumns.add(timeColumnIndex); + requiredColumns.addAll(partitionIndexes); + requiredColumns.addAll(participantIndexes); + + return TableFunctionAnalysis.builder() + .properColumnSchema(schemaBuilder.build()) + .requireRecordSnapshot(false) + .requiredColumns(DATA_PARAMETER_NAME, requiredColumns) + .handle(handleBuilder.build()) + .build(); + } + + private static long getLong(Map arguments, String name) { + return (long) ((ScalarArgument) arguments.get(name)).getValue(); + } + + @Override + public TableFunctionHandle createTableFunctionHandle() { + return new MapTableFunctionHandle(); + } + + @Override + public TableFunctionProcessorProvider getProcessorProvider( + TableFunctionHandle tableFunctionHandle) { + MapTableFunctionHandle handle = (MapTableFunctionHandle) tableFunctionHandle; + Mode mode = Mode.valueOf((String) handle.getProperty(MODE_PROPERTY)); + Type[] partitionTypes = + WindowTVFUtils.parseTypes((String) handle.getProperty(PARTITION_TYPES_PROPERTY)); + Type[] participantTypes = + WindowTVFUtils.parseTypes((String) handle.getProperty(PARTICIPANT_TYPES_PROPERTY)); + + return new TableFunctionProcessorProvider() { + @Override + public TableFunctionDataProcessor getDataProcessor() { + ParticipantColumn[] participantColumns = + createParticipantColumns(participantTypes, partitionTypes.length + 1); + switch (mode) { + case TARGET_COUNT: + return new TargetCountLTTBDataProcessor( + (long) handle.getProperty(N_PARAMETER_NAME), partitionTypes, participantColumns); + case TIME_WINDOW: + return new TimeWindowLTTBDataProcessor( + (long) handle.getProperty(SIZE_PARAMETER_NAME), + (long) handle.getProperty(SLIDE_PARAMETER_NAME), + (long) handle.getProperty(ORIGIN_PARAMETER_NAME), + partitionTypes, + participantColumns); + case COUNT_WINDOW: + return new CountWindowLTTBDataProcessor( + (long) handle.getProperty(SIZE_PARAMETER_NAME), + (long) handle.getProperty(SLIDE_PARAMETER_NAME), + partitionTypes, + participantColumns); + default: + throw new IllegalStateException(String.valueOf(mode)); + } + } + }; + } + + private static ParticipantColumn[] createParticipantColumns(Type[] types, int firstInputIndex) { + ParticipantColumn[] columns = new ParticipantColumn[types.length]; + for (int i = 0; i < types.length; i++) { + columns[i] = new ParticipantColumn(firstInputIndex + i, types[i]); + } + return columns; + } + + // --------------------------------------------------------------------------------------------- + // LTTB core + // --------------------------------------------------------------------------------------------- + + /** + * Standard LTTB selection over an ordered point sequence. + * + *

When {@code count <= n} every index is returned. Otherwise the first and last points are + * retained and the {@code count - 2} intermediate points are divided into {@code n - 2} buckets; + * bucket {@code i} covers {@code [floor(i*(count-2)/(n-2))+1, floor((i+1)*(count-2)/(n-2))+1)}. + * For every bucket the candidate forming the largest triangle with the previously selected point + * and the average point of the next bucket wins (the last bucket uses the final point). Ties are + * broken by the earliest timestamp. + * + * @return ascending indexes of the selected points + */ + static int[] selectTargetCount(long[] times, double[] values, int count, long n) { + if (count <= n) { + int[] all = new int[count]; + for (int i = 0; i < count; i++) { + all[i] = i; + } + return all; + } + + int[] selected = new int[(int) n]; + int selectedCount = 0; + selected[selectedCount++] = 0; + long bucketCount = n - 2; + long intermediate = count - 2L; + int anchor = 0; + for (long bucket = 0; bucket < bucketCount; bucket++) { + int currentStart = (int) (bucket * intermediate / bucketCount) + 1; + int currentEnd = (int) ((bucket + 1) * intermediate / bucketCount) + 1; + + double avgTime; + double avgValue; + if (bucket == bucketCount - 1) { + avgTime = times[count - 1]; + avgValue = values[count - 1]; + } else { + int nextEnd = (int) ((bucket + 2) * intermediate / bucketCount) + 1; + double sumTime = 0; + double sumValue = 0; + for (int i = currentEnd; i < nextEnd; i++) { + sumTime += times[i]; + sumValue += values[i]; + } + int nextSize = nextEnd - currentEnd; + avgTime = sumTime / nextSize; + avgValue = sumValue / nextSize; + } + + int pick = + argMaxArea( + times, + values, + currentStart, + currentEnd, + times[anchor], + values[anchor], + avgTime, + avgValue); + selected[selectedCount++] = pick; + anchor = pick; + } + selected[selectedCount] = count - 1; + return selected; + } + + /** Returns the index in {@code [start, end)} maximizing the triangle area; earliest wins ties. */ + static int argMaxArea( + long[] times, + double[] values, + int start, + int end, + double anchorTime, + double anchorValue, + double nextTime, + double nextValue) { + int pick = start; + double maxArea = -1; + for (int i = start; i < end; i++) { + double area = + Math.abs( + (anchorTime - nextTime) * (values[i] - anchorValue) + - (anchorTime - times[i]) * (nextValue - anchorValue)); + if (area > maxArea) { + maxArea = area; + pick = i; + } + } + return pick; + } + + // --------------------------------------------------------------------------------------------- + // Column access and buffering + // --------------------------------------------------------------------------------------------- + + private static final class ParticipantColumn { + private final int inputIndex; + private final Type type; + + private ParticipantColumn(int inputIndex, Type type) { + this.inputIndex = inputIndex; + this.type = type; + } + + private boolean isIntegral() { + return type == Type.INT32 || type == Type.INT64; + } + + /** Appends the record's value to {@code buffer} if it is a non-null finite number. */ + private void collect(Record record, long time, PointBuffer buffer) { + if (record.isNull(inputIndex)) { + return; + } + switch (type) { + case INT32: + buffer.addIntegral(time, record.getInt(inputIndex)); + break; + case INT64: + buffer.addIntegral(time, record.getLong(inputIndex)); + break; + case FLOAT: + float floatValue = record.getFloat(inputIndex); + if (Float.isFinite(floatValue)) { + buffer.addFloating(time, floatValue); + } + break; + case DOUBLE: + double doubleValue = record.getDouble(inputIndex); + if (Double.isFinite(doubleValue)) { + buffer.addFloating(time, doubleValue); + } + break; + default: + throw new IllegalArgumentException( + String.format( + CommonMessages.EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE, type)); + } + } + + private void write( + ColumnBuilder timeBuilder, ColumnBuilder valueBuilder, PointBuffer buffer, int index) { + timeBuilder.writeLong(buffer.times[index]); + switch (type) { + case INT32: + valueBuilder.writeInt((int) buffer.integralValues[index]); + break; + case INT64: + valueBuilder.writeLong(buffer.integralValues[index]); + break; + case FLOAT: + valueBuilder.writeFloat((float) buffer.values[index]); + break; + case DOUBLE: + valueBuilder.writeDouble(buffer.values[index]); + break; + default: + throw new IllegalArgumentException( + String.format( + CommonMessages.EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE, type)); + } + } + } + + /** + * Growable buffer of eligible points for one participant column. Integral columns additionally + * keep the exact {@code long} value so INT64 output is not rounded through {@code double}. + */ + static final class PointBuffer { + private long[] times = new long[INITIAL_CAPACITY]; + private double[] values = new double[INITIAL_CAPACITY]; + private long[] integralValues; + private int size; + + PointBuffer(boolean integral) { + if (integral) { + integralValues = new long[INITIAL_CAPACITY]; + } + } + + void addFloating(long time, double value) { + ensureCapacity(); + times[size] = time; + values[size] = value; + size++; + } + + void addIntegral(long time, long value) { + ensureCapacity(); + times[size] = time; + values[size] = value; + integralValues[size] = value; + size++; + } + + private void ensureCapacity() { + if (size < times.length) { + return; + } + if (size >= MAX_BUFFERED_POINTS) { + throw new SemanticException( + String.format( + CommonMessages + .EXCEPTION_LTTB_BUFFERS_AT_MOST_ARG_ELIGIBLE_POINTS_PER_COLUMN_IN_ONE_PARTITION_OR_WINDOW_4C746D82, + MAX_BUFFERED_POINTS)); + } + int newCapacity = (int) Math.min((long) times.length * 2, MAX_BUFFERED_POINTS); + times = Arrays.copyOf(times, newCapacity); + values = Arrays.copyOf(values, newCapacity); + if (integralValues != null) { + integralValues = Arrays.copyOf(integralValues, newCapacity); + } + } + + int size() { + return size; + } + + boolean isEmpty() { + return size == 0; + } + + long time(int index) { + return times[index]; + } + + double value(int index) { + return values[index]; + } + + double averageTime() { + double sum = 0; + for (int i = 0; i < size; i++) { + sum += times[i]; + } + return sum / size; + } + + double averageValue() { + double sum = 0; + for (int i = 0; i < size; i++) { + sum += values[i]; + } + return sum / size; + } + } + + // --------------------------------------------------------------------------------------------- + // Processors + // --------------------------------------------------------------------------------------------- + + private abstract static class AbstractLTTBDataProcessor implements TableFunctionDataProcessor { + protected final Type[] partitionTypes; + protected final Object[] partitionValues; + protected final ParticipantColumn[] participantColumns; + + protected AbstractLTTBDataProcessor( + Type[] partitionTypes, ParticipantColumn[] participantColumns) { + this.partitionTypes = partitionTypes; + this.partitionValues = new Object[partitionTypes.length]; + this.participantColumns = participantColumns; + } + + protected final PointBuffer[] newBuffers() { + PointBuffer[] buffers = new PointBuffer[participantColumns.length]; + for (int i = 0; i < buffers.length; i++) { + buffers[i] = new PointBuffer(participantColumns[i].isIntegral()); + } + return buffers; + } + + /** Partition columns are constant within a partition; keep the first non-null value. */ + protected final void capturePartitionValues(Record input) { + for (int i = 0; i < partitionTypes.length; i++) { + int inputIndex = i + 1; + if (partitionValues[i] == null && !input.isNull(inputIndex)) { + partitionValues[i] = WindowTVFUtils.readValue(input, inputIndex, partitionTypes[i]); + } + } + } + + protected final void collect(Record input, long time, PointBuffer[] buffers) { + for (int i = 0; i < participantColumns.length; i++) { + participantColumns[i].collect(input, time, buffers[i]); + } + } + + /** + * Writes one output row. {@code windowColumnCount} leading builders were already filled by the + * caller; {@code selectedIndexes[i]} is {@link #NOT_SELECTED} when column {@code i} has no + * point. + */ + protected final void writeRow( + List builders, + int windowColumnCount, + PointBuffer[] buffers, + int[] selectedIndexes) { + int builderIndex = windowColumnCount; + for (int i = 0; i < partitionTypes.length; i++) { + WindowTVFUtils.writeValue( + builders.get(builderIndex++), partitionValues[i], partitionTypes[i]); + } + for (int i = 0; i < participantColumns.length; i++) { + ColumnBuilder timeBuilder = builders.get(builderIndex++); + ColumnBuilder valueBuilder = builders.get(builderIndex++); + if (selectedIndexes[i] == NOT_SELECTED) { + timeBuilder.appendNull(); + valueBuilder.appendNull(); + } else { + participantColumns[i].write(timeBuilder, valueBuilder, buffers[i], selectedIndexes[i]); + } + } + } + } + + /** Buffers the whole partition and applies the standard LTTB algorithm in {@link #finish}. */ + private static final class TargetCountLTTBDataProcessor extends AbstractLTTBDataProcessor { + private final long n; + private final PointBuffer[] buffers; + private boolean hasRows; + + private TargetCountLTTBDataProcessor( + long n, Type[] partitionTypes, ParticipantColumn[] participantColumns) { + super(partitionTypes, participantColumns); + this.n = n; + this.buffers = newBuffers(); + } + + @Override + public void process( + Record input, + List properColumnBuilders, + ColumnBuilder passThroughIndexBuilder) { + hasRows = true; + capturePartitionValues(input); + collect(input, input.getLong(0), buffers); + } + + @Override + public void finish( + List properColumnBuilders, ColumnBuilder passThroughIndexBuilder) { + if (!hasRows) { + return; + } + int[][] selections = new int[participantColumns.length][]; + int rowCount = 1; + for (int i = 0; i < participantColumns.length; i++) { + PointBuffer buffer = buffers[i]; + selections[i] = selectTargetCount(buffer.times, buffer.values, buffer.size, n); + rowCount = Math.max(rowCount, selections[i].length); + } + + int[] selectedIndexes = new int[participantColumns.length]; + for (int row = 0; row < rowCount; row++) { + properColumnBuilders.get(0).writeLong(0L); + for (int i = 0; i < participantColumns.length; i++) { + selectedIndexes[i] = row < selections[i].length ? selections[i][row] : NOT_SELECTED; + } + writeRow(properColumnBuilders, 1, buffers, selectedIndexes); + } + } + } + + /** One window (= one LTTB bucket) worth of eligible points. */ + private abstract static class WindowState { + protected final PointBuffer[] buffers; + protected final long endExclusive; + + private WindowState(PointBuffer[] buffers, long endExclusive) { + this.buffers = buffers; + this.endExclusive = endExclusive; + } + + protected abstract void writeWindowColumns(List builders); + + protected abstract int getWindowColumnCount(); + } + + private static final class TimeWindowState extends WindowState { + private final long windowStart; + + private TimeWindowState(PointBuffer[] buffers, long windowStart, long endExclusive) { + super(buffers, endExclusive); + this.windowStart = windowStart; + } + + @Override + protected void writeWindowColumns(List builders) { + builders.get(0).writeLong(windowStart); + builders.get(1).writeLong(endExclusive); + } + + @Override + protected int getWindowColumnCount() { + return 2; + } + } + + private static final class CountWindowState extends WindowState { + private final long windowIndex; + + private CountWindowState(PointBuffer[] buffers, long endExclusive, long windowIndex) { + super(buffers, endExclusive); + this.windowIndex = windowIndex; + } + + @Override + protected void writeWindowColumns(List builders) { + builders.get(0).writeLong(windowIndex); + } + + @Override + protected int getWindowColumnCount() { + return 1; + } + } + + /** + * Window/bucket mode. Windows are constructed like M4 and emitted in window order; a window is + * emitted only once the following window with data is closed (or in {@link #finish}), because the + * average point of that following window is the third vertex of the LTTB triangle. Anchors are + * chained per participant column across the windows of the partition; windows without data for a + * column leave its anchor untouched. + */ + private abstract static class AbstractWindowLTTBDataProcessor + extends AbstractLTTBDataProcessor { + protected final long size; + protected final long slide; + protected final Deque activeWindows = new ArrayDeque<>(); + private final boolean[] hasAnchor; + private final long[] anchorTimes; + private final double[] anchorValues; + + protected AbstractWindowLTTBDataProcessor( + long size, long slide, Type[] partitionTypes, ParticipantColumn[] participantColumns) { + super(partitionTypes, participantColumns); + this.size = size; + this.slide = slide; + this.hasAnchor = new boolean[participantColumns.length]; + this.anchorTimes = new long[participantColumns.length]; + this.anchorValues = new double[participantColumns.length]; + } + + @Override + public final void process( + Record input, + List properColumnBuilders, + ColumnBuilder passThroughIndexBuilder) { + capturePartitionValues(input); + processRecord(input, input.getLong(0), properColumnBuilders); + } + + protected abstract void processRecord( + Record input, long time, List properColumnBuilders); + + /** Emits every window whose successor is already closed at {@code position}. */ + protected final void emitClosedWindows(long position, List builders) { + while (activeWindows.size() >= 2) { + W current = activeWindows.peekFirst(); + W next = nextOf(current); + if (next.endExclusive > position) { + return; + } + activeWindows.removeFirst(); + emitWindow(current, next, builders); + } + } + + private W nextOf(W first) { + Iterator iterator = activeWindows.iterator(); + iterator.next(); + return iterator.next(); + } + + protected final long getWindowEnd(long windowStart) { + return windowStart + size; + } + + @Override + public final void finish( + List properColumnBuilders, ColumnBuilder passThroughIndexBuilder) { + while (!activeWindows.isEmpty()) { + W current = activeWindows.removeFirst(); + emitWindow(current, activeWindows.peekFirst(), properColumnBuilders); + } + } + + private void emitWindow(W current, W next, List builders) { + int[] selectedIndexes = new int[participantColumns.length]; + for (int i = 0; i < participantColumns.length; i++) { + PointBuffer bucket = current.buffers[i]; + if (bucket.isEmpty()) { + selectedIndexes[i] = NOT_SELECTED; + continue; + } + if (!hasAnchor[i]) { + // The first eligible point of the partition seeds the anchor, as in standard LTTB. + hasAnchor[i] = true; + anchorTimes[i] = bucket.time(0); + anchorValues[i] = bucket.value(0); + } + double nextTime; + double nextValue; + if (next != null && !next.buffers[i].isEmpty()) { + nextTime = next.buffers[i].averageTime(); + nextValue = next.buffers[i].averageValue(); + } else { + int last = bucket.size() - 1; + nextTime = bucket.time(last); + nextValue = bucket.value(last); + } + int pick = + argMaxArea( + bucket.times, + bucket.values, + 0, + bucket.size(), + anchorTimes[i], + anchorValues[i], + nextTime, + nextValue); + selectedIndexes[i] = pick; + anchorTimes[i] = bucket.time(pick); + anchorValues[i] = bucket.value(pick); + } + current.writeWindowColumns(builders); + writeRow(builders, current.getWindowColumnCount(), current.buffers, selectedIndexes); + } + } + + private static final class TimeWindowLTTBDataProcessor + extends AbstractWindowLTTBDataProcessor { + private final long origin; + private boolean nextWindowStartInitialized = false; + private long nextWindowStart; + + private TimeWindowLTTBDataProcessor( + long size, + long slide, + long origin, + Type[] partitionTypes, + ParticipantColumn[] participantColumns) { + super(size, slide, partitionTypes, participantColumns); + this.origin = origin; + } + + @Override + protected void processRecord( + Record input, long time, List properColumnBuilders) { + emitClosedWindows(time, properColumnBuilders); + + long firstCandidateStart = + origin + Math.floorDiv(time - origin - size, slide) * slide + slide; + while (getWindowEnd(firstCandidateStart) <= time) { + firstCandidateStart += slide; + } + if (!nextWindowStartInitialized) { + nextWindowStart = firstCandidateStart; + nextWindowStartInitialized = true; + } else if (nextWindowStart < firstCandidateStart) { + nextWindowStart = firstCandidateStart; + } + + while (nextWindowStart <= time && getWindowEnd(nextWindowStart) > time) { + activeWindows.addLast( + new TimeWindowState(newBuffers(), nextWindowStart, getWindowEnd(nextWindowStart))); + nextWindowStart += slide; + } + + for (TimeWindowState window : activeWindows) { + // Closed windows waiting for their successor stay in the deque but must not absorb rows. + if (time < window.endExclusive) { + collect(input, time, window.buffers); + } + } + } + } + + private static final class CountWindowLTTBDataProcessor + extends AbstractWindowLTTBDataProcessor { + private long rowCount = 0; + private long nextWindowStart = 0; + private long nextWindowIndex = 0; + + private CountWindowLTTBDataProcessor( + long size, long slide, Type[] partitionTypes, ParticipantColumn[] participantColumns) { + super(size, slide, partitionTypes, participantColumns); + } + + @Override + protected void processRecord( + Record input, long time, List properColumnBuilders) { + emitClosedWindows(rowCount, properColumnBuilders); + + while (getWindowEnd(nextWindowStart) <= rowCount) { + nextWindowStart += slide; + } + while (nextWindowStart <= rowCount) { + activeWindows.addLast( + new CountWindowState(newBuffers(), getWindowEnd(nextWindowStart), nextWindowIndex++)); + nextWindowStart += slide; + } + + for (CountWindowState window : activeWindows) { + if (rowCount < window.endExclusive) { + collect(input, time, window.buffers); + } + } + rowCount++; + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java new file mode 100644 index 0000000000000..fcd05018e5e4f --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.udf.builtin.relational.tvf; + +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.i18n.CommonMessages; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; +import org.apache.iotdb.udf.api.relational.table.argument.Argument; +import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; +import org.apache.iotdb.udf.api.relational.table.argument.TableArgument; +import org.apache.iotdb.udf.api.type.Type; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class LTTBTableFunctionTest { + + @Test + public void testTargetCountPreservesEndpointsAndPeak() { + long[] times = {0, 1, 2, 3, 4, 5}; + double[] values = {0, 1, 8, 2, 0, 0}; + + assertArrayEquals( + new int[] {0, 2, 5}, LTTBTableFunction.selectTargetCount(times, values, 6, 3)); + } + + @Test + public void testTargetCountReturnsAllShortSeries() { + long[] times = {0, 1}; + double[] values = {1, 2}; + + assertArrayEquals(new int[] {0, 1}, LTTBTableFunction.selectTargetCount(times, values, 2, 3)); + } + + @Test + public void testTargetCountBreaksAreaTiesByStableInputOrder() { + long[] times = {0, 1, 2, 3, 4}; + double[] values = {0, 4, 4, 1, 0}; + + assertArrayEquals( + new int[] {0, 1, 4}, LTTBTableFunction.selectTargetCount(times, values, 5, 3)); + } + + @Test + public void testAnalyzeTargetCountOutputSchema() throws UDFException { + LTTBTableFunction function = new LTTBTableFunction(); + TableFunctionAnalysis analysis = function.analyze(targetCountArguments(3L)); + + assertEquals( + "window_index", analysis.getProperColumnSchema().get().getFields().get(0).getName().get()); + assertEquals( + "device", analysis.getProperColumnSchema().get().getFields().get(1).getName().get()); + assertEquals( + "s1_time", analysis.getProperColumnSchema().get().getFields().get(2).getName().get()); + assertEquals("s1", analysis.getProperColumnSchema().get().getFields().get(3).getName().get()); + assertEquals( + Arrays.asList(0, 1, 2), + analysis.getRequiredColumns().get(LTTBTableFunction.DATA_PARAMETER_NAME)); + } + + @Test + public void testAnalyzeRejectsTargetCountBelowThree() throws UDFException { + try { + new LTTBTableFunction().analyze(targetCountArguments(2L)); + fail(); + } catch (SemanticException e) { + // compare against the constant so the test also passes under -P with-zh-locale + assertEquals( + CommonMessages.EXCEPTION_THE_N_ARGUMENT_OF_LTTB_MUST_BE_AT_LEAST_3_29CE2B87, + e.getMessage()); + } + } + + @Test + public void testAnalyzeRejectsNAndSizeTogether() throws UDFException { + Map arguments = targetCountArguments(3L); + arguments.put(LTTBTableFunction.SIZE_PARAMETER_NAME, new ScalarArgument(Type.INT64, 2L)); + try { + new LTTBTableFunction().analyze(arguments); + fail(); + } catch (SemanticException e) { + assertEquals( + CommonMessages + .EXCEPTION_EXACTLY_ONE_OF_THE_N_AND_SIZE_ARGUMENTS_MUST_BE_SPECIFIED_FOR_LTTB_54AF0733, + e.getMessage()); + } + } + + private static Map targetCountArguments(long n) { + Map arguments = new HashMap<>(); + arguments.put( + LTTBTableFunction.DATA_PARAMETER_NAME, + new TableArgument( + Arrays.asList(Optional.of("time"), Optional.of("device"), Optional.of("s1")), + Arrays.asList(Type.TIMESTAMP, Type.STRING, Type.DOUBLE), + Collections.singletonList("device"), + Collections.singletonList("time"), + false)); + arguments.put( + LTTBTableFunction.TIMECOL_PARAMETER_NAME, new ScalarArgument(Type.STRING, "time")); + arguments.put(LTTBTableFunction.N_PARAMETER_NAME, new ScalarArgument(Type.INT64, n)); + arguments.put( + LTTBTableFunction.SIZE_PARAMETER_NAME, + new ScalarArgument(Type.INT64, LTTBTableFunction.UNSPECIFIED_SIZE)); + arguments.put( + LTTBTableFunction.SLIDE_PARAMETER_NAME, + new ScalarArgument(Type.INT64, LTTBTableFunction.UNSPECIFIED_SLIDE)); + arguments.put(LTTBTableFunction.ORIGIN_PARAMETER_NAME, new ScalarArgument(Type.TIMESTAMP, 0L)); + arguments.put( + LTTBTableFunction.WINDOW_MODE_PARAMETER_NAME, new ScalarArgument(Type.BOOLEAN, false)); + return arguments; + } +} From 472494e70e2336c19d6bc136ba834f11b38e8950 Mon Sep 17 00:00:00 2001 From: DaZuiZui Date: Thu, 8 Oct 2026 16:00:35 +0800 Subject: [PATCH 02/21] fix(table): bound LTTB overlapping window buffer memory --- .../iotdb/commons/i18n/CommonMessages.java | 3 + .../iotdb/commons/i18n/CommonMessages.java | 2 + .../relational/tvf/LTTBTableFunction.java | 107 ++++++++- .../relational/tvf/LTTBWindowMemoryTest.java | 220 ++++++++++++++++++ 4 files changed, 327 insertions(+), 5 deletions(-) create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java index 5200e2da0481f..47ccbb90c03c5 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -361,6 +361,9 @@ private CommonMessages() {} "LTTB buffers at most %d eligible points per column in one partition or window."; public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = "Unsupported LTTB value type: %s"; + public static final String LTTB_WINDOW_BUFFER_MEMORY_LIMIT_EXCEEDED = + "LTTB window buffers exceed the per-processor memory limit of %d bytes. Reduce SIZE or the" + + " number of numeric columns, or increase SLIDE."; public static final String EXCEPTION_SNAPSHOT_BUFFER_SIZE_MUST_NOT_EXCEED_ARG_BYTES_BUT_WAS_ARG_D1DA6F7E = "Snapshot buffer size must not exceed %d bytes, but was %d."; diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java index 9b44c9b590cc0..b890a5072f853 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -257,6 +257,8 @@ private CommonMessages() {} "LTTB 在单个分区或窗口内每列最多缓存 %d 个有效点。"; public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = "不支持的 LTTB 值类型:%s"; + public static final String LTTB_WINDOW_BUFFER_MEMORY_LIMIT_EXCEEDED = + "LTTB 窗口缓冲区超过每个处理器 %d 字节的内存上限。请减小 SIZE 或数值列数量,或增大 SLIDE。"; public static final String EXCEPTION_SNAPSHOT_BUFFER_SIZE_MUST_NOT_EXCEED_ARG_BYTES_BUT_WAS_ARG_D1DA6F7E = "快照缓冲区大小不得超过 %d 字节,但实际为 %d。"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index 0a161fc236128..fe333d375f826 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -39,6 +39,7 @@ import org.apache.iotdb.udf.api.type.Type; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.utils.RamUsageEstimator; import java.util.ArrayDeque; import java.util.ArrayList; @@ -97,6 +98,9 @@ public class LTTBTableFunction implements TableFunction { /** Hard cap on eligible points buffered per participant column in one partition or window. */ public static final int MAX_BUFFERED_POINTS = 1 << 22; + /** Shared budget for point buffers in all live windows and columns of one processor. */ + static final long MAX_WINDOW_BUFFER_BYTES = 64L * 1024 * 1024; + public static final String MODE_PROPERTY = "__LTTB_MODE"; public static final String PARTITION_TYPES_PROPERTY = "__LTTB_PARTITION_TYPES"; public static final String PARTICIPANT_TYPES_PROPERTY = "__LTTB_PARTICIPANT_TYPES"; @@ -412,6 +416,28 @@ static int argMaxArea( // Column access and buffering // --------------------------------------------------------------------------------------------- + /** Reservations include array capacity, including both old and new arrays during growth. */ + static final class WindowMemoryBudget { + private final long limit; + private long reservedBytes; + + WindowMemoryBudget(long limit) { + this.limit = limit; + } + + void reserve(long bytes) { + if (bytes > limit - reservedBytes) { + throw new SemanticException( + String.format(CommonMessages.LTTB_WINDOW_BUFFER_MEMORY_LIMIT_EXCEEDED, limit)); + } + reservedBytes += bytes; + } + + void release(long bytes) { + reservedBytes -= bytes; + } + } + private static final class ParticipantColumn { private final int inputIndex; private final Type type; @@ -485,17 +511,43 @@ private void write( * keep the exact {@code long} value so INT64 output is not rounded through {@code double}. */ static final class PointBuffer { - private long[] times = new long[INITIAL_CAPACITY]; - private double[] values = new double[INITIAL_CAPACITY]; + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(PointBuffer.class); + + private final WindowMemoryBudget memoryBudget; + private long[] times; + private double[] values; private long[] integralValues; private int size; PointBuffer(boolean integral) { + this(integral, null); + } + + PointBuffer(boolean integral, WindowMemoryBudget memoryBudget) { + this.memoryBudget = memoryBudget; + if (memoryBudget != null) { + memoryBudget.reserve(INSTANCE_SIZE + arrayBytes(INITIAL_CAPACITY, integral)); + } + times = new long[INITIAL_CAPACITY]; + values = new double[INITIAL_CAPACITY]; if (integral) { integralValues = new long[INITIAL_CAPACITY]; } } + private static long arrayBytes(int capacity, boolean integral) { + return RamUsageEstimator.sizeOfLongArray(capacity) + + RamUsageEstimator.sizeOfDoubleArray(capacity) + + (integral ? RamUsageEstimator.sizeOfLongArray(capacity) : 0); + } + + private void release() { + if (memoryBudget != null) { + memoryBudget.release(INSTANCE_SIZE + arrayBytes(times.length, integralValues != null)); + } + } + void addFloating(long time, double value) { ensureCapacity(); times[size] = time; @@ -523,11 +575,19 @@ private void ensureCapacity() { MAX_BUFFERED_POINTS)); } int newCapacity = (int) Math.min((long) times.length * 2, MAX_BUFFERED_POINTS); + long oldArrayBytes = arrayBytes(times.length, integralValues != null); + if (memoryBudget != null) { + // Reserve before copying: the old arrays remain live until their replacements exist. + memoryBudget.reserve(arrayBytes(newCapacity, integralValues != null)); + } times = Arrays.copyOf(times, newCapacity); values = Arrays.copyOf(values, newCapacity); if (integralValues != null) { integralValues = Arrays.copyOf(integralValues, newCapacity); } + if (memoryBudget != null) { + memoryBudget.release(oldArrayBytes); + } } int size() { @@ -580,9 +640,13 @@ protected AbstractLTTBDataProcessor( } protected final PointBuffer[] newBuffers() { + return newBuffers(null); + } + + protected final PointBuffer[] newBuffers(WindowMemoryBudget memoryBudget) { PointBuffer[] buffers = new PointBuffer[participantColumns.length]; for (int i = 0; i < buffers.length; i++) { - buffers[i] = new PointBuffer(participantColumns[i].isIntegral()); + buffers[i] = new PointBuffer(participantColumns[i].isIntegral(), memoryBudget); } return buffers; } @@ -742,9 +806,16 @@ protected int getWindowColumnCount() { */ private abstract static class AbstractWindowLTTBDataProcessor extends AbstractLTTBDataProcessor { + private static final long WINDOW_INSTANCE_SIZE = + Math.max( + RamUsageEstimator.shallowSizeOfInstance(TimeWindowState.class), + RamUsageEstimator.shallowSizeOfInstance(CountWindowState.class)); + protected final long size; protected final long slide; protected final Deque activeWindows = new ArrayDeque<>(); + private final WindowMemoryBudget memoryBudget = new WindowMemoryBudget(MAX_WINDOW_BUFFER_BYTES); + private final long windowMetadataBytes; private final boolean[] hasAnchor; private final long[] anchorTimes; private final double[] anchorValues; @@ -754,11 +825,26 @@ protected AbstractWindowLTTBDataProcessor( super(partitionTypes, participantColumns); this.size = size; this.slide = slide; + this.windowMetadataBytes = + WINDOW_INSTANCE_SIZE + RamUsageEstimator.sizeOfObjectArray(participantColumns.length); this.hasAnchor = new boolean[participantColumns.length]; this.anchorTimes = new long[participantColumns.length]; this.anchorValues = new double[participantColumns.length]; } + protected final PointBuffer[] newWindowBuffers() { + // Even an all-NULL window allocates state, so reserve before creating any of its buffers. + memoryBudget.reserve(windowMetadataBytes); + return newBuffers(memoryBudget); + } + + private void releaseWindow(W window) { + for (PointBuffer buffer : window.buffers) { + buffer.release(); + } + memoryBudget.release(windowMetadataBytes); + } + @Override public final void process( Record input, @@ -781,6 +867,7 @@ protected final void emitClosedWindows(long position, List builde } activeWindows.removeFirst(); emitWindow(current, next, builders); + releaseWindow(current); } } @@ -800,9 +887,17 @@ public final void finish( while (!activeWindows.isEmpty()) { W current = activeWindows.removeFirst(); emitWindow(current, activeWindows.peekFirst(), properColumnBuilders); + releaseWindow(current); } } + @Override + public final void beforeDestroy() { + activeWindows.clear(); + // Also covers a reservation made by a window whose construction failed at the limit. + memoryBudget.reservedBytes = 0; + } + private void emitWindow(W current, W next, List builders) { int[] selectedIndexes = new int[participantColumns.length]; for (int i = 0; i < participantColumns.length; i++) { @@ -881,7 +976,8 @@ protected void processRecord( while (nextWindowStart <= time && getWindowEnd(nextWindowStart) > time) { activeWindows.addLast( - new TimeWindowState(newBuffers(), nextWindowStart, getWindowEnd(nextWindowStart))); + new TimeWindowState( + newWindowBuffers(), nextWindowStart, getWindowEnd(nextWindowStart))); nextWindowStart += slide; } @@ -915,7 +1011,8 @@ protected void processRecord( } while (nextWindowStart <= rowCount) { activeWindows.addLast( - new CountWindowState(newBuffers(), getWindowEnd(nextWindowStart), nextWindowIndex++)); + new CountWindowState( + newWindowBuffers(), getWindowEnd(nextWindowStart), nextWindowIndex++)); nextWindowStart += slide; } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java new file mode 100644 index 0000000000000..a09afcbc21d51 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.udf.builtin.relational.tvf; + +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.i18n.CommonMessages; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import org.apache.tsfile.read.common.block.column.LongColumnBuilder; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +public class LTTBWindowMemoryTest { + + @Test + public void testCountWindowOverlapFailsBeforeExhaustingHeap() { + TableFunctionDataProcessor processor = processor(false, 1_000_000, 1, Type.DOUBLE); + Record input = record(0, 1.0); + try { + // Without the shared budget, 5000 rows retain over 280 MiB despite every individual + // window being far below MAX_BUFFERED_POINTS. This test also runs with a 128 MiB heap. + assertMemoryLimit( + () -> { + for (int i = 0; i < 5000; i++) { + processor.process(input, Collections.emptyList(), null); + } + }, + LTTBTableFunction.MAX_WINDOW_BUFFER_BYTES); + } finally { + processor.beforeDestroy(); + } + } + + @Test + public void testTimeWindowCreationIsBoundedEvenForNullInput() { + TableFunctionDataProcessor processor = processor(true, 1_000_000, 1, Type.DOUBLE); + try { + // One row belongs to a million overlapping time windows. Initial buffer allocation + // must be charged even when the input column contains no eligible points. + assertMemoryLimit( + () -> processor.process(record(0, (Number) null), Collections.emptyList(), null), + LTTBTableFunction.MAX_WINDOW_BUFFER_BYTES); + } finally { + processor.beforeDestroy(); + } + } + + @Test + public void testColumnsShareTheWindowBudget() { + TableFunctionDataProcessor processor = processor(false, 1_000_000, 1, Type.DOUBLE, Type.INT64); + Record input = record(0, 1.0, Long.MAX_VALUE); + try { + assertMemoryLimit( + () -> { + for (int i = 0; i < 1800; i++) { + processor.process(input, Collections.emptyList(), null); + } + }, + LTTBTableFunction.MAX_WINDOW_BUFFER_BYTES); + } finally { + processor.beforeDestroy(); + } + } + + @Test + public void testArrayGrowthReservesOldAndNewIntegralArrays() { + LTTBTableFunction.PointBuffer buffer = + new LTTBTableFunction.PointBuffer(true, new LTTBTableFunction.WindowMemoryBudget(4096)); + for (int i = 0; i < 64; i++) { + buffer.addIntegral(i, Long.MAX_VALUE); + } + // The retained 128-point buffer fits in 4096 bytes, but the old 64-point arrays and + // replacement arrays coexist during growth. Reject before replacing any array. + assertMemoryLimit(() -> buffer.addIntegral(64, Long.MAX_VALUE), 4096); + assertEquals(64, buffer.size()); + assertEquals(63, buffer.time(63)); + } + + @Test + public void testInitialBuffersShareCapacityBudget() { + LTTBTableFunction.WindowMemoryBudget budget = new LTTBTableFunction.WindowMemoryBudget(4096); + new LTTBTableFunction.PointBuffer(true, budget); + new LTTBTableFunction.PointBuffer(true, budget); + assertMemoryLimit(() -> new LTTBTableFunction.PointBuffer(true, budget), 4096); + } + + @Test + public void testEmittedWindowsReleaseBudgetForLongStream() { + TableFunctionDataProcessor processor = processor(false, 1, 1, Type.DOUBLE); + Record input = record(0, 42.0); + List builders = builders(false); + long outputRows = 0; + try { + // Total allocations exceed the budget, but only two windows remain live at a time. + for (int row = 0; row < 80_000; row++) { + processor.process(input, builders, null); + if (row % 512 == 511) { + outputRows += builders.get(0).getPositionCount(); + builders = builders(false); + } + } + processor.finish(builders, null); + outputRows += builders.get(0).getPositionCount(); + assertEquals(80_000, outputRows); + assertEquals(42.0, builders.get(2).build().getDouble(0), 0.0); + } finally { + processor.beforeDestroy(); + } + } + + @Test + public void testWindowResultsArePreserved() { + for (boolean timeWindow : new boolean[] {false, true}) { + TableFunctionDataProcessor processor = processor(timeWindow, 2, 2, Type.DOUBLE); + List builders = builders(timeWindow); + double[] values = {0, 1, 8, 2, 0, 0}; + try { + for (int time = 0; time < values.length; time++) { + processor.process(record(time, values[time]), builders, null); + } + processor.finish(builders, null); + ColumnBuilder times = builders.get(timeWindow ? 2 : 1); + assertEquals(3, times.getPositionCount()); + assertEquals(1, times.build().getLong(0)); + assertEquals(2, times.build().getLong(1)); + assertEquals(4, times.build().getLong(2)); + } finally { + processor.beforeDestroy(); + } + } + } + + private static TableFunctionDataProcessor processor( + boolean timeWindow, long size, long slide, Type... types) { + MapTableFunctionHandle handle = + new MapTableFunctionHandle.Builder() + .addProperty( + LTTBTableFunction.MODE_PROPERTY, timeWindow ? "TIME_WINDOW" : "COUNT_WINDOW") + .addProperty(LTTBTableFunction.PARTITION_TYPES_PROPERTY, "") + .addProperty( + LTTBTableFunction.PARTICIPANT_TYPES_PROPERTY, + WindowTVFUtils.joinTypes(Arrays.asList(types))) + .addProperty(LTTBTableFunction.SIZE_PARAMETER_NAME, size) + .addProperty(LTTBTableFunction.SLIDE_PARAMETER_NAME, slide) + .addProperty(LTTBTableFunction.ORIGIN_PARAMETER_NAME, 0L) + .build(); + return new LTTBTableFunction().getProcessorProvider(handle).getDataProcessor(); + } + + private static List builders(boolean timeWindow) { + if (timeWindow) { + return Arrays.asList( + new LongColumnBuilder(null, 16), + new LongColumnBuilder(null, 16), + new LongColumnBuilder(null, 16), + new DoubleColumnBuilder(null, 16)); + } + return Arrays.asList( + new LongColumnBuilder(null, 16), + new LongColumnBuilder(null, 16), + new DoubleColumnBuilder(null, 16)); + } + + private static Record record(long time, Number... values) { + // Avoid retaining invocation histories in the long-stream memory regression test. + Record record = mock(Record.class, withSettings().stubOnly()); + when(record.getLong(0)).thenReturn(time); + for (int i = 0; i < values.length; i++) { + when(record.isNull(i + 1)).thenReturn(values[i] == null); + if (values[i] instanceof Long) { + when(record.getLong(i + 1)).thenReturn(values[i].longValue()); + } else if (values[i] != null) { + when(record.getDouble(i + 1)).thenReturn(values[i].doubleValue()); + } + } + return record; + } + + private static void assertMemoryLimit(Runnable action, long limit) { + try { + action.run(); + fail("Expected the shared LTTB window memory limit"); + } catch (SemanticException e) { + assertEquals( + String.format(CommonMessages.LTTB_WINDOW_BUFFER_MEMORY_LIMIT_EXCEEDED, limit), + e.getMessage()); + } + } +} From b2f9b7791bdfcb083c0347be3743ea2c53c174e8 Mon Sep 17 00:00:00 2001 From: DaZuiZui Date: Thu, 8 Oct 2026 16:42:53 +0800 Subject: [PATCH 03/21] fix(table): correct LTTB memory bounds and sparse sampling --- .../it/db/it/IoTDBLTTBTableFunctionIT.java | 31 +++ .../iotdb/commons/i18n/CommonMessages.java | 6 +- .../iotdb/commons/i18n/CommonMessages.java | 5 +- .../relational/tvf/LTTBTableFunction.java | 153 ++++++++----- .../relational/tvf/LTTBRegressionTest.java | 207 ++++++++++++++++++ .../relational/tvf/LTTBWindowMemoryTest.java | 15 +- 6 files changed, 349 insertions(+), 68 deletions(-) create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java index 63ab046927233..564ccb4e4d169 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java @@ -62,6 +62,13 @@ public static void setUp() throws Exception { + "(5, 'd1', 0.0, 0.0)," + "(0, 'd2', null, null)," + "(1, 'd2', null, null)"); + statement.execute( + "CREATE TABLE sparse(device STRING TAG, value DOUBLE FIELD, marker INT32 FIELD)"); + statement.execute( + "INSERT INTO sparse(time, device, value, marker) VALUES " + + "(0, 'd1', 0.0, 1), (1, 'd1', 1.0, 1), (2, 'd1', 2.0, 1)," + + "(3, 'd1', null, 1), (4, 'd1', null, 1), (5, 'd1', null, 1)," + + "(6, 'd1', 0.0, 1), (7, 'd1', 0.0, 1), (8, 'd1', 0.0, 1)"); statement.execute("FLUSH"); } catch (Exception e) { fail("insertData failed: " + e.getMessage()); @@ -115,6 +122,30 @@ public void testCountWindowAndTimeWindowOutput() { DATABASE_NAME); } + @Test + public void testWindowSkipsNullBucketForLookahead() { + String data = + "DATA => (SELECT time, device, value FROM sparse) PARTITION BY device ORDER BY time, TIMECOL => 'time', "; + tableResultSetEqualTest( + "SELECT * FROM LTTB(" + data + "SIZE => 3) ORDER BY window_index", + new String[] {"window_index", "device", "value_time", "value"}, + new String[] { + "0,d1,1970-01-01T00:00:00.002Z,2.0,", + "1,d1,null,null,", + "2,d1,1970-01-01T00:00:00.006Z,0.0," + }, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT * FROM LTTB(" + data + "SIZE => 3ms) ORDER BY window_start", + new String[] {"window_start", "window_end", "device", "value_time", "value"}, + new String[] { + "1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.002Z,2.0,", + "1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.006Z,d1,null,null,", + "1970-01-01T00:00:00.006Z,1970-01-01T00:00:00.009Z,d1,1970-01-01T00:00:00.006Z,0.0," + }, + DATABASE_NAME); + } + @Test public void testInvalidParameterCombinations() { String prefix = diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java index 47ccbb90c03c5..44edd90841d01 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -361,9 +361,9 @@ private CommonMessages() {} "LTTB buffers at most %d eligible points per column in one partition or window."; public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = "Unsupported LTTB value type: %s"; - public static final String LTTB_WINDOW_BUFFER_MEMORY_LIMIT_EXCEEDED = - "LTTB window buffers exceed the per-processor memory limit of %d bytes. Reduce SIZE or the" - + " number of numeric columns, or increase SLIDE."; + public static final String + EXCEPTION_LTTB_BUFFERS_EXCEED_THE_PER_PROCESSOR_MEMORY_LIMIT_OF_ARG_BYTES_954FEF00 = + "LTTB buffers exceed the per-processor memory limit of %d bytes."; public static final String EXCEPTION_SNAPSHOT_BUFFER_SIZE_MUST_NOT_EXCEED_ARG_BYTES_BUT_WAS_ARG_D1DA6F7E = "Snapshot buffer size must not exceed %d bytes, but was %d."; diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java index b890a5072f853..7b2ed93291527 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -257,8 +257,9 @@ private CommonMessages() {} "LTTB 在单个分区或窗口内每列最多缓存 %d 个有效点。"; public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = "不支持的 LTTB 值类型:%s"; - public static final String LTTB_WINDOW_BUFFER_MEMORY_LIMIT_EXCEEDED = - "LTTB 窗口缓冲区超过每个处理器 %d 字节的内存上限。请减小 SIZE 或数值列数量,或增大 SLIDE。"; + public static final String + EXCEPTION_LTTB_BUFFERS_EXCEED_THE_PER_PROCESSOR_MEMORY_LIMIT_OF_ARG_BYTES_954FEF00 = + "LTTB 缓冲区超过每个处理器 %d 字节的内存上限。"; public static final String EXCEPTION_SNAPSHOT_BUFFER_SIZE_MUST_NOT_EXCEED_ARG_BYTES_BUT_WAS_ARG_D1DA6F7E = "快照缓冲区大小不得超过 %d 字节,但实际为 %d。"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index fe333d375f826..ad14d83744419 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -46,7 +46,6 @@ import java.util.Arrays; import java.util.Deque; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -98,8 +97,8 @@ public class LTTBTableFunction implements TableFunction { /** Hard cap on eligible points buffered per participant column in one partition or window. */ public static final int MAX_BUFFERED_POINTS = 1 << 22; - /** Shared budget for point buffers in all live windows and columns of one processor. */ - static final long MAX_WINDOW_BUFFER_BYTES = 64L * 1024 * 1024; + /** Shared budget for point buffers and selection state across all columns of one processor. */ + static final long MAX_BUFFER_BYTES = 64L * 1024 * 1024; public static final String MODE_PROPERTY = "__LTTB_MODE"; public static final String PARTITION_TYPES_PROPERTY = "__LTTB_PARTITION_TYPES"; @@ -352,17 +351,18 @@ static int[] selectTargetCount(long[] times, double[] values, int count, long n) int currentStart = (int) (bucket * intermediate / bucketCount) + 1; int currentEnd = (int) ((bucket + 1) * intermediate / bucketCount) + 1; + long anchorTime = times[anchor]; double avgTime; double avgValue; if (bucket == bucketCount - 1) { - avgTime = times[count - 1]; + avgTime = relativeTime(times[count - 1], anchorTime); avgValue = values[count - 1]; } else { int nextEnd = (int) ((bucket + 2) * intermediate / bucketCount) + 1; double sumTime = 0; double sumValue = 0; for (int i = currentEnd; i < nextEnd; i++) { - sumTime += times[i]; + sumTime += relativeTime(times[i], anchorTime); sumValue += values[i]; } int nextSize = nextEnd - currentEnd; @@ -376,7 +376,7 @@ static int[] selectTargetCount(long[] times, double[] values, int count, long n) values, currentStart, currentEnd, - times[anchor], + anchorTime, values[anchor], avgTime, avgValue); @@ -387,13 +387,17 @@ static int[] selectTargetCount(long[] times, double[] values, int count, long n) return selected; } - /** Returns the index in {@code [start, end)} maximizing the triangle area; earliest wins ties. */ + /** + * Returns the index in {@code [start, end)} maximizing the triangle area; earliest wins ties. + * {@code nextTime} is relative to {@code anchorTime}, avoiding rounding absolute nanosecond + * epochs. + */ static int argMaxArea( long[] times, double[] values, int start, int end, - double anchorTime, + long anchorTime, double anchorValue, double nextTime, double nextValue) { @@ -402,8 +406,8 @@ static int argMaxArea( for (int i = start; i < end; i++) { double area = Math.abs( - (anchorTime - nextTime) * (values[i] - anchorValue) - - (anchorTime - times[i]) * (nextValue - anchorValue)); + nextTime * (values[i] - anchorValue) + - relativeTime(times[i], anchorTime) * (nextValue - anchorValue)); if (area > maxArea) { maxArea = area; pick = i; @@ -412,23 +416,35 @@ static int argMaxArea( return pick; } + private static double relativeTime(long time, long origin) { + long difference = time - origin; + // Subtract before converting so nearby timestamps retain their precision. For spans exceeding + // the long range, use floating-point subtraction instead of the overflowed long difference. + return ((time ^ origin) & (time ^ difference)) < 0 + ? (double) time - (double) origin + : (double) difference; + } + // --------------------------------------------------------------------------------------------- // Column access and buffering // --------------------------------------------------------------------------------------------- /** Reservations include array capacity, including both old and new arrays during growth. */ - static final class WindowMemoryBudget { + static final class BufferMemoryBudget { private final long limit; private long reservedBytes; - WindowMemoryBudget(long limit) { + BufferMemoryBudget(long limit) { this.limit = limit; } void reserve(long bytes) { if (bytes > limit - reservedBytes) { throw new SemanticException( - String.format(CommonMessages.LTTB_WINDOW_BUFFER_MEMORY_LIMIT_EXCEEDED, limit)); + String.format( + CommonMessages + .EXCEPTION_LTTB_BUFFERS_EXCEED_THE_PER_PROCESSOR_MEMORY_LIMIT_OF_ARG_BYTES_954FEF00, + limit)); } reservedBytes += bytes; } @@ -514,17 +530,13 @@ static final class PointBuffer { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(PointBuffer.class); - private final WindowMemoryBudget memoryBudget; + private final BufferMemoryBudget memoryBudget; private long[] times; private double[] values; private long[] integralValues; private int size; - PointBuffer(boolean integral) { - this(integral, null); - } - - PointBuffer(boolean integral, WindowMemoryBudget memoryBudget) { + PointBuffer(boolean integral, BufferMemoryBudget memoryBudget) { this.memoryBudget = memoryBudget; if (memoryBudget != null) { memoryBudget.reserve(INSTANCE_SIZE + arrayBytes(INITIAL_CAPACITY, integral)); @@ -606,10 +618,10 @@ long time(int index) { return values[index]; } - double averageTime() { + double averageTime(long origin) { double sum = 0; for (int i = 0; i < size; i++) { - sum += times[i]; + sum += relativeTime(times[i], origin); } return sum / size; } @@ -639,11 +651,7 @@ protected AbstractLTTBDataProcessor( this.participantColumns = participantColumns; } - protected final PointBuffer[] newBuffers() { - return newBuffers(null); - } - - protected final PointBuffer[] newBuffers(WindowMemoryBudget memoryBudget) { + protected final PointBuffer[] newBuffers(BufferMemoryBudget memoryBudget) { PointBuffer[] buffers = new PointBuffer[participantColumns.length]; for (int i = 0; i < buffers.length; i++) { buffers[i] = new PointBuffer(participantColumns[i].isIntegral(), memoryBudget); @@ -699,13 +707,15 @@ protected final void writeRow( private static final class TargetCountLTTBDataProcessor extends AbstractLTTBDataProcessor { private final long n; private final PointBuffer[] buffers; + private final BufferMemoryBudget memoryBudget = new BufferMemoryBudget(MAX_BUFFER_BYTES); private boolean hasRows; private TargetCountLTTBDataProcessor( long n, Type[] partitionTypes, ParticipantColumn[] participantColumns) { super(partitionTypes, participantColumns); this.n = n; - this.buffers = newBuffers(); + memoryBudget.reserve(RamUsageEstimator.sizeOfObjectArray(participantColumns.length)); + this.buffers = newBuffers(memoryBudget); } @Override @@ -724,6 +734,13 @@ public void finish( if (!hasRows) { return; } + long selectionBytes = + RamUsageEstimator.sizeOfObjectArray(participantColumns.length) + + RamUsageEstimator.sizeOfIntArray(participantColumns.length); + for (PointBuffer buffer : buffers) { + selectionBytes += RamUsageEstimator.sizeOfIntArray((int) Math.min(buffer.size, n)); + } + memoryBudget.reserve(selectionBytes); int[][] selections = new int[participantColumns.length][]; int rowCount = 1; for (int i = 0; i < participantColumns.length; i++) { @@ -741,15 +758,24 @@ public void finish( writeRow(properColumnBuilders, 1, buffers, selectedIndexes); } } + + @Override + public void beforeDestroy() { + Arrays.fill(buffers, null); + memoryBudget.reservedBytes = 0; + } } /** One window (= one LTTB bucket) worth of eligible points. */ private abstract static class WindowState { protected final PointBuffer[] buffers; + protected final PointBuffer[] nextNonemptyBuffers; protected final long endExclusive; + protected int unresolvedColumns; private WindowState(PointBuffer[] buffers, long endExclusive) { this.buffers = buffers; + this.nextNonemptyBuffers = new PointBuffer[buffers.length]; this.endExclusive = endExclusive; } @@ -814,7 +840,9 @@ private abstract static class AbstractWindowLTTBDataProcessor activeWindows = new ArrayDeque<>(); - private final WindowMemoryBudget memoryBudget = new WindowMemoryBudget(MAX_WINDOW_BUFFER_BYTES); + private final Deque pendingWindows = new ArrayDeque<>(); + private final WindowState[] lastNonemptyWindows; + private final BufferMemoryBudget memoryBudget = new BufferMemoryBudget(MAX_BUFFER_BYTES); private final long windowMetadataBytes; private final boolean[] hasAnchor; private final long[] anchorTimes; @@ -826,7 +854,8 @@ protected AbstractWindowLTTBDataProcessor( this.size = size; this.slide = slide; this.windowMetadataBytes = - WINDOW_INSTANCE_SIZE + RamUsageEstimator.sizeOfObjectArray(participantColumns.length); + WINDOW_INSTANCE_SIZE + 2 * RamUsageEstimator.sizeOfObjectArray(participantColumns.length); + this.lastNonemptyWindows = new WindowState[participantColumns.length]; this.hasAnchor = new boolean[participantColumns.length]; this.anchorTimes = new long[participantColumns.length]; this.anchorValues = new double[participantColumns.length]; @@ -857,24 +886,36 @@ public final void process( protected abstract void processRecord( Record input, long time, List properColumnBuilders); - /** Emits every window whose successor is already closed at {@code position}. */ + /** Close windows in order and link each column to its next nonempty, closed bucket. */ protected final void emitClosedWindows(long position, List builders) { - while (activeWindows.size() >= 2) { - W current = activeWindows.peekFirst(); - W next = nextOf(current); - if (next.endExclusive > position) { - return; + while (!activeWindows.isEmpty() && activeWindows.peekFirst().endExclusive <= position) { + closeWindow(activeWindows.removeFirst()); + } + emitReadyWindows(builders, false); + } + + private void closeWindow(W window) { + for (int i = 0; i < participantColumns.length; i++) { + if (!window.buffers[i].isEmpty()) { + WindowState previous = lastNonemptyWindows[i]; + if (previous != null) { + previous.nextNonemptyBuffers[i] = window.buffers[i]; + previous.unresolvedColumns--; + } + lastNonemptyWindows[i] = window; + window.unresolvedColumns++; } - activeWindows.removeFirst(); - emitWindow(current, next, builders); - releaseWindow(current); } + pendingWindows.addLast(window); } - private W nextOf(W first) { - Iterator iterator = activeWindows.iterator(); - iterator.next(); - return iterator.next(); + private void emitReadyWindows(List builders, boolean finishing) { + while (!pendingWindows.isEmpty() + && (finishing || pendingWindows.peekFirst().unresolvedColumns == 0)) { + W current = pendingWindows.removeFirst(); + emitWindow(current, builders); + releaseWindow(current); + } } protected final long getWindowEnd(long windowStart) { @@ -885,20 +926,22 @@ protected final long getWindowEnd(long windowStart) { public final void finish( List properColumnBuilders, ColumnBuilder passThroughIndexBuilder) { while (!activeWindows.isEmpty()) { - W current = activeWindows.removeFirst(); - emitWindow(current, activeWindows.peekFirst(), properColumnBuilders); - releaseWindow(current); + closeWindow(activeWindows.removeFirst()); } + emitReadyWindows(properColumnBuilders, true); + Arrays.fill(lastNonemptyWindows, null); } @Override public final void beforeDestroy() { activeWindows.clear(); + pendingWindows.clear(); + Arrays.fill(lastNonemptyWindows, null); // Also covers a reservation made by a window whose construction failed at the limit. memoryBudget.reservedBytes = 0; } - private void emitWindow(W current, W next, List builders) { + private void emitWindow(W current, List builders) { int[] selectedIndexes = new int[participantColumns.length]; for (int i = 0; i < participantColumns.length; i++) { PointBuffer bucket = current.buffers[i]; @@ -914,12 +957,13 @@ private void emitWindow(W current, W next, List builders) { } double nextTime; double nextValue; - if (next != null && !next.buffers[i].isEmpty()) { - nextTime = next.buffers[i].averageTime(); - nextValue = next.buffers[i].averageValue(); + PointBuffer next = current.nextNonemptyBuffers[i]; + if (next != null) { + nextTime = next.averageTime(anchorTimes[i]); + nextValue = next.averageValue(); } else { int last = bucket.size() - 1; - nextTime = bucket.time(last); + nextTime = relativeTime(bucket.time(last), anchorTimes[i]); nextValue = bucket.value(last); } int pick = @@ -982,10 +1026,7 @@ protected void processRecord( } for (TimeWindowState window : activeWindows) { - // Closed windows waiting for their successor stay in the deque but must not absorb rows. - if (time < window.endExclusive) { - collect(input, time, window.buffers); - } + collect(input, time, window.buffers); } } } @@ -1017,9 +1058,7 @@ protected void processRecord( } for (CountWindowState window : activeWindows) { - if (rowCount < window.endExclusive) { - collect(input, time, window.buffers); - } + collect(input, time, window.buffers); } rowCount++; } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java new file mode 100644 index 0000000000000..d67d6be45a70d --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.udf.builtin.relational.tvf; + +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.i18n.CommonMessages; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; + +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import org.apache.tsfile.read.common.block.column.LongColumnBuilder; +import org.junit.Test; + +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class LTTBRegressionTest { + private static final long NANOSECOND_EPOCH = 1_780_000_000_000_000_000L; + + @Test + public void testTargetCountIsInvariantUnderTimestampTranslation() { + double[] values = {0, 1, 8, 2, 0, 0, 9, 1, -3, 0}; + long[] times = new long[values.length]; + for (int i = 0; i < times.length; i++) { + times[i] = i; + } + for (int n : new int[] {3, 4, 5}) { + int[] expected = LTTBTableFunction.selectTargetCount(times, values, times.length, n); + for (long origin : new long[] {NANOSECOND_EPOCH, Long.MIN_VALUE, Long.MAX_VALUE - 10}) { + long[] shifted = Arrays.stream(times).map(time -> time + origin).toArray(); + assertArrayEquals( + expected, LTTBTableFunction.selectTargetCount(shifted, values, shifted.length, n)); + } + } + } + + @Test + public void testWindowsAreInvariantUnderTimestampTranslation() { + Double[][] rows = {{0.0}, {1.0}, {8.0}, {2.0}, {0.0}, {0.0}}; + for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { + for (long slide : new long[] {1, 2, 3}) { + List expected = sample(mode, 2, slide, 0, rows); + List actual = sample(mode, 2, slide, NANOSECOND_EPOCH, rows); + int timeIndex = actual.size() - 2; + assertEquals( + expected.get(timeIndex).getPositionCount(), actual.get(timeIndex).getPositionCount()); + for (int i = 0; i < actual.get(timeIndex).getPositionCount(); i++) { + assertEquals( + expected.get(timeIndex).build().getLong(i), + actual.get(timeIndex).build().getLong(i) - NANOSECOND_EPOCH); + assertEquals( + expected.get(timeIndex + 1).build().getDouble(i), + actual.get(timeIndex + 1).build().getDouble(i), + 0.0); + } + } + } + } + + @Test + public void testWindowsSkipNullBucketsIndependentlyPerColumn() { + Double[][] rows = { + {0.0, 0.0}, {1.0, 0.0}, {2.0, 0.0}, + {null, 0.0}, {null, 1.0}, {null, 2.0}, + {null, null}, {null, null}, {null, null}, + {0.0, 0.0}, {0.0, 0.0}, {0.0, 0.0} + }; + for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { + List out = sample(mode, 3, 3, 0, rows); + int timeIndex = mode.equals("TIME_WINDOW") ? 2 : 1; + assertEquals(4, out.get(0).getPositionCount()); + assertEquals(2, out.get(timeIndex).build().getLong(0)); + assertTrue(out.get(timeIndex).build().isNull(1)); + assertTrue(out.get(timeIndex).build().isNull(2)); + assertEquals(9, out.get(timeIndex).build().getLong(3)); + assertEquals(2, out.get(timeIndex + 2).build().getLong(0)); + assertEquals(5, out.get(timeIndex + 2).build().getLong(1)); + assertTrue(out.get(timeIndex + 2).build().isNull(2)); + // This column keeps its own anchor at (5, 2), so the final bucket chooses t=9. + assertEquals(9, out.get(timeIndex + 2).build().getLong(3)); + } + } + + @Test + public void testTrailingNullWindowsUseFinalNonemptyBucketFallback() { + Double[][] rows = {{0.0}, {5.0}, {0.0}, {null}, {null}, {null}}; + for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { + List out = sample(mode, 3, 3, 0, rows); + int timeIndex = out.size() - 2; + assertEquals(2, out.get(timeIndex).getPositionCount()); + assertEquals(1, out.get(timeIndex).build().getLong(0)); + assertTrue(out.get(timeIndex).build().isNull(1)); + } + } + + @Test + public void testTargetColumnsShareMemoryBudget() { + TableFunctionDataProcessor processor = processor("TARGET_COUNT", 3, 3, 0, 8); + Record input = record(0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0); + try { + for (int i = 0; i < 350_000; i++) { + processor.process(input, Collections.emptyList(), null); + } + fail( + "Expected a controlled memory-limit exception before buffering 350000 rows in eight columns"); + } catch (SemanticException expected) { + // Match the same localized message as the window-memory regression suite. + assertEquals( + String.format( + CommonMessages + .EXCEPTION_LTTB_BUFFERS_EXCEED_THE_PER_PROCESSOR_MEMORY_LIMIT_OF_ARG_BYTES_954FEF00, + LTTBTableFunction.MAX_BUFFER_BYTES), + expected.getMessage()); + } finally { + processor.beforeDestroy(); + } + } + + private static List sample( + String mode, long size, long slide, long origin, Double[][] rows) { + int columns = rows[0].length; + TableFunctionDataProcessor processor = processor(mode, size, slide, origin, columns); + List builders = new ArrayList<>(); + builders.add(new LongColumnBuilder(null, 16)); + if (mode.equals("TIME_WINDOW")) { + builders.add(new LongColumnBuilder(null, 16)); + } + for (int i = 0; i < columns; i++) { + builders.add(new LongColumnBuilder(null, 16)); + builders.add(new DoubleColumnBuilder(null, 16)); + } + try { + for (int i = 0; i < rows.length; i++) { + processor.process(record(origin + i, rows[i]), builders, null); + } + processor.finish(builders, null); + return builders; + } finally { + processor.beforeDestroy(); + } + } + + private static TableFunctionDataProcessor processor( + String mode, long size, long slide, long origin, int columns) { + MapTableFunctionHandle handle = + new MapTableFunctionHandle.Builder() + .addProperty(LTTBTableFunction.MODE_PROPERTY, mode) + .addProperty(LTTBTableFunction.PARTITION_TYPES_PROPERTY, "") + .addProperty( + LTTBTableFunction.PARTICIPANT_TYPES_PROPERTY, + String.join(",", Collections.nCopies(columns, "DOUBLE"))) + .addProperty(LTTBTableFunction.N_PARAMETER_NAME, size) + .addProperty(LTTBTableFunction.SIZE_PARAMETER_NAME, size) + .addProperty(LTTBTableFunction.SLIDE_PARAMETER_NAME, slide) + .addProperty(LTTBTableFunction.ORIGIN_PARAMETER_NAME, origin) + .build(); + return new LTTBTableFunction().getProcessorProvider(handle).getDataProcessor(); + } + + private static Record record(long time, Double... values) { + // A lightweight row avoids Mockito invocation bookkeeping in the memory-limit test. + return (Record) + Proxy.newProxyInstance( + Record.class.getClassLoader(), + new Class[] {Record.class}, + (proxy, method, arguments) -> { + int index = (int) arguments[0]; + switch (method.getName()) { + case "getLong": + return time; + case "getDouble": + return values[index - 1]; + case "isNull": + return index != 0 && values[index - 1] == null; + default: + throw new UnsupportedOperationException(method.getName()); + } + }); + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java index a09afcbc21d51..dcdf0ed3dedde 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java @@ -56,7 +56,7 @@ public void testCountWindowOverlapFailsBeforeExhaustingHeap() { processor.process(input, Collections.emptyList(), null); } }, - LTTBTableFunction.MAX_WINDOW_BUFFER_BYTES); + LTTBTableFunction.MAX_BUFFER_BYTES); } finally { processor.beforeDestroy(); } @@ -70,7 +70,7 @@ public void testTimeWindowCreationIsBoundedEvenForNullInput() { // must be charged even when the input column contains no eligible points. assertMemoryLimit( () -> processor.process(record(0, (Number) null), Collections.emptyList(), null), - LTTBTableFunction.MAX_WINDOW_BUFFER_BYTES); + LTTBTableFunction.MAX_BUFFER_BYTES); } finally { processor.beforeDestroy(); } @@ -87,7 +87,7 @@ public void testColumnsShareTheWindowBudget() { processor.process(input, Collections.emptyList(), null); } }, - LTTBTableFunction.MAX_WINDOW_BUFFER_BYTES); + LTTBTableFunction.MAX_BUFFER_BYTES); } finally { processor.beforeDestroy(); } @@ -96,7 +96,7 @@ public void testColumnsShareTheWindowBudget() { @Test public void testArrayGrowthReservesOldAndNewIntegralArrays() { LTTBTableFunction.PointBuffer buffer = - new LTTBTableFunction.PointBuffer(true, new LTTBTableFunction.WindowMemoryBudget(4096)); + new LTTBTableFunction.PointBuffer(true, new LTTBTableFunction.BufferMemoryBudget(4096)); for (int i = 0; i < 64; i++) { buffer.addIntegral(i, Long.MAX_VALUE); } @@ -109,7 +109,7 @@ public void testArrayGrowthReservesOldAndNewIntegralArrays() { @Test public void testInitialBuffersShareCapacityBudget() { - LTTBTableFunction.WindowMemoryBudget budget = new LTTBTableFunction.WindowMemoryBudget(4096); + LTTBTableFunction.BufferMemoryBudget budget = new LTTBTableFunction.BufferMemoryBudget(4096); new LTTBTableFunction.PointBuffer(true, budget); new LTTBTableFunction.PointBuffer(true, budget); assertMemoryLimit(() -> new LTTBTableFunction.PointBuffer(true, budget), 4096); @@ -213,7 +213,10 @@ private static void assertMemoryLimit(Runnable action, long limit) { fail("Expected the shared LTTB window memory limit"); } catch (SemanticException e) { assertEquals( - String.format(CommonMessages.LTTB_WINDOW_BUFFER_MEMORY_LIMIT_EXCEEDED, limit), + String.format( + CommonMessages + .EXCEPTION_LTTB_BUFFERS_EXCEED_THE_PER_PROCESSOR_MEMORY_LIMIT_OF_ARG_BYTES_954FEF00, + limit), e.getMessage()); } } From 4074d36abf93ce987d09d748a26a0cc1d49730ba Mon Sep 17 00:00:00 2001 From: DaZuiZui Date: Thu, 8 Oct 2026 17:27:04 +0800 Subject: [PATCH 04/21] fix(table): preserve LTTB integer precision and guard window bounds --- .../it/db/it/IoTDBLTTBTableFunctionIT.java | 36 +++ .../iotdb/commons/i18n/CommonMessages.java | 2 + .../iotdb/commons/i18n/CommonMessages.java | 2 + .../relational/tvf/LTTBTableFunction.java | 158 ++++++++--- .../tvf/LTTBNumericBoundaryTest.java | 255 ++++++++++++++++++ 5 files changed, 415 insertions(+), 38 deletions(-) create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java index 564ccb4e4d169..4596dabc07102 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java @@ -69,6 +69,17 @@ public static void setUp() throws Exception { + "(0, 'd1', 0.0, 1), (1, 'd1', 1.0, 1), (2, 'd1', 2.0, 1)," + "(3, 'd1', null, 1), (4, 'd1', null, 1), (5, 'd1', null, 1)," + "(6, 'd1', 0.0, 1), (7, 'd1', 0.0, 1), (8, 'd1', 0.0, 1)"); + statement.execute("CREATE TABLE integral_values(device STRING TAG, value INT64 FIELD)"); + long baseline = 1L << 60; + long[] deltas = {0, 1, 8, 2, 0, 0}; + for (int i = 0; i < deltas.length; i++) { + statement.execute( + "INSERT INTO integral_values(time, device, value) VALUES (" + + i + + ", 'd1', " + + (baseline + deltas[i]) + + ")"); + } statement.execute("FLUSH"); } catch (Exception e) { fail("insertData failed: " + e.getMessage()); @@ -146,6 +157,31 @@ public void testWindowSkipsNullBucketForLookahead() { DATABASE_NAME); } + @Test + public void testInt64SamplingPreservesSmallPeaksAtLargeBaselines() { + String prefix = + "SELECT * FROM LTTB(DATA => integral_values PARTITION BY device ORDER BY time, " + + "TIMECOL => 'time', "; + tableResultSetEqualTest( + prefix + "N => 3) ORDER BY value_time", + new String[] {"window_index", "device", "value_time", "value"}, + new String[] { + "0,d1,1970-01-01T00:00:00.000Z,1152921504606846976,", + "0,d1,1970-01-01T00:00:00.002Z,1152921504606846984,", + "0,d1,1970-01-01T00:00:00.005Z,1152921504606846976," + }, + DATABASE_NAME); + tableResultSetEqualTest( + prefix + "SIZE => 2) ORDER BY window_index", + new String[] {"window_index", "device", "value_time", "value"}, + new String[] { + "0,d1,1970-01-01T00:00:00.001Z,1152921504606846977,", + "1,d1,1970-01-01T00:00:00.002Z,1152921504606846984,", + "2,d1,1970-01-01T00:00:00.004Z,1152921504606846976," + }, + DATABASE_NAME); + } + @Test public void testInvalidParameterCombinations() { String prefix = diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java index 44edd90841d01..d390afba9a19f 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -361,6 +361,8 @@ private CommonMessages() {} "LTTB buffers at most %d eligible points per column in one partition or window."; public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = "Unsupported LTTB value type: %s"; + public static final String EXCEPTION_LTTB_WINDOW_BOUNDARIES_EXCEED_THE_TIMESTAMP_RANGE_7C0FC7E2 = + "LTTB window boundaries exceed the TIMESTAMP range."; public static final String EXCEPTION_LTTB_BUFFERS_EXCEED_THE_PER_PROCESSOR_MEMORY_LIMIT_OF_ARG_BYTES_954FEF00 = "LTTB buffers exceed the per-processor memory limit of %d bytes."; diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java index 7b2ed93291527..343875c4eb42e 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -257,6 +257,8 @@ private CommonMessages() {} "LTTB 在单个分区或窗口内每列最多缓存 %d 个有效点。"; public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = "不支持的 LTTB 值类型:%s"; + public static final String EXCEPTION_LTTB_WINDOW_BOUNDARIES_EXCEED_THE_TIMESTAMP_RANGE_7C0FC7E2 = + "LTTB 窗口边界超出 TIMESTAMP 的取值范围。"; public static final String EXCEPTION_LTTB_BUFFERS_EXCEED_THE_PER_PROCESSOR_MEMORY_LIMIT_OF_ARG_BYTES_954FEF00 = "LTTB 缓冲区超过每个处理器 %d 字节的内存上限。"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index ad14d83744419..62e890aa190c1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -41,6 +41,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; +import java.math.BigInteger; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; @@ -333,6 +334,11 @@ private static ParticipantColumn[] createParticipantColumns(Type[] types, int fi * @return ascending indexes of the selected points */ static int[] selectTargetCount(long[] times, double[] values, int count, long n) { + return selectTargetCount(times, values, null, count, n); + } + + private static int[] selectTargetCount( + long[] times, double[] values, long[] integralValues, int count, long n) { if (count <= n) { int[] all = new int[count]; for (int i = 0; i < count; i++) { @@ -352,18 +358,20 @@ static int[] selectTargetCount(long[] times, double[] values, int count, long n) int currentEnd = (int) ((bucket + 1) * intermediate / bucketCount) + 1; long anchorTime = times[anchor]; + double anchorValue = values[anchor]; + long integralAnchor = integralValues == null ? 0 : integralValues[anchor]; double avgTime; double avgValue; if (bucket == bucketCount - 1) { - avgTime = relativeTime(times[count - 1], anchorTime); - avgValue = values[count - 1]; + avgTime = differenceAsDouble(times[count - 1], anchorTime); + avgValue = relativeValue(values, integralValues, count - 1, anchorValue, integralAnchor); } else { int nextEnd = (int) ((bucket + 2) * intermediate / bucketCount) + 1; double sumTime = 0; double sumValue = 0; for (int i = currentEnd; i < nextEnd; i++) { - sumTime += relativeTime(times[i], anchorTime); - sumValue += values[i]; + sumTime += differenceAsDouble(times[i], anchorTime); + sumValue += relativeValue(values, integralValues, i, anchorValue, integralAnchor); } int nextSize = nextEnd - currentEnd; avgTime = sumTime / nextSize; @@ -374,10 +382,12 @@ static int[] selectTargetCount(long[] times, double[] values, int count, long n) argMaxArea( times, values, + integralValues, currentStart, currentEnd, anchorTime, - values[anchor], + anchorValue, + integralAnchor, avgTime, avgValue); selected[selectedCount++] = pick; @@ -389,16 +399,18 @@ static int[] selectTargetCount(long[] times, double[] values, int count, long n) /** * Returns the index in {@code [start, end)} maximizing the triangle area; earliest wins ties. - * {@code nextTime} is relative to {@code anchorTime}, avoiding rounding absolute nanosecond - * epochs. + * {@code nextTime} and {@code nextValue} are relative to the anchor. Integral differences are + * computed before converting to double so large epochs and INT64 baselines retain small changes. */ - static int argMaxArea( + private static int argMaxArea( long[] times, double[] values, + long[] integralValues, int start, int end, long anchorTime, double anchorValue, + long integralAnchor, double nextTime, double nextValue) { int pick = start; @@ -406,8 +418,8 @@ static int argMaxArea( for (int i = start; i < end; i++) { double area = Math.abs( - nextTime * (values[i] - anchorValue) - - relativeTime(times[i], anchorTime) * (nextValue - anchorValue)); + nextTime * relativeValue(values, integralValues, i, anchorValue, integralAnchor) + - differenceAsDouble(times[i], anchorTime) * nextValue); if (area > maxArea) { maxArea = area; pick = i; @@ -416,12 +428,19 @@ static int argMaxArea( return pick; } - private static double relativeTime(long time, long origin) { - long difference = time - origin; - // Subtract before converting so nearby timestamps retain their precision. For spans exceeding + private static double relativeValue( + double[] values, long[] integralValues, int index, double origin, long integralOrigin) { + return integralValues == null + ? values[index] - origin + : differenceAsDouble(integralValues[index], integralOrigin); + } + + private static double differenceAsDouble(long value, long origin) { + long difference = value - origin; + // Subtract before converting so nearby integers retain their precision. For spans exceeding // the long range, use floating-point subtraction instead of the overflowed long difference. - return ((time ^ origin) & (time ^ difference)) < 0 - ? (double) time - (double) origin + return ((value ^ origin) & (value ^ difference)) < 0 + ? (double) value - (double) origin : (double) difference; } @@ -621,15 +640,15 @@ long time(int index) { double averageTime(long origin) { double sum = 0; for (int i = 0; i < size; i++) { - sum += relativeTime(times[i], origin); + sum += differenceAsDouble(times[i], origin); } return sum / size; } - double averageValue() { + double averageValue(double origin, long integralOrigin) { double sum = 0; for (int i = 0; i < size; i++) { - sum += values[i]; + sum += relativeValue(values, integralValues, i, origin, integralOrigin); } return sum / size; } @@ -745,7 +764,8 @@ public void finish( int rowCount = 1; for (int i = 0; i < participantColumns.length; i++) { PointBuffer buffer = buffers[i]; - selections[i] = selectTargetCount(buffer.times, buffer.values, buffer.size, n); + selections[i] = + selectTargetCount(buffer.times, buffer.values, buffer.integralValues, buffer.size, n); rowCount = Math.max(rowCount, selections[i].length); } @@ -847,6 +867,7 @@ private abstract static class AbstractWindowLTTBDataProcessor builders) { hasAnchor[i] = true; anchorTimes[i] = bucket.time(0); anchorValues[i] = bucket.value(0); + if (bucket.integralValues != null) { + integralAnchorValues[i] = bucket.integralValues[0]; + } } double nextTime; double nextValue; PointBuffer next = current.nextNonemptyBuffers[i]; if (next != null) { nextTime = next.averageTime(anchorTimes[i]); - nextValue = next.averageValue(); + nextValue = next.averageValue(anchorValues[i], integralAnchorValues[i]); } else { int last = bucket.size() - 1; - nextTime = relativeTime(bucket.time(last), anchorTimes[i]); - nextValue = bucket.value(last); + nextTime = differenceAsDouble(bucket.time(last), anchorTimes[i]); + nextValue = + relativeValue( + bucket.values, + bucket.integralValues, + last, + anchorValues[i], + integralAnchorValues[i]); } int pick = argMaxArea( bucket.times, bucket.values, + bucket.integralValues, 0, bucket.size(), anchorTimes[i], anchorValues[i], + integralAnchorValues[i], nextTime, nextValue); selectedIndexes[i] = pick; anchorTimes[i] = bucket.time(pick); anchorValues[i] = bucket.value(pick); + if (bucket.integralValues != null) { + integralAnchorValues[i] = bucket.integralValues[pick]; + } } current.writeWindowColumns(builders); writeRow(builders, current.getWindowColumnCount(), current.buffers, selectedIndexes); @@ -989,6 +1025,7 @@ private static final class TimeWindowLTTBDataProcessor extends AbstractWindowLTTBDataProcessor { private final long origin; private boolean nextWindowStartInitialized = false; + private boolean windowsExhausted; private long nextWindowStart; private TimeWindowLTTBDataProcessor( @@ -1006,29 +1043,74 @@ protected void processRecord( Record input, long time, List properColumnBuilders) { emitClosedWindows(time, properColumnBuilders); - long firstCandidateStart = - origin + Math.floorDiv(time - origin - size, slide) * slide + slide; - while (getWindowEnd(firstCandidateStart) <= time) { - firstCandidateStart += slide; - } - if (!nextWindowStartInitialized) { - nextWindowStart = firstCandidateStart; - nextWindowStartInitialized = true; - } else if (nextWindowStart < firstCandidateStart) { - nextWindowStart = firstCandidateStart; - } + if (!windowsExhausted) { + long firstCandidateStart = getFirstWindowStart(time); + if (!nextWindowStartInitialized || nextWindowStart < firstCandidateStart) { + nextWindowStart = firstCandidateStart; + nextWindowStartInitialized = true; + } - while (nextWindowStart <= time && getWindowEnd(nextWindowStart) > time) { - activeWindows.addLast( - new TimeWindowState( - newWindowBuffers(), nextWindowStart, getWindowEnd(nextWindowStart))); - nextWindowStart += slide; + while (!windowsExhausted && nextWindowStart <= time) { + long windowEnd; + try { + windowEnd = Math.addExact(nextWindowStart, size); + } catch (ArithmeticException e) { + throw windowBoundaryOverflow(); + } + activeWindows.addLast( + new TimeWindowState(newWindowBuffers(), nextWindowStart, windowEnd)); + // An unrepresentable next start means there are no more windows, but the current + // window can still be valid and must collect this and subsequent input rows. + if (nextWindowStart > Long.MAX_VALUE - slide) { + windowsExhausted = true; + } else { + nextWindowStart += slide; + } + } } for (TimeWindowState window : activeWindows) { collect(input, time, window.buffers); } } + + /** First aligned start whose window ends strictly after the current timestamp. */ + private long getFirstWindowStart(long time) { + try { + long distance = Math.subtractExact(Math.subtractExact(time, origin), size); + long steps = Math.addExact(Math.floorDiv(distance, slide), 1); + return Math.addExact(origin, Math.multiplyExact(steps, slide)); + } catch (ArithmeticException e) { + // Only intermediate arithmetic may overflow, e.g. time near Long.MIN_VALUE. + // Fall back to exact arithmetic before deciding whether the actual boundary is valid. + BigInteger step = BigInteger.valueOf(slide); + BigInteger[] quotientAndRemainder = + BigInteger.valueOf(time) + .subtract(BigInteger.valueOf(origin)) + .subtract(BigInteger.valueOf(size)) + .divideAndRemainder(step); + BigInteger steps = quotientAndRemainder[0]; + if (quotientAndRemainder[1].signum() >= 0) { + steps = steps.add(BigInteger.ONE); + } + BigInteger start = BigInteger.valueOf(origin).add(steps.multiply(step)); + if (start.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + // The input lies in a gap and the next aligned window is beyond the time domain. + windowsExhausted = true; + return Long.MAX_VALUE; + } + try { + return start.longValueExact(); + } catch (ArithmeticException overflow) { + throw windowBoundaryOverflow(); + } + } + } + + private SemanticException windowBoundaryOverflow() { + return new SemanticException( + CommonMessages.EXCEPTION_LTTB_WINDOW_BOUNDARIES_EXCEED_THE_TIMESTAMP_RANGE_7C0FC7E2); + } } private static final class CountWindowLTTBDataProcessor diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java new file mode 100644 index 0000000000000..15b05cd02bb09 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.udf.builtin.relational.tvf; + +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.i18n.CommonMessages; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.column.LongColumnBuilder; +import org.junit.Test; + +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +public class LTTBNumericBoundaryTest { + + @Test + public void testInt64TargetCountPreservesPeakAtLargeBaselines() { + Long[][] values = {{0L}, {1L}, {8L}, {2L}, {0L}, {0L}}; + for (long offset : new long[] {0, 1L << 60, Long.MIN_VALUE, Long.MAX_VALUE - 8}) { + Column[] output = sample("TARGET_COUNT", 3, 3, 0, times(6), translate(values, offset)); + assertArrayEquals(new long[] {0, 2, 5}, longValues(output[1])); + assertArrayEquals(new long[] {offset, offset + 8, offset}, longValues(output[2])); + } + } + + @Test + public void testInt64TargetCountUsesRelativeNextBucketAverage() { + Long[][] values = {{0L}, {1L}, {8L}, {2L}, {0L}, {0L}, {9L}, {1L}, {3L}, {0L}}; + long[][] expected = {{0, 6, 9}, {0, 2, 6, 9}, {0, 2, 4, 6, 9}}; + for (long offset : new long[] {1L << 60, Long.MIN_VALUE, Long.MAX_VALUE - 9}) { + for (int n = 3; n <= 5; n++) { + Column[] output = sample("TARGET_COUNT", n, n, 0, times(10), translate(values, offset)); + assertArrayEquals(expected[n - 3], longValues(output[1])); + for (int row = 0; row < output[1].getPositionCount(); row++) { + int selectedTime = (int) output[1].getLong(row); + assertEquals(offset + values[selectedTime][0], output[2].getLong(row)); + } + } + } + } + + @Test + public void testInt64WindowsKeepIndependentAnchorsAcrossNullBuckets() { + Long[][] values = { + {0L, 0L}, {1L, 0L}, {2L, 0L}, + {null, 0L}, {null, 1L}, {null, 2L}, + {null, null}, {null, null}, {null, null}, + {0L, 0L}, {0L, 0L}, {0L, 0L} + }; + long[] offsets = {1L << 60, Long.MIN_VALUE}; + for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { + for (long slide : new long[] {1, 3, 4}) { + Column[] expected = sample(mode, 3, slide, 0, times(12), values); + Column[] actual = sample(mode, 3, slide, 0, times(12), translate(values, offsets)); + int firstValueColumn = mode.equals("TIME_WINDOW") ? 3 : 2; + for (int column = 0; column < actual.length; column++) { + assertEquals(expected[column].getPositionCount(), actual[column].getPositionCount()); + for (int row = 0; row < actual[column].getPositionCount(); row++) { + assertEquals(expected[column].isNull(row), actual[column].isNull(row)); + if (!actual[column].isNull(row)) { + long offset = + column >= firstValueColumn && (column - firstValueColumn) % 2 == 0 + ? offsets[(column - firstValueColumn) / 2] + : 0; + assertEquals(expected[column].getLong(row) + offset, actual[column].getLong(row)); + } + } + } + } + } + } + + @Test + public void testInt64DifferencesSpanningLongRangeDoNotWrap() { + Long[][] values = { + {Long.MIN_VALUE}, {0L}, {Long.MAX_VALUE}, {0L}, {Long.MIN_VALUE}, {Long.MIN_VALUE} + }; + Column[] target = sample("TARGET_COUNT", 3, 3, 0, times(6), values); + assertArrayEquals(new long[] {0, 2, 5}, longValues(target[1])); + for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { + Column[] output = sample(mode, 2, 2, 0, times(6), values); + assertArrayEquals(new long[] {1, 2, 4}, longValues(output[output.length - 2])); + } + } + + @Test(timeout = 5000) + public void testTimeWindowRejectsUnrepresentableEnd() { + assertWindowOverflow(2, 2, 0, Long.MAX_VALUE - 1); + assertWindowOverflow(1, 1, 0, Long.MAX_VALUE); + } + + @Test(timeout = 5000) + public void testTimeWindowRejectsUnrepresentableStart() { + assertWindowOverflow(3, 3, 0, Long.MIN_VALUE); + } + + @Test(timeout = 5000) + public void testTimeWindowAllowsIntermediateOverflow() { + for (long origin : new long[] {0, Long.MIN_VALUE, Long.MAX_VALUE - 1}) { + Column[] output = + sample("TIME_WINDOW", 2, 2, origin, new long[] {Long.MIN_VALUE}, new Long[][] {{7L}}); + assertEquals(1, output[0].getPositionCount()); + assertEquals(Long.MIN_VALUE, output[0].getLong(0)); + assertEquals(Long.MIN_VALUE + 2, output[1].getLong(0)); + assertEquals(Long.MIN_VALUE, output[2].getLong(0)); + assertEquals(7L, output[3].getLong(0)); + } + } + + @Test(timeout = 5000) + public void testTimeWindowStopsAdvancingPastLongMax() { + Column[] output = + sample( + "TIME_WINDOW", + 2, + 3, + Long.MAX_VALUE - 2, + new long[] {Long.MAX_VALUE - 2, Long.MAX_VALUE - 1, Long.MAX_VALUE}, + new Long[][] {{0L}, {8L}, {1L}}); + assertEquals(1, output[0].getPositionCount()); + assertEquals(Long.MAX_VALUE - 2, output[0].getLong(0)); + assertEquals(Long.MAX_VALUE, output[1].getLong(0)); + // Both candidate endpoints form zero-area triangles. The last input is in the following gap. + assertEquals(Long.MAX_VALUE - 2, output[2].getLong(0)); + } + + @Test(timeout = 5000) + public void testTimeWindowSkipsGapBeforeUnrepresentableNextStart() { + Column[] output = + sample( + "TIME_WINDOW", + 2, + 3, + Long.MAX_VALUE - 2, + new long[] {Long.MAX_VALUE}, + new Long[][] {{1L}}); + assertEquals(0, output[0].getPositionCount()); + } + + private static void assertWindowOverflow(long size, long slide, long origin, long time) { + SemanticException failure = + assertThrows( + SemanticException.class, + () -> + sample("TIME_WINDOW", size, slide, origin, new long[] {time}, new Long[][] {{1L}})); + assertEquals( + CommonMessages.EXCEPTION_LTTB_WINDOW_BOUNDARIES_EXCEED_THE_TIMESTAMP_RANGE_7C0FC7E2, + failure.getMessage()); + } + + private static long[] longValues(Column column) { + long[] values = new long[column.getPositionCount()]; + for (int i = 0; i < values.length; i++) { + values[i] = column.getLong(i); + } + return values; + } + + private static long[] times(int count) { + long[] times = new long[count]; + for (int i = 0; i < count; i++) { + times[i] = i; + } + return times; + } + + private static Long[][] translate(Long[][] values, long... offsets) { + Long[][] translated = new Long[values.length][offsets.length]; + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < offsets.length; j++) { + translated[i][j] = values[i][j] == null ? null : Math.addExact(values[i][j], offsets[j]); + } + } + return translated; + } + + private static Column[] sample( + String mode, long size, long slide, long origin, long[] times, Long[][] values) { + int columns = values[0].length; + MapTableFunctionHandle handle = + new MapTableFunctionHandle.Builder() + .addProperty(LTTBTableFunction.MODE_PROPERTY, mode) + .addProperty(LTTBTableFunction.PARTITION_TYPES_PROPERTY, "") + .addProperty( + LTTBTableFunction.PARTICIPANT_TYPES_PROPERTY, + String.join(",", Collections.nCopies(columns, "INT64"))) + .addProperty(LTTBTableFunction.N_PARAMETER_NAME, size) + .addProperty(LTTBTableFunction.SIZE_PARAMETER_NAME, size) + .addProperty(LTTBTableFunction.SLIDE_PARAMETER_NAME, slide) + .addProperty(LTTBTableFunction.ORIGIN_PARAMETER_NAME, origin) + .build(); + TableFunctionDataProcessor processor = + new LTTBTableFunction().getProcessorProvider(handle).getDataProcessor(); + List builders = new ArrayList<>(); + int outputColumns = (mode.equals("TIME_WINDOW") ? 2 : 1) + 2 * columns; + for (int i = 0; i < outputColumns; i++) { + builders.add(new LongColumnBuilder(null, 16)); + } + try { + for (int i = 0; i < times.length; i++) { + processor.process(record(times[i], values[i]), builders, null); + } + processor.finish(builders, null); + return builders.stream().map(ColumnBuilder::build).toArray(Column[]::new); + } finally { + processor.beforeDestroy(); + } + } + + private static Record record(long time, Long[] values) { + return (Record) + Proxy.newProxyInstance( + Record.class.getClassLoader(), + new Class[] {Record.class}, + (proxy, method, arguments) -> { + int index = (int) arguments[0]; + switch (method.getName()) { + case "getLong": + return index == 0 ? time : values[index - 1]; + case "isNull": + return index != 0 && values[index - 1] == null; + default: + throw new UnsupportedOperationException(method.getName()); + } + }); + } +} From edbb76aba3b5dbd112e861580bba62ec67d54aec Mon Sep 17 00:00:00 2001 From: DaZuiZui Date: Fri, 9 Oct 2026 19:28:44 +0800 Subject: [PATCH 05/21] fix(table): handle LTTB sampling and count-window overflow Recompute a bucket with exact arithmetic when floating-point area or lookahead calculations overflow, preserving finite peaks and stable ties. Keep the existing fast path for ordinary input values. Check count-window ends only when their start is reached, reject an unrepresentable end, and stop advancing before the next start overflows. This prevents stalled processing and spurious windows for large sizes. Add six numeric boundary regressions covering all three sampling modes, lookahead sums, opposite-sign differences, area ties, and count bounds. --- .../iotdb/commons/i18n/CommonMessages.java | 2 + .../iotdb/commons/i18n/CommonMessages.java | 2 + .../relational/tvf/LTTBTableFunction.java | 123 ++++++++++++++++-- .../tvf/LTTBNumericBoundaryTest.java | 94 ++++++++++++- 4 files changed, 205 insertions(+), 16 deletions(-) diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java index d390afba9a19f..dbf8daca9db4e 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -361,6 +361,8 @@ private CommonMessages() {} "LTTB buffers at most %d eligible points per column in one partition or window."; public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = "Unsupported LTTB value type: %s"; + public static final String EXCEPTION_LTTB_COUNT_WINDOW_BOUNDARIES_EXCEED_THE_INT64_RANGE = + "LTTB count window boundaries exceed the INT64 range."; public static final String EXCEPTION_LTTB_WINDOW_BOUNDARIES_EXCEED_THE_TIMESTAMP_RANGE_7C0FC7E2 = "LTTB window boundaries exceed the TIMESTAMP range."; public static final String diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java index 343875c4eb42e..a12fa965693e0 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -257,6 +257,8 @@ private CommonMessages() {} "LTTB 在单个分区或窗口内每列最多缓存 %d 个有效点。"; public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = "不支持的 LTTB 值类型:%s"; + public static final String EXCEPTION_LTTB_COUNT_WINDOW_BOUNDARIES_EXCEED_THE_INT64_RANGE = + "LTTB 计数窗口边界超出 INT64 的取值范围。"; public static final String EXCEPTION_LTTB_WINDOW_BOUNDARIES_EXCEED_THE_TIMESTAMP_RANGE_7C0FC7E2 = "LTTB 窗口边界超出 TIMESTAMP 的取值范围。"; public static final String diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index 62e890aa190c1..d2a6a9f4e4794 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -41,6 +41,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; +import java.math.BigDecimal; import java.math.BigInteger; import java.util.ArrayDeque; import java.util.ArrayList; @@ -362,11 +363,12 @@ private static int[] selectTargetCount( long integralAnchor = integralValues == null ? 0 : integralValues[anchor]; double avgTime; double avgValue; + int nextEnd = + bucket == bucketCount - 1 ? count : (int) ((bucket + 2) * intermediate / bucketCount) + 1; if (bucket == bucketCount - 1) { avgTime = differenceAsDouble(times[count - 1], anchorTime); avgValue = relativeValue(values, integralValues, count - 1, anchorValue, integralAnchor); } else { - int nextEnd = (int) ((bucket + 2) * intermediate / bucketCount) + 1; double sumTime = 0; double sumValue = 0; for (int i = currentEnd; i < nextEnd; i++) { @@ -390,6 +392,23 @@ private static int[] selectTargetCount( integralAnchor, avgTime, avgValue); + if (pick == NOT_SELECTED) { + pick = + argMaxAreaExactly( + times, + values, + integralValues, + currentStart, + currentEnd, + anchorTime, + anchorValue, + integralAnchor, + times, + values, + integralValues, + currentEnd, + nextEnd); + } selected[selectedCount++] = pick; anchor = pick; } @@ -401,6 +420,8 @@ private static int[] selectTargetCount( * Returns the index in {@code [start, end)} maximizing the triangle area; earliest wins ties. * {@code nextTime} and {@code nextValue} are relative to the anchor. Integral differences are * computed before converting to double so large epochs and INT64 baselines retain small changes. + * Returns {@link #NOT_SELECTED} if intermediate arithmetic overflows, so the caller can compare + * the entire bucket using its original points and exact arithmetic. */ private static int argMaxArea( long[] times, @@ -420,6 +441,9 @@ private static int argMaxArea( Math.abs( nextTime * relativeValue(values, integralValues, i, anchorValue, integralAnchor) - differenceAsDouble(times[i], anchorTime) * nextValue); + if (!Double.isFinite(area)) { + return NOT_SELECTED; + } if (area > maxArea) { maxArea = area; pick = i; @@ -428,6 +452,56 @@ private static int argMaxArea( return pick; } + /** + * Overflow fallback for finite input values. Recomputes every candidate so an overflowing area + * cannot tie with or hide a larger one. Multiplying all areas by the lookahead point count lets + * us use sums instead of dividing by that count, keeping the comparison exact even for recurring + * averages. The normal path does not allocate decimal values. + */ + private static int argMaxAreaExactly( + long[] times, + double[] values, + long[] integralValues, + int start, + int end, + long anchorTime, + double anchorValue, + long integralAnchor, + long[] nextTimes, + double[] nextValues, + long[] nextIntegralValues, + int nextStart, + int nextEnd) { + BigDecimal timeOrigin = BigDecimal.valueOf(anchorTime); + BigDecimal valueOrigin = + integralValues == null ? new BigDecimal(anchorValue) : BigDecimal.valueOf(integralAnchor); + BigDecimal sumTime = BigDecimal.ZERO; + BigDecimal sumValue = BigDecimal.ZERO; + for (int i = nextStart; i < nextEnd; i++) { + sumTime = sumTime.add(BigDecimal.valueOf(nextTimes[i]).subtract(timeOrigin)); + sumValue = sumValue.add(exactValue(nextValues, nextIntegralValues, i).subtract(valueOrigin)); + } + int pick = start; + BigDecimal maxArea = null; + for (int i = start; i < end; i++) { + BigDecimal time = BigDecimal.valueOf(times[i]).subtract(timeOrigin); + BigDecimal value = exactValue(values, integralValues, i).subtract(valueOrigin); + BigDecimal area = sumTime.multiply(value).subtract(time.multiply(sumValue)).abs(); + if (maxArea == null || area.compareTo(maxArea) > 0) { + maxArea = area; + pick = i; + } + } + return pick; + } + + private static BigDecimal exactValue(double[] values, long[] integralValues, int index) { + // The constructor preserves the actual binary floating-point value, including tiny deltas. + return integralValues == null + ? new BigDecimal(values[index]) + : BigDecimal.valueOf(integralValues[index]); + } + private static double relativeValue( double[] values, long[] integralValues, int index, double origin, long integralOrigin) { return integralValues == null @@ -940,10 +1014,6 @@ private void emitReadyWindows(List builders, boolean finishing) { } } - protected final long getWindowEnd(long windowStart) { - return windowStart + size; - } - @Override public final void finish( List properColumnBuilders, ColumnBuilder passThroughIndexBuilder) { @@ -1009,6 +1079,24 @@ private void emitWindow(W current, List builders) { integralAnchorValues[i], nextTime, nextValue); + if (pick == NOT_SELECTED) { + PointBuffer lookahead = next == null ? bucket : next; + pick = + argMaxAreaExactly( + bucket.times, + bucket.values, + bucket.integralValues, + 0, + bucket.size(), + anchorTimes[i], + anchorValues[i], + integralAnchorValues[i], + lookahead.times, + lookahead.values, + lookahead.integralValues, + next == null ? bucket.size() - 1 : 0, + lookahead.size()); + } selectedIndexes[i] = pick; anchorTimes[i] = bucket.time(pick); anchorValues[i] = bucket.value(pick); @@ -1118,6 +1206,7 @@ private static final class CountWindowLTTBDataProcessor private long rowCount = 0; private long nextWindowStart = 0; private long nextWindowIndex = 0; + private boolean windowsExhausted; private CountWindowLTTBDataProcessor( long size, long slide, Type[] partitionTypes, ParticipantColumn[] participantColumns) { @@ -1129,14 +1218,24 @@ protected void processRecord( Record input, long time, List properColumnBuilders) { emitClosedWindows(rowCount, properColumnBuilders); - while (getWindowEnd(nextWindowStart) <= rowCount) { - nextWindowStart += slide; - } - while (nextWindowStart <= rowCount) { + // Input row positions advance one at a time, so no expired starts need to be skipped. + // Check the end only when a window is reached: a distant future end may be unrepresentable + // even though every window that actually receives input is valid. + if (!windowsExhausted && nextWindowStart == rowCount) { + long windowEnd; + try { + windowEnd = Math.addExact(nextWindowStart, size); + } catch (ArithmeticException e) { + throw new SemanticException( + CommonMessages.EXCEPTION_LTTB_COUNT_WINDOW_BOUNDARIES_EXCEED_THE_INT64_RANGE); + } activeWindows.addLast( - new CountWindowState( - newWindowBuffers(), getWindowEnd(nextWindowStart), nextWindowIndex++)); - nextWindowStart += slide; + new CountWindowState(newWindowBuffers(), windowEnd, nextWindowIndex++)); + if (nextWindowStart > Long.MAX_VALUE - slide) { + windowsExhausted = true; + } else { + nextWindowStart += slide; + } } for (CountWindowState window : activeWindows) { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java index 15b05cd02bb09..e306c4f38deaf 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java @@ -24,9 +24,11 @@ import org.apache.iotdb.udf.api.relational.access.Record; import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; +import org.apache.iotdb.udf.api.type.Type; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; import org.junit.Test; @@ -111,6 +113,79 @@ public void testInt64DifferencesSpanningLongRangeDoNotWrap() { } } + @Test + public void testFiniteDoublesPreserveSelectionsWhenAreasOverflow() { + assertFloatingScaleInvariant(new double[] {0, 4, 8, 2, 0, 0}); + } + + @Test + public void testFiniteDoublesPreserveSelectionsWhenLookaheadSumOverflows() { + assertFloatingScaleInvariant(new double[] {0, 4, 8, 7, 7, 0}); + } + + @Test + public void testFiniteDoublesPreserveSelectionsWhenDifferencesOverflow() { + assertFloatingScaleInvariant(new double[] {-8, 4, 8, 2, -8, -8}); + } + + @Test + public void testOverflowFallbackKeepsEarliestAreaTie() { + Column[] output = + sampleFloating("TARGET_COUNT", 3, new double[] {0, 1.6e308, 1.6e308, 0, 0, 0}); + assertArrayEquals(new long[] {0, 1, 5}, longValues(output[1])); + } + + @Test(timeout = 5000) + public void testCountWindowRejectsUnrepresentableEnd() { + SemanticException failure = + assertThrows( + SemanticException.class, + () -> + sample("COUNT_WINDOW", Long.MAX_VALUE, 1, 0, times(2), new Long[][] {{1L}, {2L}})); + assertEquals( + CommonMessages.EXCEPTION_LTTB_COUNT_WINDOW_BOUNDARIES_EXCEED_THE_INT64_RANGE, + failure.getMessage()); + } + + @Test(timeout = 5000) + public void testCountWindowWithLargeSlideDoesNotCreateExtraWindow() { + Column[] output = + sample( + "COUNT_WINDOW", Long.MAX_VALUE, Long.MAX_VALUE, 0, times(2), new Long[][] {{1L}, {2L}}); + assertArrayEquals(new long[] {0}, longValues(output[0])); + assertArrayEquals(new long[] {0}, longValues(output[1])); + } + + private static void assertFloatingScaleInvariant(double[] values) { + double[] scaled = new double[values.length]; + for (int i = 0; i < values.length; i++) { + scaled[i] = values[i] * 2e307; + } + for (String mode : new String[] {"TARGET_COUNT", "COUNT_WINDOW", "TIME_WINDOW"}) { + for (int size : new int[] {3, 4}) { + Column[] expected = sampleFloating(mode, size, values); + Column[] actual = sampleFloating(mode, size, scaled); + int timeColumn = actual.length - 2; + assertArrayEquals( + mode + ": " + size, longValues(expected[timeColumn]), longValues(actual[timeColumn])); + for (int row = 0; row < actual[timeColumn].getPositionCount(); row++) { + assertEquals( + scaled[(int) actual[timeColumn].getLong(row)], + actual[timeColumn + 1].getDouble(row), + 0.0); + } + } + } + } + + private static Column[] sampleFloating(String mode, long size, double[] values) { + Double[][] rows = new Double[values.length][1]; + for (int i = 0; i < values.length; i++) { + rows[i][0] = values[i]; + } + return sample(mode, size, size, 0, times(values.length), rows, Type.DOUBLE); + } + @Test(timeout = 5000) public void testTimeWindowRejectsUnrepresentableEnd() { assertWindowOverflow(2, 2, 0, Long.MAX_VALUE - 1); @@ -204,6 +279,11 @@ private static Long[][] translate(Long[][] values, long... offsets) { private static Column[] sample( String mode, long size, long slide, long origin, long[] times, Long[][] values) { + return sample(mode, size, slide, origin, times, values, Type.INT64); + } + + private static Column[] sample( + String mode, long size, long slide, long origin, long[] times, Number[][] values, Type type) { int columns = values[0].length; MapTableFunctionHandle handle = new MapTableFunctionHandle.Builder() @@ -211,7 +291,7 @@ private static Column[] sample( .addProperty(LTTBTableFunction.PARTITION_TYPES_PROPERTY, "") .addProperty( LTTBTableFunction.PARTICIPANT_TYPES_PROPERTY, - String.join(",", Collections.nCopies(columns, "INT64"))) + String.join(",", Collections.nCopies(columns, type.name()))) .addProperty(LTTBTableFunction.N_PARAMETER_NAME, size) .addProperty(LTTBTableFunction.SIZE_PARAMETER_NAME, size) .addProperty(LTTBTableFunction.SLIDE_PARAMETER_NAME, slide) @@ -222,7 +302,11 @@ private static Column[] sample( List builders = new ArrayList<>(); int outputColumns = (mode.equals("TIME_WINDOW") ? 2 : 1) + 2 * columns; for (int i = 0; i < outputColumns; i++) { - builders.add(new LongColumnBuilder(null, 16)); + int firstValueColumn = mode.equals("TIME_WINDOW") ? 3 : 2; + builders.add( + type == Type.DOUBLE && i >= firstValueColumn && (i - firstValueColumn) % 2 == 0 + ? new DoubleColumnBuilder(null, 16) + : new LongColumnBuilder(null, 16)); } try { for (int i = 0; i < times.length; i++) { @@ -235,7 +319,7 @@ private static Column[] sample( } } - private static Record record(long time, Long[] values) { + private static Record record(long time, Number[] values) { return (Record) Proxy.newProxyInstance( Record.class.getClassLoader(), @@ -244,7 +328,9 @@ private static Record record(long time, Long[] values) { int index = (int) arguments[0]; switch (method.getName()) { case "getLong": - return index == 0 ? time : values[index - 1]; + return index == 0 ? time : values[index - 1].longValue(); + case "getDouble": + return values[index - 1].doubleValue(); case "isNull": return index != 0 && values[index - 1] == null; default: From 98730f310112c435631cc37621aa3c1321ada683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:33:27 +0800 Subject: [PATCH 06/21] fix(lttb): reject duplicate timestamps with 701 and keep non-finite values Address the review findings on the LTTB table function: - Duplicate timestamps within one input partition are now rejected with SEMANTIC_ERROR (701) instead of silently selecting one of the rows. The check runs at the partition input boundary, before per-column NULL filtering and before window expansion, and carries the last seen timestamp across input blocks. Equal timestamps in different partitions stay valid, and a row shared by overlapping windows is still a single input row. - Non-NULL participant values are no longer dropped when they are NaN or +/-Infinity. The definition only excludes NULL participants, so discarding non-finite values silently turned populated input into an all-NULL result for short series. Add unit coverage for the uniqueness checker (including the block boundary) and for keeping non-finite points, plus integration coverage for all three invocations (N, SIZE count window, SIZE time window), the different-partition case, and the non-finite short series. Co-Authored-By: Claude Opus 5 (1M context) --- .../it/db/it/IoTDBLTTBTableFunctionIT.java | 53 ++++++++++++++ .../iotdb/commons/i18n/CommonMessages.java | 4 ++ .../iotdb/commons/i18n/CommonMessages.java | 3 + .../relational/tvf/LTTBTableFunction.java | 70 +++++++++++++++---- .../relational/tvf/LTTBTableFunctionTest.java | 49 +++++++++++++ 5 files changed, 166 insertions(+), 13 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java index 63ab046927233..f4d5046e89398 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java @@ -115,6 +115,59 @@ public void testCountWindowAndTimeWindowOutput() { DATABASE_NAME); } + @Test + public void testDuplicateTimestampsAreRejectedWith701() { + // one input partition holding (1, 1.0), (1, 9.0), (2, 2.0), (3, 3.0) + String duplicated = + "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1' " + + "UNION ALL SELECT 1, 'd1', 9.0) " + + "PARTITION BY device ORDER BY time, TIMECOL => 'time', "; + String expected = + "701: LTTB does not support duplicate timestamps, but timestamp 1 appears more than once" + + " in the same partition."; + + tableAssertTestFail(duplicated + "N => 3)", expected, DATABASE_NAME); + tableAssertTestFail(duplicated + "SIZE => 2)", expected, DATABASE_NAME); + tableAssertTestFail( + duplicated + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.000+00:00)", + expected, + DATABASE_NAME); + } + + @Test + public void testDuplicateTimestampsInDifferentPartitionsRemainValid() { + // each partition is unique on its own, so the query must succeed and keep both rows + tableResultSetEqualTest( + "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics " + + "WHERE time = 0 AND (device = 'd1' OR device = 'd2')) " + + "PARTITION BY device ORDER BY time, TIMECOL => 'time', N => 3) " + + "ORDER BY device", + new String[] {"window_index", "device", "s1_time", "s1"}, + new String[] { + "0,d1,1970-01-01T00:00:00.000Z,0.0,", "0,d2,null,null,", + }, + DATABASE_NAME); + } + + @Test + public void testNonFiniteValuesAreKeptInShortSeries() { + // three non-NULL input points whose values are NaN/Infinity for N => 3 + String nonFinite = + "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 / 0.0 AS s1 FROM metrics " + + "WHERE device = 'd1' AND time <= 2) " + + "PARTITION BY device ORDER BY time, TIMECOL => 'time', "; + + tableResultSetEqualTest( + nonFinite + "N => 3) ORDER BY s1_time", + new String[] {"window_index", "device", "s1_time", "s1"}, + new String[] { + "0,d1,1970-01-01T00:00:00.000Z,NaN,", + "0,d1,1970-01-01T00:00:00.001Z,Infinity,", + "0,d1,1970-01-01T00:00:00.002Z,Infinity,", + }, + DATABASE_NAME); + } + @Test public void testInvalidParameterCombinations() { String prefix = diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java index 5200e2da0481f..993578c3f255b 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -359,6 +359,10 @@ private CommonMessages() {} public static final String EXCEPTION_LTTB_BUFFERS_AT_MOST_ARG_ELIGIBLE_POINTS_PER_COLUMN_IN_ONE_PARTITION_OR_WINDOW_4C746D82 = "LTTB buffers at most %d eligible points per column in one partition or window."; + public static final String + EXCEPTION_LTTB_DOES_NOT_SUPPORT_DUPLICATE_TIMESTAMPS_BUT_TIMESTAMP_ARG_APPEARS_MORE_THAN_ONCE_IN_THE_SAME_PARTITION_7713FEDD = + "LTTB does not support duplicate timestamps, but timestamp %s appears more than once in" + + " the same partition."; public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = "Unsupported LTTB value type: %s"; public static final String diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java index 9b44c9b590cc0..9fc0a39e0f215 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -255,6 +255,9 @@ private CommonMessages() {} public static final String EXCEPTION_LTTB_BUFFERS_AT_MOST_ARG_ELIGIBLE_POINTS_PER_COLUMN_IN_ONE_PARTITION_OR_WINDOW_4C746D82 = "LTTB 在单个分区或窗口内每列最多缓存 %d 个有效点。"; + public static final String + EXCEPTION_LTTB_DOES_NOT_SUPPORT_DUPLICATE_TIMESTAMPS_BUT_TIMESTAMP_ARG_APPEARS_MORE_THAN_ONCE_IN_THE_SAME_PARTITION_7713FEDD = + "LTTB 不支持重复的时间戳,但时间戳 %s 在同一分区中出现了多次。"; public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = "不支持的 LTTB 值类型:%s"; public static final String diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index 0a161fc236128..d8d2187ca9360 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -74,8 +74,17 @@ * window). * * - *

{@code NULL} and non-finite participant values are ignored per column. Ties in the triangle - * area are broken deterministically by the earliest timestamp (stable input order). + *

{@code NULL} participant values are ignored per column; every non-null value, including {@code + * NaN} and the infinities, participates in the selection and is reproduced in the output. Ties in + * the triangle area are broken deterministically by the earliest timestamp (stable input order). + * + *

The timestamps of the input rows of one partition must be unique. As soon as two rows of the + * same partition carry the same timestamp the query fails with a semantic error ({@code 701}); the + * check runs at the partition input boundary, before per-column {@code NULL} filtering and before + * window expansion, so a row that several overlapping windows share is still a single input row. + * Equal timestamps in different partitions remain valid. Since the input is ordered by {@code + * TIMECOL}, equal timestamps are adjacent, and the check compares each row with the previous one + * and carries the last seen timestamp across input blocks. */ public class LTTBTableFunction implements TableFunction { @@ -425,7 +434,12 @@ private boolean isIntegral() { return type == Type.INT32 || type == Type.INT64; } - /** Appends the record's value to {@code buffer} if it is a non-null finite number. */ + /** + * Appends the record's value to {@code buffer} if it is non-null. All non-null values are + * eligible, including {@code NaN} and the infinities: the definition only excludes {@code NULL} + * participants, and dropping non-finite values would silently turn populated input into an + * all-{@code NULL} result. + */ private void collect(Record record, long time, PointBuffer buffer) { if (record.isNull(inputIndex)) { return; @@ -438,16 +452,10 @@ private void collect(Record record, long time, PointBuffer buffer) { buffer.addIntegral(time, record.getLong(inputIndex)); break; case FLOAT: - float floatValue = record.getFloat(inputIndex); - if (Float.isFinite(floatValue)) { - buffer.addFloating(time, floatValue); - } + buffer.addFloating(time, record.getFloat(inputIndex)); break; case DOUBLE: - double doubleValue = record.getDouble(inputIndex); - if (Double.isFinite(doubleValue)) { - buffer.addFloating(time, doubleValue); - } + buffer.addFloating(time, record.getDouble(inputIndex)); break; default: throw new IllegalArgumentException( @@ -567,10 +575,33 @@ long time(int index) { // Processors // --------------------------------------------------------------------------------------------- + /** + * Partition-local timestamp uniqueness of the input rows. The input is ordered by {@code + * TIMECOL}, so a duplicate timestamp is always adjacent to its predecessor; the last accepted + * timestamp is kept in the instance so the rule also spans input blocks of the same partition. + */ + static final class TimestampUniquenessChecker { + private long previousTimestamp; + private boolean hasPreviousTimestamp; + + void check(long time) { + if (hasPreviousTimestamp && time == previousTimestamp) { + throw new SemanticException( + String.format( + CommonMessages + .EXCEPTION_LTTB_DOES_NOT_SUPPORT_DUPLICATE_TIMESTAMPS_BUT_TIMESTAMP_ARG_APPEARS_MORE_THAN_ONCE_IN_THE_SAME_PARTITION_7713FEDD, + time)); + } + previousTimestamp = time; + hasPreviousTimestamp = true; + } + } + private abstract static class AbstractLTTBDataProcessor implements TableFunctionDataProcessor { protected final Type[] partitionTypes; protected final Object[] partitionValues; protected final ParticipantColumn[] participantColumns; + private final TimestampUniquenessChecker timestampChecker = new TimestampUniquenessChecker(); protected AbstractLTTBDataProcessor( Type[] partitionTypes, ParticipantColumn[] participantColumns) { @@ -579,6 +610,15 @@ protected AbstractLTTBDataProcessor( this.participantColumns = participantColumns; } + /** + * Rejects a repeated input timestamp of the partition. Runs at the partition input boundary, + * before per-column {@code NULL} filtering and before window expansion, so a row shared by + * several overlapping windows is still a single input row. + */ + protected final void checkTimestamp(long time) { + timestampChecker.check(time); + } + protected final PointBuffer[] newBuffers() { PointBuffer[] buffers = new PointBuffer[participantColumns.length]; for (int i = 0; i < buffers.length; i++) { @@ -650,8 +690,10 @@ public void process( List properColumnBuilders, ColumnBuilder passThroughIndexBuilder) { hasRows = true; + long time = input.getLong(0); + checkTimestamp(time); capturePartitionValues(input); - collect(input, input.getLong(0), buffers); + collect(input, time, buffers); } @Override @@ -764,8 +806,10 @@ public final void process( Record input, List properColumnBuilders, ColumnBuilder passThroughIndexBuilder) { + long time = input.getLong(0); + checkTimestamp(time); capturePartitionValues(input); - processRecord(input, input.getLong(0), properColumnBuilders); + processRecord(input, time, properColumnBuilders); } protected abstract void processRecord( diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java index fcd05018e5e4f..80e8871e2dca1 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java @@ -113,6 +113,55 @@ public void testAnalyzeRejectsNAndSizeTogether() throws UDFException { } } + @Test + public void testTargetCountKeepsNonFiniteValues() { + long[] times = {0, 1, 2}; + double[] values = {Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}; + + // three eligible points with N => 3: every point must be returned, not discarded + assertArrayEquals( + new int[] {0, 1, 2}, LTTBTableFunction.selectTargetCount(times, values, 3, 3)); + } + + @Test + public void testTimestampUniquenessAcceptsDistinctAndRejectsRepeatedTimestamps() { + LTTBTableFunction.TimestampUniquenessChecker checker = + new LTTBTableFunction.TimestampUniquenessChecker(); + checker.check(1L); + checker.check(2L); + checker.check(3L); + try { + checker.check(3L); + fail(); + } catch (SemanticException e) { + assertEquals( + String.format( + CommonMessages + .EXCEPTION_LTTB_DOES_NOT_SUPPORT_DUPLICATE_TIMESTAMPS_BUT_TIMESTAMP_ARG_APPEARS_MORE_THAN_ONCE_IN_THE_SAME_PARTITION_7713FEDD, + 3L), + e.getMessage()); + } + } + + /** The duplicate check must survive the block boundary: the state outlives one block's rows. */ + @Test + public void testTimestampUniquenessSpansInputBlocks() { + LTTBTableFunction.TimestampUniquenessChecker checker = + new LTTBTableFunction.TimestampUniquenessChecker(); + checker.check(7L); // last row of the first input block + try { + checker.check(7L); // first row of the second input block + fail(); + } catch (SemanticException e) { + assertEquals( + String.format( + CommonMessages + .EXCEPTION_LTTB_DOES_NOT_SUPPORT_DUPLICATE_TIMESTAMPS_BUT_TIMESTAMP_ARG_APPEARS_MORE_THAN_ONCE_IN_THE_SAME_PARTITION_7713FEDD, + 7L), + e.getMessage()); + } + } + private static Map targetCountArguments(long n) { Map arguments = new HashMap<>(); arguments.put( From cb528339c7dddbd33a8a61742072f030d4d267fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:34:18 +0800 Subject: [PATCH 07/21] test(lttb): source duplicate timestamps from a real table in the 701 IT The 701 integration test built its duplicated partition with a UNION ALL whose second branch was a literal row. That plan carries a ValuesNode with inlined rows, and dispatching such a fragment over the cluster fails in ValuesNode deserialization with BufferUnderflowException, so the test reported 301 instead of 701 under TableClusterIT. Both branches now scan the real metrics table, matching the UNION ALL shape already used by other table-model ITs. The partition still repeats every d1 row, so the duplicate-timestamp check is exercised identically, and the expected timestamp is now 0. Verified with TableSimpleIT and TableClusterIT: 8 tests pass in both. Co-Authored-By: Claude Opus 5 (1M context) --- .../iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java index 95b9b78176676..ece266187316b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java @@ -135,13 +135,13 @@ public void testCountWindowAndTimeWindowOutput() { @Test public void testDuplicateTimestampsAreRejectedWith701() { - // one input partition holding (1, 1.0), (1, 9.0), (2, 2.0), (3, 3.0) + // the same six d1 rows twice: (0, 1, 2, 3, 4, 5) each occurs twice in one partition String duplicated = "SELECT * FROM LTTB(DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1' " - + "UNION ALL SELECT 1, 'd1', 9.0) " + + "UNION ALL SELECT time, device, s1 FROM metrics WHERE device = 'd1') " + "PARTITION BY device ORDER BY time, TIMECOL => 'time', "; String expected = - "701: LTTB does not support duplicate timestamps, but timestamp 1 appears more than once" + "701: LTTB does not support duplicate timestamps, but timestamp 0 appears more than once" + " in the same partition."; tableAssertTestFail(duplicated + "N => 3)", expected, DATABASE_NAME); From 5cdb3362d05a50edd1661b72536a3967acc01c39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:40:39 +0800 Subject: [PATCH 08/21] fix(lttb): address review findings on the LTTB table function - Guard the exact-area fallback against non-finite participants: a NaN or Infinity candidate is scored as the smallest area (earliest point wins) instead of reaching `new BigDecimal(Double.NaN)`, which threw an unchecked NumberFormatException whenever N < M and a bucket held such a value. The fallback still runs for a non-finite lookahead average over representable points, since it rebuilds the sum from the original points. - Replace the compile-time MAX_BUFFER_BYTES with the hot-reloadable `lttb_buffer_size_in_bytes` parameter, resolved through loadFixedSizeLimitForQuery so the default scales with the node's operator memory (as sort_buffer_size_in_bytes does) rather than charging a fixed 64 MB to every LTTB fragment, which tripped the fragment admission check on small-memory nodes. - Report the LTTB buffers to the engine: a new TableFunctionProcessorProvider#getEstimatedMaxBufferedBytes hook feeds TableFunctionOperator's peek and retained estimates, matching how AbstractSortOperator accounts for its sort buffer. - Drop LTTB from the planner's name-based split exclusion. The partitioned GroupNode the function requires already guarantees whole partitions per fragment, so fragment-local samples are valid standard LTTB results. - Give the count-window INT64 overflow constant its hash suffix. Add unit coverage for non-finite sampling (N < M, all-non-finite bucket) and integration coverage for the new configuration parameter. --- .../db/it/IoTDBLoadConfigurationTableIT.java | 8 +- .../table/TableFunctionProcessorProvider.java | 12 ++ .../function/TableFunctionOperator.java | 11 +- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 15 +++ .../TableDistributedPlanGenerator.java | 9 +- .../conf/iotdb-system.properties.template | 10 ++ .../iotdb/commons/i18n/CommonMessages.java | 5 +- .../iotdb/commons/i18n/CommonMessages.java | 5 +- .../iotdb/commons/conf/CommonConfig.java | 11 ++ .../relational/tvf/LTTBTableFunction.java | 121 +++++++++++++++--- .../tvf/LTTBNumericBoundaryTest.java | 2 +- .../relational/tvf/LTTBRegressionTest.java | 2 +- .../relational/tvf/LTTBTableFunctionTest.java | 40 ++++++ .../relational/tvf/LTTBWindowMemoryTest.java | 6 +- 14 files changed, 223 insertions(+), 34 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java index f573426334087..0c3f413938b4b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java @@ -119,13 +119,19 @@ public void showConfigurationDisplaysEffectiveValue() throws Exception { // Case 2: a valid value is not rewritten, so display must equal the file value (regression). Map afterValidReload = - appendLinesLoadAndShow(dataNodeId, confPath, "cte_buffer_size_in_bytes=262144"); + appendLinesLoadAndShow( + dataNodeId, + confPath, + "cte_buffer_size_in_bytes=262144", + "lttb_buffer_size_in_bytes=262144"); Assert.assertEquals("262144", afterValidReload.get("cte_buffer_size_in_bytes")); + Assert.assertEquals("262144", afterValidReload.get("lttb_buffer_size_in_bytes")); // Case 3: params whose template default is 0 are always rewritten to a computed default, // so they must never display 0 (this was always wrong, even without a bad file value). Map defaultsReload = appendLinesLoadAndShow(dataNodeId, confPath); assertPositiveNonZero(defaultsReload, "sort_buffer_size_in_bytes"); + assertPositiveNonZero(defaultsReload, "lttb_buffer_size_in_bytes"); assertPositiveNonZero(defaultsReload, "mods_cache_size_limit_per_fi_in_bytes"); } diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionProcessorProvider.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionProcessorProvider.java index f86d631f5989a..9ca917070d7e5 100644 --- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionProcessorProvider.java +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionProcessorProvider.java @@ -43,4 +43,16 @@ default TableFunctionLeafProcessor getSplitProcessor() { throw new UnsupportedOperationException( UdfApiMessages.TABLE_FUNCTION_DOES_NOT_PROCESS_LEAF_DATA); } + + /** + * The bytes the processors of this provider may hold in their own buffers while a partition is + * being processed, above what {@code process} returns. The engine adds this to the estimated + * memory of the {@linkplain TableFunctionDataProcessor} operator, so a table function that + * accumulates input (rather than emitting a result per record) is visible to memory management + * and can be back-pressured. The default is {@code 0} for table functions that keep no state + * across calls. + */ + default long getEstimatedMaxBufferedBytes() { + return 0L; + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java index fb9b90c9f604a..d8508a32e2d30 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java @@ -72,6 +72,9 @@ public class TableFunctionOperator extends AbstractOperator implements ProcessOp private final boolean isDeclaredAsPassThrough; private final IoTDBLocal ioTDBLocal; + /** Bytes the data processors may buffer internally while a partition is processed. */ + private final long estimatedMaxBufferedBytes; + private TableFunctionDataProcessor processor; private PartitionState partitionState; private ListenableFuture isBlocked; @@ -101,6 +104,7 @@ public TableFunctionOperator( new PartitionRecognizer( partitionChannels, requiredChannels, passThroughChannels, inputDataTypes); this.isDeclaredAsPassThrough = isDeclaredAsPassThrough; + this.estimatedMaxBufferedBytes = processorProvider.getEstimatedMaxBufferedBytes(); this.needPassThrough = properChannelCount != outputDataTypes.size(); this.partitionState = null; this.properBlockBuilder = new TsBlockBuilder(outputDataTypes.subList(0, properChannelCount)); @@ -291,7 +295,10 @@ public boolean isFinished() throws Exception { @Override public long calculateMaxPeekMemory() { + // A processor that buffers its input (LTTB) holds up to estimatedMaxBufferedBytes while the + // partition is processed, in addition to the input it reads and the block it builds. return inputOperator.calculateMaxPeekMemory() + + estimatedMaxBufferedBytes + Math.max(maxReturnSize, properBlockBuilder.getRetainedSizeInBytes()); } @@ -302,7 +309,9 @@ public long calculateMaxReturnSize() { @Override public long calculateRetainedSizeAfterCallingNext() { - return inputOperator.calculateRetainedSizeAfterCallingNext(); + // The buffers survive across next() calls until the partition completes, so they are retained + // here rather than charged to the peek of a single call. + return inputOperator.calculateRetainedSizeAfterCallingNext() + estimatedMaxBufferedBytes; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 44971c718a104..cc8ab3714517d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1100,6 +1100,13 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException commonConfig.setCteBufferSize(cteBufferSizeInBytes); } + // The per-processor buffer budget of the LTTB table function. Like sort_buffer_size_in_bytes + // this is charged to the estimated memory of every fragment that runs the function, so the + // non-positive default resolves to a value derived from the operator memory of this node + // instead of one that would reject the query outright on a small-memory node. + loadFixedSizeLimitForQuery( + properties, "lttb_buffer_size_in_bytes", commonConfig::setLttbBufferSize); + // max number of rows for cte materialization int maxRowsInCteBuffer = Integer.parseInt( @@ -2392,6 +2399,12 @@ public synchronized void loadHotModifiedProps(TrimProperties properties) if (cteBufferSizeInBytes > 0) { commonConfig.setCteBufferSize(cteBufferSizeInBytes); } + // The per-processor buffer budget of the LTTB table function. Like sort_buffer_size_in_bytes + // this is charged to the estimated memory of every fragment that runs the function, so the + // non-positive default resolves to a value derived from the operator memory of this node + // instead of one that would reject the query outright on a small-memory node. + loadFixedSizeLimitForQuery( + properties, "lttb_buffer_size_in_bytes", commonConfig::setLttbBufferSize); // max number of rows for cte materialization int maxRowsInCteBuffer = Integer.parseInt( @@ -2439,6 +2452,8 @@ public synchronized void loadHotModifiedProps(TrimProperties properties) private void overlayEffectiveConfigurationValues() { ConfigurationFileUtils.updateAppliedProperties( "cte_buffer_size_in_bytes", Long.toString(commonConfig.getCteBufferSize())); + ConfigurationFileUtils.updateAppliedProperties( + "lttb_buffer_size_in_bytes", Long.toString(commonConfig.getLttbBufferSize())); ConfigurationFileUtils.updateAppliedProperties( "max_rows_in_cte_buffer", Integer.toString(commonConfig.getMaxRowsInCteBuffer())); ConfigurationFileUtils.updateAppliedProperties( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index a67c1c8f01c30..06b97fa7b7d69 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -2899,10 +2899,11 @@ private boolean canSplitTableFunctionProcessor(TableFunctionProcessorNode node) if (!isPartitionedGroup(node.getChild())) { return false; } - // LTTB is not mergeable: fragment-local samples cannot be combined into a globally correct - // result, so the whole partition must be gathered and ordered before the function runs. - return !TableBuiltinTableFunction.FFT.getFunctionName().equalsIgnoreCase(node.getName()) - && !TableBuiltinTableFunction.LTTB.getFunctionName().equalsIgnoreCase(node.getName()); + // LTTB needs whole partitions, which the partitioned GroupNode above this node already + // guarantees: every fragment receives complete partitions and the ordering is preserved, so a + // fragment-local sample is a valid standard LTTB result for that partition and no exclusion is + // needed. FFT is excluded because its transform over the whole partition is not splittable. + return !TableBuiltinTableFunction.FFT.getFunctionName().equalsIgnoreCase(node.getName()); } private boolean isPartitionedGroup(PlanNode node) { diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 866dde0a94e5b..783d23b7f6987 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1280,6 +1280,16 @@ sort_buffer_size_in_bytes=0 # Unit: bytes cte_buffer_size_in_bytes=131072 +# The per-processor buffer budget of the LTTB table function. It is charged to the estimated memory of every fragment that +# runs LTTB, so if lttb_buffer_size_in_bytes <= 0 a default value is used, default value = min(32MB, memory for query operators / +# query_thread_count / 2), matching sort_buffer_size_in_bytes. Raise it for queries with very large partitions or many overlapping +# windows. +# effectiveMode: hot_reload +# Datatype: long +# Privilege: SYSTEM +# Unit: bytes +lttb_buffer_size_in_bytes=0 + # Max rows for CTE materialization # effectiveMode: hot_reload diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java index b091beccfa062..1014c2a686572 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -365,8 +365,9 @@ private CommonMessages() {} + " the same partition."; public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = "Unsupported LTTB value type: %s"; - public static final String EXCEPTION_LTTB_COUNT_WINDOW_BOUNDARIES_EXCEED_THE_INT64_RANGE = - "LTTB count window boundaries exceed the INT64 range."; + public static final String + EXCEPTION_LTTB_COUNT_WINDOW_BOUNDARIES_EXCEED_THE_INT64_RANGE_5BC980CA = + "LTTB count window boundaries exceed the INT64 range."; public static final String EXCEPTION_LTTB_WINDOW_BOUNDARIES_EXCEED_THE_TIMESTAMP_RANGE_7C0FC7E2 = "LTTB window boundaries exceed the TIMESTAMP range."; public static final String diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java index 9561a4a6dd1c2..a32d4c1df13cc 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -260,8 +260,9 @@ private CommonMessages() {} "LTTB 不支持重复的时间戳,但时间戳 %s 在同一分区中出现了多次。"; public static final String EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE = "不支持的 LTTB 值类型:%s"; - public static final String EXCEPTION_LTTB_COUNT_WINDOW_BOUNDARIES_EXCEED_THE_INT64_RANGE = - "LTTB 计数窗口边界超出 INT64 的取值范围。"; + public static final String + EXCEPTION_LTTB_COUNT_WINDOW_BOUNDARIES_EXCEED_THE_INT64_RANGE_5BC980CA = + "LTTB 计数窗口边界超出 INT64 的取值范围。"; public static final String EXCEPTION_LTTB_WINDOW_BOUNDARIES_EXCEED_THE_TIMESTAMP_RANGE_7C0FC7E2 = "LTTB 窗口边界超出 TIMESTAMP 的取值范围。"; public static final String diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index cb5902986b8e8..653e616fc2f46 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -569,6 +569,9 @@ public class CommonConfig { /** Max number of rows for cte materialization */ private int maxRowsInCteBuffer = 1000; + /** The per-processor memory budget of the LTTB table function */ + private long lttbBufferSize = 64 * 1024 * 1024L; + CommonConfig() { // Empty constructor } @@ -3325,4 +3328,12 @@ public void setMaxRowsInCteBuffer(int maxRowsInCteBuffer) { public int getMaxRowsInCteBuffer() { return maxRowsInCteBuffer; } + + public void setLttbBufferSize(long lttbBufferSize) { + this.lttbBufferSize = lttbBufferSize; + } + + public long getLttbBufferSize() { + return lttbBufferSize; + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index 4b3ebd82c3127..ba9402a34f377 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -19,6 +19,7 @@ package org.apache.iotdb.commons.udf.builtin.relational.tvf; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.i18n.CommonMessages; import org.apache.iotdb.udf.api.exception.UDFException; @@ -108,8 +109,13 @@ public class LTTBTableFunction implements TableFunction { /** Hard cap on eligible points buffered per participant column in one partition or window. */ public static final int MAX_BUFFERED_POINTS = 1 << 22; - /** Shared budget for point buffers and selection state across all columns of one processor. */ - static final long MAX_BUFFER_BYTES = 64L * 1024 * 1024; + /** + * Shared budget for point buffers and selection state across all columns of one processor. It is + * the default of {@code lttb_buffer_size_in_bytes}; operators can tune the live value with that + * hot-reloadable configuration parameter for workloads with large partitions or many overlapping + * windows. + */ + public static final long DEFAULT_MAX_BUFFER_BYTES = 64L * 1024 * 1024; public static final String MODE_PROPERTY = "__LTTB_MODE"; public static final String PARTITION_TYPES_PROPERTY = "__LTTB_PARTITION_TYPES"; @@ -129,6 +135,18 @@ public enum Mode { COUNT_WINDOW } + /** + * The live per-processor buffer budget, read from the hot-reloadable {@code + * lttb_buffer_size_in_bytes} configuration parameter. Values that are not positive fall back to + * {@link #DEFAULT_MAX_BUFFER_BYTES} so a misconfigured node still bounds memory instead of + * buffering without limit. Processors read this once at construction, so a reload applies to + * queries that start afterwards. + */ + public static long maxBufferBytes() { + long configured = CommonDescriptor.getInstance().getConfig().getLttbBufferSize(); + return configured > 0 ? configured : DEFAULT_MAX_BUFFER_BYTES; + } + @Override public List getArgumentsSpecifications() { return Arrays.asList( @@ -316,6 +334,16 @@ public TableFunctionDataProcessor getDataProcessor() { throw new IllegalStateException(String.valueOf(mode)); } } + + /** + * Every LTTB processor buffers the points of the partition (or of the windows it keeps open) + * until it can select the output points, so the whole per-processor budget is reportable as + * memory the operator holds while the partition is processed. + */ + @Override + public long getEstimatedMaxBufferedBytes() { + return maxBufferBytes(); + } }; } @@ -426,12 +454,36 @@ private static int[] selectTargetCount( } /** - * Returns the index in {@code [start, end)} maximizing the triangle area; earliest wins ties. - * {@code nextTime} and {@code nextValue} are relative to the anchor. Integral differences are - * computed before converting to double so large epochs and INT64 baselines retain small changes. - * Returns {@link #NOT_SELECTED} if intermediate arithmetic overflows, so the caller can compare - * the entire bucket using its original points and exact arithmetic. + * Area of the triangle formed by the candidate {@code index}, the anchor and the lookahead point, + * or {@code -1} when no finite area exists. Areas are non-negative, so {@code -1} unambiguously + * marks a candidate that cannot be scored. + * + *

A {@code NaN} or infinite participant value yields a non-finite (undefined) area rather than + * an overflow: no arithmetic can make it finite. Such a candidate never beats a finite one, so a + * bucket whose candidates are all non-finite keeps its earliest point. */ + private static double boundedArea( + long[] times, + double[] values, + long[] integralValues, + int index, + long anchorTime, + double anchorValue, + long integralAnchor, + double nextTime, + double nextValue) { + double area = + Math.abs( + nextTime * relativeValue(values, integralValues, index, anchorValue, integralAnchor) + - differenceAsDouble(times[index], anchorTime) * nextValue); + return Double.isFinite(area) ? area : -1; + } + + /** Whether the participant value at {@code index} is representable as an exact decimal. */ + private static boolean hasExactValue(double[] values, long[] integralValues, int index) { + return integralValues != null || Double.isFinite(values[index]); + } + private static int argMaxArea( long[] times, double[] values, @@ -443,15 +495,34 @@ private static int argMaxArea( long integralAnchor, double nextTime, double nextValue) { + // The exact fallback rebuilds the lookahead sum from the original points, so a non-finite + // average over representable points is still recoverable; only the anchor origin has to be + // representable, because it is converted to BigDecimal directly. A non-finite anchor value can + // only arise from non-finite input, which no arithmetic can repair. + boolean exactIsPossible = Double.isFinite(anchorValue); int pick = start; double maxArea = -1; for (int i = start; i < end; i++) { double area = - Math.abs( - nextTime * relativeValue(values, integralValues, i, anchorValue, integralAnchor) - - differenceAsDouble(times[i], anchorTime) * nextValue); - if (!Double.isFinite(area)) { - return NOT_SELECTED; + boundedArea( + times, + values, + integralValues, + i, + anchorTime, + anchorValue, + integralAnchor, + nextTime, + nextValue); + if (area < 0) { + if (!hasExactValue(values, integralValues, i)) { + // Non-finite participant: boundedArea scored it as the smallest area. + continue; + } + if (exactIsPossible) { + // Overflow on representable input: let the caller compare the whole bucket exactly. + return NOT_SELECTED; + } } if (area > maxArea) { maxArea = area; @@ -462,10 +533,15 @@ private static int argMaxArea( } /** - * Overflow fallback for finite input values. Recomputes every candidate so an overflowing area - * cannot tie with or hide a larger one. Multiplying all areas by the lookahead point count lets - * us use sums instead of dividing by that count, keeping the comparison exact even for recurring - * averages. The normal path does not allocate decimal values. + * Overflow fallback for input values representable as exact decimals. Recomputes every candidate + * so an overflowing area cannot tie with or hide a larger one. Multiplying all areas by the + * lookahead point count lets us use sums instead of dividing by that count, keeping the + * comparison exact even for recurring averages. The normal path does not allocate decimal values. + * + *

A {@code NaN} or infinite participant cannot be represented by {@link BigDecimal}, so such a + * candidate is skipped and ties resolve to the earliest point. The caller only reaches this + * method when the anchor and the lookahead point are finite, so the {@link BigDecimal} + * conversions of those two origins and of the lookahead values are always well defined. */ private static int argMaxAreaExactly( long[] times, @@ -487,12 +563,18 @@ private static int argMaxAreaExactly( BigDecimal sumTime = BigDecimal.ZERO; BigDecimal sumValue = BigDecimal.ZERO; for (int i = nextStart; i < nextEnd; i++) { + if (!hasExactValue(nextValues, nextIntegralValues, i)) { + continue; + } sumTime = sumTime.add(BigDecimal.valueOf(nextTimes[i]).subtract(timeOrigin)); sumValue = sumValue.add(exactValue(nextValues, nextIntegralValues, i).subtract(valueOrigin)); } int pick = start; BigDecimal maxArea = null; for (int i = start; i < end; i++) { + if (!hasExactValue(values, integralValues, i)) { + continue; + } BigDecimal time = BigDecimal.valueOf(times[i]).subtract(timeOrigin); BigDecimal value = exactValue(values, integralValues, i).subtract(valueOrigin); BigDecimal area = sumTime.multiply(value).subtract(time.multiply(sumValue)).abs(); @@ -840,7 +922,7 @@ protected final void writeRow( private static final class TargetCountLTTBDataProcessor extends AbstractLTTBDataProcessor { private final long n; private final PointBuffer[] buffers; - private final BufferMemoryBudget memoryBudget = new BufferMemoryBudget(MAX_BUFFER_BYTES); + private final BufferMemoryBudget memoryBudget = new BufferMemoryBudget(maxBufferBytes()); private boolean hasRows; private TargetCountLTTBDataProcessor( @@ -978,7 +1060,7 @@ private abstract static class AbstractWindowLTTBDataProcessor activeWindows = new ArrayDeque<>(); private final Deque pendingWindows = new ArrayDeque<>(); private final WindowState[] lastNonemptyWindows; - private final BufferMemoryBudget memoryBudget = new BufferMemoryBudget(MAX_BUFFER_BYTES); + private final BufferMemoryBudget memoryBudget = new BufferMemoryBudget(maxBufferBytes()); private final long windowMetadataBytes; private final boolean[] hasAnchor; private final long[] anchorTimes; @@ -1271,7 +1353,8 @@ protected void processRecord( windowEnd = Math.addExact(nextWindowStart, size); } catch (ArithmeticException e) { throw new SemanticException( - CommonMessages.EXCEPTION_LTTB_COUNT_WINDOW_BOUNDARIES_EXCEED_THE_INT64_RANGE); + CommonMessages + .EXCEPTION_LTTB_COUNT_WINDOW_BOUNDARIES_EXCEED_THE_INT64_RANGE_5BC980CA); } activeWindows.addLast( new CountWindowState(newWindowBuffers(), windowEnd, nextWindowIndex++)); diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java index e306c4f38deaf..3fa9cc10264f6 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java @@ -143,7 +143,7 @@ public void testCountWindowRejectsUnrepresentableEnd() { () -> sample("COUNT_WINDOW", Long.MAX_VALUE, 1, 0, times(2), new Long[][] {{1L}, {2L}})); assertEquals( - CommonMessages.EXCEPTION_LTTB_COUNT_WINDOW_BOUNDARIES_EXCEED_THE_INT64_RANGE, + CommonMessages.EXCEPTION_LTTB_COUNT_WINDOW_BOUNDARIES_EXCEED_THE_INT64_RANGE_5BC980CA, failure.getMessage()); } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java index 4bb4158df1a8a..63d821dd89c3f 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java @@ -137,7 +137,7 @@ public void testTargetColumnsShareMemoryBudget() { String.format( CommonMessages .EXCEPTION_LTTB_BUFFERS_EXCEED_THE_PER_PROCESSOR_MEMORY_LIMIT_OF_ARG_BYTES_954FEF00, - LTTBTableFunction.MAX_BUFFER_BYTES), + LTTBTableFunction.DEFAULT_MAX_BUFFER_BYTES), expected.getMessage()); } finally { processor.beforeDestroy(); diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java index 80e8871e2dca1..a19ef63097890 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java @@ -38,6 +38,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class LTTBTableFunctionTest { @@ -123,6 +124,45 @@ public void testTargetCountKeepsNonFiniteValues() { new int[] {0, 1, 2}, LTTBTableFunction.selectTargetCount(times, values, 3, 3)); } + /** + * With {@code N < M} the algorithm computes triangle areas, so a non-finite participant reaches + * the area comparison. NaN and the infinities cannot form a finite area and {@link + * java.math.BigDecimal} cannot represent them, so such candidates must be skipped (earliest wins) + * instead of aborting the query with an unchecked {@code NumberFormatException}. + */ + @Test + public void testTargetCountSkipsNonFiniteCandidatesWhenSampling() { + long[] times = {0, 1, 2, 3, 4, 5}; + double[] values = {1.0, Double.NaN, Double.POSITIVE_INFINITY, 2.0, 3.0, 4.0}; + + int[] selected = LTTBTableFunction.selectTargetCount(times, values, 6, 3); + + // The first and last eligible points are always preserved. + assertEquals(0, selected[0]); + assertEquals(5, selected[selected.length - 1]); + assertEquals(3, selected.length); + for (int index : selected) { + assertTrue("selected index out of range: " + index, index >= 0 && index < 6); + } + } + + /** A bucket whose candidates are all non-finite keeps its earliest point, deterministically. */ + @Test + public void testTargetCountKeepsEarliestPointWhenAllCandidatesAreNonFinite() { + long[] times = {0, 1, 2, 3, 4, 5}; + double[] values = { + Double.NaN, + Double.NaN, + Double.POSITIVE_INFINITY, + Double.NaN, + Double.NEGATIVE_INFINITY, + Double.NaN + }; + + assertArrayEquals( + new int[] {0, 1, 5}, LTTBTableFunction.selectTargetCount(times, values, 6, 3)); + } + @Test public void testTimestampUniquenessAcceptsDistinctAndRejectsRepeatedTimestamps() { LTTBTableFunction.TimestampUniquenessChecker checker = diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java index c2f44808dd573..4bd6437856578 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java @@ -56,7 +56,7 @@ public void testCountWindowOverlapFailsBeforeExhaustingHeap() { processor.process(record(i, 1.0), Collections.emptyList(), null); } }, - LTTBTableFunction.MAX_BUFFER_BYTES); + LTTBTableFunction.DEFAULT_MAX_BUFFER_BYTES); } finally { processor.beforeDestroy(); } @@ -70,7 +70,7 @@ public void testTimeWindowCreationIsBoundedEvenForNullInput() { // must be charged even when the input column contains no eligible points. assertMemoryLimit( () -> processor.process(record(0, (Number) null), Collections.emptyList(), null), - LTTBTableFunction.MAX_BUFFER_BYTES); + LTTBTableFunction.DEFAULT_MAX_BUFFER_BYTES); } finally { processor.beforeDestroy(); } @@ -87,7 +87,7 @@ public void testColumnsShareTheWindowBudget() { processor.process(record(i, 1.0, Long.MAX_VALUE), Collections.emptyList(), null); } }, - LTTBTableFunction.MAX_BUFFER_BYTES); + LTTBTableFunction.DEFAULT_MAX_BUFFER_BYTES); } finally { processor.beforeDestroy(); } From 213393ea6f57407e15d36068f764b589f970546f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:51:38 +0800 Subject: [PATCH 09/21] docs(lttb): state the settled SIZE-mode semantics and cover the boundary rules Document what the clarification fixes and pin it with tests: - SIZE defines a bucket, not the input range of a separate LTTB invocation. One anchor chain per partition and participant column: A is the point the preceding bucket selected, B becomes the anchor of the following bucket, and overlapping buckets (SLIDE < SIZE) do not restart the chain. State stays independent between partitions and between columns. - Bucket mode emits one row per window that carries input, at most one point per column. A column with no eligible point emits NULL, and an all-NULL window still emits a single all-NULL row. - Bucket mode does not preserve the first and last eligible points: an endpoint used as A or as C is a triangle vertex, not a selected point, so it is returned only when it also wins its bucket. A bucket with two eligible points contributes one row, the tie resolving to the earlier point. First/last preservation and return-all-points remain target-count behaviour. Tests: the anchor chain across overlapping buckets (with the count-window and time-window expected indexes), two eligible points in one bucket emitting one row, and an all-NULL column still emitting one NULL row per window. The count-window case selects t=1 in bucket [1,4), which is only reachable with a chained anchor. --- .../relational/tvf/LTTBTableFunction.java | 23 ++++-- .../relational/tvf/LTTBRegressionTest.java | 70 +++++++++++++++++++ 2 files changed, 88 insertions(+), 5 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index ba9402a34f377..68793615c82b9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -70,13 +70,26 @@ * window_index = 0}. *

  • Window/bucket mode ({@code SIZE}, optional {@code SLIDE}/{@code ORIGIN}): windows * are built exactly like M4 (time windows for duration literals, count windows for integers). - * Each window is one LTTB bucket that selects at most one point per participant column: the - * anchor is the previously selected point of that column, the candidates are the eligible - * points in the window, and the third vertex is the average point of the next window that - * carries data for that column (or the window's own last eligible point for the final - * window). + * {@code SIZE} defines a bucket, not the input range of a separate standard LTTB invocation. + * One anchor chain is maintained per partition and participant column: for the current bucket + * {@code A} is the point selected by the preceding bucket of that column ({@code B} in turn + * becomes the anchor of the following bucket), {@code B} ranges over the current bucket's + * eligible points, and {@code C} is the average point of the next bucket that carries data + * for that column (or the bucket's own last eligible point when there is none). Overlapping + * buckets ({@code SLIDE < SIZE}) do not restart the chain, and the state is independent + * between partitions and between participant columns. * * + *

    Bucket mode emits one row per window that carries input, and at most one selected point per + * participant column in a row; a column with no eligible point in a window emits {@code NULL} in + * its place, and a window in which every participant column is {@code NULL} still emits a single + * all-{@code NULL} row (padding by output position). Bucket mode does not preserve the + * first and last eligible points: an endpoint used as {@code A} or as {@code C} is a triangle + * vertex, not a selected point, so it appears in the output only when it also wins its bucket. A + * bucket holding just one or two eligible points therefore still contributes a single row per + * column: with two points the areas are equal and the tie resolves to the earlier one. First/last + * preservation and the return-all-points case apply to target-count mode only. + * *

    {@code NULL} participant values are ignored per column; every non-null value, including {@code * NaN} and the infinities, participates in the selection and is reproduced in the output. Ties in * the triangle area are broken deterministically by the earliest timestamp (stable input order). diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java index 63d821dd89c3f..8468049cb3f64 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java @@ -25,6 +25,7 @@ import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; @@ -120,6 +121,75 @@ public void testTrailingNullWindowsUseFinalNonemptyBucketFallback() { } } + /** + * SIZE defines a bucket, not the input range of a separate standard LTTB invocation: the buckets + * share one anchor chain per partition and column, so an overlapping window continues from the + * point the preceding window selected instead of restarting at its own first point. Bucket {@code + * [1,4)} selecting {@code t=1} is only reachable with a chained anchor: with a fresh anchor at + * its own first point the largest triangle of that bucket would be {@code t=2} instead. Every + * window emits exactly one row, so the overlapping run yields one row per window. + */ + @Test + public void testOverlappingWindowsContinueTheAnchorChain() { + Double[][] rows = {{0.0}, {1.0}, {8.0}, {2.0}, {0.0}, {0.0}}; + + // Count buckets [0,3), [1,4), ..., [5,8) over six rows: one row per bucket. + List countWindow = sample("COUNT_WINDOW", 3, 1, 0, rows); + Column countPicks = countWindow.get(countWindow.size() - 2).build(); + assertEquals(6, countPicks.getPositionCount()); + assertEquals(2, countPicks.getLong(0)); + assertEquals(1, countPicks.getLong(1)); + assertEquals(2, countPicks.getLong(2)); + assertEquals(3, countPicks.getLong(3)); + + // Time buckets [0,3), [1,4), ... with SLIDE = 1 a row also opens the buckets that started + // before the first timestamp but have not ended yet, so eight buckets cover the six rows. The + // bucket [2,5) selecting t=2 and the next one selecting t=1 show the chain: [3,6) alone would + // pick t=3 or t=4, not the point its predecessor selected. + List timeWindow = sample("TIME_WINDOW", 3, 1, 0, rows); + Column timePicks = timeWindow.get(timeWindow.size() - 2).build(); + assertEquals(8, timePicks.getPositionCount()); + assertEquals(0, timePicks.getLong(0)); + assertEquals(2, timePicks.getLong(2)); + assertEquals(1, timePicks.getLong(3)); + } + + /** + * A bucket holds at most one selected point per participant column, so two eligible points in one + * bucket still emit a single row. The endpoints are not preserved: here the anchor and the third + * vertex are the two eligible points themselves, so the area is zero for both candidates and the + * tie resolves to the earliest, leaving the final point unreturned. + */ + @Test + public void testTwoEligiblePointsInOneBucketEmitOneRow() { + Double[][] rows = {{0.0}, {8.0}}; + for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { + List out = sample(mode, 3, 3, 0, rows); + int timeIndex = out.size() - 2; + assertEquals(1, out.get(timeIndex).getPositionCount()); + assertEquals(0, out.get(timeIndex).build().getLong(0)); + assertEquals(0.0, out.get(timeIndex + 1).build().getDouble(0), 0.0); + } + } + + /** + * A window emits one row per partition even when no participant column has an eligible point, so + * an all-NULL column produces NULL rows rather than no rows. Each row is padded per column. + */ + @Test + public void testWindowWithNoEligiblePointsStillEmitsARow() { + Double[][] rows = {{null}, {null}, {null}, {null}}; + for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { + List out = sample(mode, 2, 2, 0, rows); + int timeIndex = out.size() - 2; + assertEquals(2, out.get(timeIndex).getPositionCount()); + assertTrue(out.get(timeIndex).build().isNull(0)); + assertTrue(out.get(timeIndex + 1).build().isNull(0)); + assertTrue(out.get(timeIndex).build().isNull(1)); + assertTrue(out.get(timeIndex + 1).build().isNull(1)); + } + } + @Test public void testTargetColumnsShareMemoryBudget() { TableFunctionDataProcessor processor = processor("TARGET_COUNT", 3, 3, 0, 8); From b7c5a8324c7ad1a448658003a5b7bc25e07f34e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Wed, 21 Oct 2026 14:55:18 +0800 Subject: [PATCH 10/21] fix(lttb): move the memory budget out of the UDF API and run LTTB over merged partitions - Introduce TableFunctionProcessorMemoryBudget in node-commons so only engine-internal providers declare a buffered-byte estimate; a user-defined table function no longer sees the new method on its provider interface. TableFunctionOperator now receives the resolved value, so the operator's reservation and the limit enforced by the processors always agree. - Exclude LTTB from fragment-local execution alongside FFT: a partitioned GroupNode does not make a partition complete within one branch, so LTTB must run once over the ordered merged partitions. Read the ordering of per-branch GroupNodes from the node itself, since they are absent from nodeOrderingMap. - Default lttb_buffer_size_in_bytes to 32 MiB, matching sort_buffer_size, and name the i18n message after the configuration key. - Extend the IT, analyzer and processor tests to cover the new semantics. --- .../it/db/it/IoTDBLTTBTableFunctionIT.java | 224 ++++- .../table/TableFunctionProcessorProvider.java | 12 - .../function/TableFunctionOperator.java | 9 +- .../plan/planner/TableOperatorGenerator.java | 7 + .../TableDistributedPlanGenerator.java | 20 +- .../tvf/TableFunctionOperatorTest.java | 3 + .../analyzer/TableFunctionTest.java | 29 + .../iotdb/commons/i18n/CommonMessages.java | 4 +- .../iotdb/commons/i18n/CommonMessages.java | 4 +- .../iotdb/commons/conf/CommonConfig.java | 8 +- .../TableFunctionProcessorMemoryBudget.java | 39 + .../relational/tvf/LTTBTableFunction.java | 912 +++++++++++++----- .../tvf/LTTBNumericBoundaryTest.java | 20 +- .../relational/tvf/LTTBRegressionTest.java | 81 +- .../relational/tvf/LTTBWindowMemoryTest.java | 20 +- 15 files changed, 1082 insertions(+), 310 deletions(-) create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/function/TableFunctionProcessorMemoryBudget.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java index ece266187316b..edfd5467415ac 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java @@ -31,10 +31,15 @@ import org.junit.runner.RunWith; import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -45,6 +50,13 @@ public class IoTDBLTTBTableFunctionIT { @BeforeClass public static void setUp() throws Exception { + // Four data region groups per database, so a partition can be split across regions and a + // per-fragment LTTB or an unordered merge would produce visibly wrong results. + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setDataRegionGroupExtensionPolicy("CUSTOM") + .setDefaultDataRegionGroupNumPerDatabase(4); EnvFactory.getEnv().initClusterEnvironment(); try (Connection connection = EnvFactory.getEnv().getTableConnection(); Statement statement = connection.createStatement()) { @@ -60,7 +72,7 @@ public static void setUp() throws Exception { + "(3, 'd1', 2.0, 9.0)," + "(4, 'd1', 0.0, 1.0)," + "(5, 'd1', 0.0, 0.0)," - + "(0, 'd2', null, null)," + + "(0, 'd2', 0.0, 0.0)," + "(1, 'd2', null, null)"); statement.execute( "CREATE TABLE sparse(device STRING TAG, value DOUBLE FIELD, marker INT32 FIELD)"); @@ -69,6 +81,12 @@ public static void setUp() throws Exception { + "(0, 'd1', 0.0, 1), (1, 'd1', 1.0, 1), (2, 'd1', 2.0, 1)," + "(3, 'd1', null, 1), (4, 'd1', null, 1), (5, 'd1', null, 1)," + "(6, 'd1', 0.0, 1), (7, 'd1', 0.0, 1), (8, 'd1', 0.0, 1)"); + // Two devices share one site tag; both write t = 0 so the partition has a duplicate + // timestamp. + statement.execute("CREATE TABLE shared(site STRING TAG, device STRING TAG, s1 DOUBLE FIELD)"); + statement.execute( + "INSERT INTO shared(time, site, device, s1) VALUES " + + "(0, 'a', 'x1', 1.0), (1, 'a', 'x1', 2.0), (0, 'a', 'x2', 3.0), (1, 'a', 'x2', 4.0)"); statement.execute("CREATE TABLE integral_values(device STRING TAG, value INT64 FIELD)"); long baseline = 1L << 60; long[] deltas = {0, 1, 8, 2, 0, 0}; @@ -80,6 +98,33 @@ public static void setUp() throws Exception { + (baseline + deltas[i]) + ")"); } + // Two partitions ('p' and 'q'), eight devices each, interleaved timestamps: the devices of + // one + // partition carry disjoint time ranges, so the partition is ordered once the rows are merged + // and a fragment-local sample would return one sample per region. + statement.execute( + "CREATE TABLE regions(part STRING TAG, device STRING TAG, s1 DOUBLE FIELD)"); + for (int device = 0; device < 8; device++) { + for (int row = 0; row < 4; row++) { + long time = device * 4L + row; + statement.execute( + "INSERT INTO regions(time, part, device, s1) VALUES (" + + time + + ", 'p', 'p" + + device + + "', " + + (time % 7) + + ".0)"); + statement.execute( + "INSERT INTO regions(time, part, device, s1) VALUES (" + + time + + ", 'q', 'q" + + device + + "', " + + (time % 5) + + ".0)"); + } + } statement.execute("FLUSH"); } catch (Exception e) { fail("insertData failed: " + e.getMessage()); @@ -100,8 +145,7 @@ public void testTargetCountPreservesEndpointsAndAlignsColumns() { new String[] { "0,d1,1970-01-01T00:00:00.000Z,0.0,1970-01-01T00:00:00.000Z,0.0,", "0,d1,1970-01-01T00:00:00.002Z,8.0,1970-01-01T00:00:00.003Z,9.0,", - "0,d1,1970-01-01T00:00:00.005Z,0.0,1970-01-01T00:00:00.005Z,0.0,", - "0,d2,null,null,null,null," + "0,d1,1970-01-01T00:00:00.005Z,0.0,1970-01-01T00:00:00.005Z,0.0," }, DATABASE_NAME); } @@ -114,9 +158,11 @@ public void testCountWindowAndTimeWindowOutput() { + "ORDER BY window_index", new String[] {"window_index", "device", "s1_time", "s1"}, new String[] { + "0,d1,1970-01-01T00:00:00.000Z,0.0,", "0,d1,1970-01-01T00:00:00.001Z,1.0,", "1,d1,1970-01-01T00:00:00.002Z,8.0,", - "2,d1,1970-01-01T00:00:00.004Z,0.0," + "2,d1,1970-01-01T00:00:00.004Z,0.0,", + "2,d1,1970-01-01T00:00:00.005Z,0.0," }, DATABASE_NAME); @@ -126,9 +172,11 @@ public void testCountWindowAndTimeWindowOutput() { + "ORIGIN => 1970-01-01T00:00:00.000+00:00) ORDER BY window_start", new String[] {"window_start", "window_end", "device", "s1_time", "s1"}, new String[] { + "1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.002Z,d1,1970-01-01T00:00:00.000Z,0.0,", "1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.002Z,d1,1970-01-01T00:00:00.001Z,1.0,", "1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.004Z,d1,1970-01-01T00:00:00.002Z,8.0,", - "1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.004Z,0.0," + "1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.004Z,0.0,", + "1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.005Z,0.0," }, DATABASE_NAME); } @@ -162,7 +210,7 @@ public void testDuplicateTimestampsInDifferentPartitionsRemainValid() { + "ORDER BY device", new String[] {"window_index", "device", "s1_time", "s1"}, new String[] { - "0,d1,1970-01-01T00:00:00.000Z,0.0,", "0,d2,null,null,", + "0,d1,1970-01-01T00:00:00.000Z,0.0,", "0,d2,1970-01-01T00:00:00.000Z,0.0,", }, DATABASE_NAME); } @@ -175,18 +223,20 @@ public void testWindowSkipsNullBucketForLookahead() { "SELECT * FROM LTTB(" + data + "SIZE => 3) ORDER BY window_index", new String[] {"window_index", "device", "value_time", "value"}, new String[] { + "0,d1,1970-01-01T00:00:00.000Z,0.0,", "0,d1,1970-01-01T00:00:00.002Z,2.0,", - "1,d1,null,null,", - "2,d1,1970-01-01T00:00:00.006Z,0.0," + "2,d1,1970-01-01T00:00:00.006Z,0.0,", + "2,d1,1970-01-01T00:00:00.008Z,0.0," }, DATABASE_NAME); tableResultSetEqualTest( "SELECT * FROM LTTB(" + data + "SIZE => 3ms) ORDER BY window_start", new String[] {"window_start", "window_end", "device", "value_time", "value"}, new String[] { + "1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.000Z,0.0,", "1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.002Z,2.0,", - "1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.006Z,d1,null,null,", - "1970-01-01T00:00:00.006Z,1970-01-01T00:00:00.009Z,d1,1970-01-01T00:00:00.006Z,0.0," + "1970-01-01T00:00:00.006Z,1970-01-01T00:00:00.009Z,d1,1970-01-01T00:00:00.006Z,0.0,", + "1970-01-01T00:00:00.006Z,1970-01-01T00:00:00.009Z,d1,1970-01-01T00:00:00.008Z,0.0," }, DATABASE_NAME); } @@ -228,13 +278,165 @@ public void testInt64SamplingPreservesSmallPeaksAtLargeBaselines() { prefix + "SIZE => 2) ORDER BY window_index", new String[] {"window_index", "device", "value_time", "value"}, new String[] { + "0,d1,1970-01-01T00:00:00.000Z,1152921504606846976,", "0,d1,1970-01-01T00:00:00.001Z,1152921504606846977,", "1,d1,1970-01-01T00:00:00.002Z,1152921504606846984,", - "2,d1,1970-01-01T00:00:00.004Z,1152921504606846976," + "2,d1,1970-01-01T00:00:00.004Z,1152921504606846976,", + "2,d1,1970-01-01T00:00:00.005Z,1152921504606846976," + }, + DATABASE_NAME); + } + + @Test + public void testSlideSmallerThanSizeKeepsOneAnchorChain() { + // SIZE => 2, SLIDE => 1: overlapping count windows. F(t0) is reported by the first window and + // L(t5) by the last one, and the windows in between continue the same anchor chain. + String data = + "DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') " + + "PARTITION BY device ORDER BY time, TIMECOL => 'time', "; + tableResultSetEqualTest( + "SELECT * FROM LTTB(" + data + "SIZE => 2, SLIDE => 1) ORDER BY window_index, s1_time", + new String[] {"window_index", "device", "s1_time", "s1"}, + new String[] { + "0,d1,1970-01-01T00:00:00.000Z,0.0,", + "0,d1,1970-01-01T00:00:00.001Z,1.0,", + "1,d1,1970-01-01T00:00:00.002Z,8.0,", + "2,d1,1970-01-01T00:00:00.003Z,2.0,", + "3,d1,1970-01-01T00:00:00.003Z,2.0,", + "4,d1,1970-01-01T00:00:00.004Z,0.0,", + "5,d1,1970-01-01T00:00:00.005Z,0.0," + }, + DATABASE_NAME); + // The time-window form with SLIDE < SIZE starts a window one slide before the first point. + tableResultSetEqualTest( + "SELECT * FROM LTTB(" + data + "SIZE => 2ms, SLIDE => 1ms) ORDER BY window_start, s1_time", + new String[] {"window_start", "window_end", "device", "s1_time", "s1"}, + new String[] { + "1969-12-31T23:59:59.999Z,1970-01-01T00:00:00.001Z,d1,1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.002Z,d1,1970-01-01T00:00:00.001Z,1.0,", + "1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.002Z,8.0,", + "1970-01-01T00:00:00.002Z,1970-01-01T00:00:00.004Z,d1,1970-01-01T00:00:00.003Z,2.0,", + "1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.005Z,d1,1970-01-01T00:00:00.003Z,2.0,", + "1970-01-01T00:00:00.004Z,1970-01-01T00:00:00.006Z,d1,1970-01-01T00:00:00.004Z,0.0,", + "1970-01-01T00:00:00.005Z,1970-01-01T00:00:00.007Z,d1,1970-01-01T00:00:00.005Z,0.0," + }, + DATABASE_NAME); + } + + @Test + public void testSlideLargerThanSizeSkipsTheGap() { + // SIZE => 1, SLIDE => 2 keeps only the windows that receive a row: [0,1), [2,3), [4,5). + String data = + "DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') " + + "PARTITION BY device ORDER BY time, TIMECOL => 'time', "; + tableResultSetEqualTest( + "SELECT * FROM LTTB(" + data + "SIZE => 1, SLIDE => 2) ORDER BY window_index, s1_time", + new String[] {"window_index", "device", "s1_time", "s1"}, + new String[] { + "0,d1,1970-01-01T00:00:00.000Z,0.0,", + "1,d1,1970-01-01T00:00:00.002Z,8.0,", + "2,d1,1970-01-01T00:00:00.004Z,0.0," }, DATABASE_NAME); } + @Test + public void testTimeWindowWithUnalignedOriginShiftsBoundaries() { + String data = + "DATA => (SELECT time, device, s1 FROM metrics WHERE device = 'd1') " + + "PARTITION BY device ORDER BY time, TIMECOL => 'time', "; + tableResultSetEqualTest( + "SELECT * FROM LTTB(" + + data + + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.001+00:00) " + + "ORDER BY window_start, s1_time", + new String[] {"window_start", "window_end", "device", "s1_time", "s1"}, + new String[] { + "1969-12-31T23:59:59.999Z,1970-01-01T00:00:00.001Z,d1,1970-01-01T00:00:00.000Z,0.0,", + "1970-01-01T00:00:00.001Z,1970-01-01T00:00:00.003Z,d1,1970-01-01T00:00:00.002Z,8.0,", + "1970-01-01T00:00:00.003Z,1970-01-01T00:00:00.005Z,d1,1970-01-01T00:00:00.003Z,2.0,", + "1970-01-01T00:00:00.005Z,1970-01-01T00:00:00.007Z,d1,1970-01-01T00:00:00.005Z,0.0," + }, + DATABASE_NAME); + } + + @Test + public void testInvalidTimeColumnIsRejected() { + tableAssertTestFail( + "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time, " + + "TIMECOL => 's1', N => 3)", + "701: ", + DATABASE_NAME); + tableAssertTestFail( + "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time, " + + "TIMECOL => 'missing', N => 3)", + "701: ", + DATABASE_NAME); + } + + @Test + public void testDuplicateTimestampsAcrossDevicesInOnePartitionAreRejected() { + // Both devices are tagged 'shared', so PARTITION BY site puts t = 0 of each device into one + // partition and the duplicate check has to reject it. + tableAssertTestFail( + "SELECT * FROM LTTB(DATA => (SELECT time, site, s1 FROM shared ORDER BY time) " + + "PARTITION BY site ORDER BY time, TIMECOL => 'time', N => 3)", + "701: LTTB does not support duplicate timestamps, but timestamp 0 appears more than once" + + " in the same partition.", + DATABASE_NAME); + } + + /** + * A partitioned GroupNode does not make a partition complete within one branch, so LTTB has to + * run once over the partitions merged in order. With four data region groups and a partition + * whose devices carry disjoint time ranges, a per-region sample or an unordered merge changes the + * result; {@code N => 1000} returns every eligible point, so all three inputs must agree. + */ + @Test + public void testNonDirectScanInputsMatchTheDirectScan() throws SQLException { + String direct = + "SELECT window_index, part, s1_time, s1 FROM LTTB(" + + "DATA => regions PARTITION BY part ORDER BY time, TIMECOL => 'time', N => 1000) " + + "ORDER BY part, s1_time"; + String windowFunctionInput = + "SELECT window_index, part, s1_time, s1 FROM LTTB(" + + "DATA => (SELECT time, part, s1, " + + "row_number() OVER (PARTITION BY part, device ORDER BY time) AS rn FROM regions) " + + "PARTITION BY part ORDER BY time, TIMECOL => 'time', N => 1000) " + + "ORDER BY part, s1_time"; + String sessionInput = + "SELECT window_index, part, s1_time, s1 FROM LTTB(" + + "DATA => (SELECT time, part, s1 FROM SESSION(" + + "DATA => regions PARTITION BY (part, device) ORDER BY time, TIMECOL => 'time', " + + "GAP => 1m)) PARTITION BY part ORDER BY time, TIMECOL => 'time', N => 1000) " + + "ORDER BY part, s1_time"; + + List expected = queryRows(direct); + // 8 devices x 4 rows per partition, 2 partitions. + assertEquals(64, expected.size()); + assertEquals(expected, queryRows(windowFunctionInput)); + assertEquals(expected, queryRows(sessionInput)); + } + + private static List queryRows(String sql) throws SQLException { + List rows = new ArrayList<>(); + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + int columnCount = resultSet.getMetaData().getColumnCount(); + while (resultSet.next()) { + StringBuilder row = new StringBuilder(); + for (int i = 1; i <= columnCount; i++) { + row.append(resultSet.getString(i)).append('|'); + } + rows.add(row.toString()); + } + } + } + return rows; + } + @Test public void testInvalidParameterCombinations() { String prefix = diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionProcessorProvider.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionProcessorProvider.java index 9ca917070d7e5..f86d631f5989a 100644 --- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionProcessorProvider.java +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/table/TableFunctionProcessorProvider.java @@ -43,16 +43,4 @@ default TableFunctionLeafProcessor getSplitProcessor() { throw new UnsupportedOperationException( UdfApiMessages.TABLE_FUNCTION_DOES_NOT_PROCESS_LEAF_DATA); } - - /** - * The bytes the processors of this provider may hold in their own buffers while a partition is - * being processed, above what {@code process} returns. The engine adds this to the estimated - * memory of the {@linkplain TableFunctionDataProcessor} operator, so a table function that - * accumulates input (rather than emitting a result per record) is visible to memory management - * and can be back-pressured. The default is {@code 0} for table functions that keep no state - * across calls. - */ - default long getEstimatedMaxBufferedBytes() { - return 0L; - } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java index d8508a32e2d30..f111c887ef7a9 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java @@ -72,7 +72,11 @@ public class TableFunctionOperator extends AbstractOperator implements ProcessOp private final boolean isDeclaredAsPassThrough; private final IoTDBLocal ioTDBLocal; - /** Bytes the data processors may buffer internally while a partition is processed. */ + /** + * Bytes the data processors may buffer internally while a partition is processed. It is resolved + * once from the provider, which also gives the same value to every processor it creates, so the + * reservation and the enforced limit cannot diverge. + */ private final long estimatedMaxBufferedBytes; private TableFunctionDataProcessor processor; @@ -85,6 +89,7 @@ public class TableFunctionOperator extends AbstractOperator implements ProcessOp public TableFunctionOperator( CommonOperatorContext operatorContext, TableFunctionProcessorProvider processorProvider, + long estimatedMaxBufferedBytes, Operator inputOperator, List inputDataTypes, List outputDataTypes, @@ -104,7 +109,7 @@ public TableFunctionOperator( new PartitionRecognizer( partitionChannels, requiredChannels, passThroughChannels, inputDataTypes); this.isDeclaredAsPassThrough = isDeclaredAsPassThrough; - this.estimatedMaxBufferedBytes = processorProvider.getEstimatedMaxBufferedBytes(); + this.estimatedMaxBufferedBytes = estimatedMaxBufferedBytes; this.needPassThrough = properChannelCount != outputDataTypes.size(); this.partitionState = null; this.properBlockBuilder = new TsBlockBuilder(outputDataTypes.subList(0, properChannelCount)); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java index ed52de7958b0d..563ffb6b293fc 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java @@ -122,6 +122,7 @@ import org.apache.iotdb.commons.queryengine.plan.relational.function.BoundSignature; import org.apache.iotdb.commons.queryengine.plan.relational.function.FunctionKind; import org.apache.iotdb.commons.queryengine.plan.relational.function.ITableFunctionFactory; +import org.apache.iotdb.commons.queryengine.plan.relational.function.TableFunctionProcessorMemoryBudget; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ResolvedFunction; import org.apache.iotdb.commons.queryengine.plan.relational.planner.OrderingScheme; import org.apache.iotdb.commons.queryengine.plan.relational.planner.SortOrder; @@ -1816,6 +1817,12 @@ public Operator visitTableFunctionProcessor(TableFunctionProcessorNode node, C c return new TableFunctionOperator( operatorContext, processorProvider, + // The budget is taken from the provider itself, so the operator reserves exactly what the + // processors of this provider enforce. A user-defined function reports nothing. + processorProvider instanceof TableFunctionProcessorMemoryBudget + ? ((TableFunctionProcessorMemoryBudget) processorProvider) + .getEstimatedMaxBufferedBytes() + : 0L, operator, inputDataTypes, outputDataTypes, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 06b97fa7b7d69..c5a137eb0f0c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -2884,7 +2884,13 @@ public List visitTableFunctionProcessor( node.setChild(childrenNodes.get(0)); return Collections.singletonList(node); } else if (!canSplitPushDown) { - OrderingScheme childOrdering = nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()); + // The per-branch GroupNodes created by visitGroup are not registered in nodeOrderingMap, so + // their ordering has to be read from the node itself. Every child is then either such a + // GroupNode or a node whose ordering starts with the same scheme, so a MergeSort is valid. + OrderingScheme childOrdering = + node.getChild() instanceof GroupNode + ? ((GroupNode) node.getChild()).getOrderingScheme() + : nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()); node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); return Collections.singletonList(node); } else { @@ -2899,11 +2905,13 @@ private boolean canSplitTableFunctionProcessor(TableFunctionProcessorNode node) if (!isPartitionedGroup(node.getChild())) { return false; } - // LTTB needs whole partitions, which the partitioned GroupNode above this node already - // guarantees: every fragment receives complete partitions and the ordering is preserved, so a - // fragment-local sample is a valid standard LTTB result for that partition and no exclusion is - // needed. FFT is excluded because its transform over the whole partition is not splittable. - return !TableBuiltinTableFunction.FFT.getFunctionName().equalsIgnoreCase(node.getName()); + // LTTB samples a whole partition and FFT transforms a whole partition, so neither may run once + // per fragment: a partitioned GroupNode does not make a partition complete within one branch. + // The function therefore has to run once, over the partitions merged in order. The split rule + // itself is a separate planner issue; the exclusion can be revisited once that is fixed. + String functionName = node.getName(); + return !TableBuiltinTableFunction.FFT.getFunctionName().equalsIgnoreCase(functionName) + && !TableBuiltinTableFunction.LTTB.getFunctionName().equalsIgnoreCase(functionName); } private boolean isPartitionedGroup(PlanNode node) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java index cf1117de40ac4..250883af2b6ad 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java @@ -371,6 +371,7 @@ private void appendRows( new TableFunctionOperator( operatorContext, provider, + 0L, singleRowChild, Collections.singletonList(TSDataType.INT64), withPassThrough @@ -444,6 +445,7 @@ public void process( new TableFunctionOperator( operatorContext, provider, + 0L, twoPartitionChild, Collections.singletonList(TSDataType.INT64), Collections.singletonList(TSDataType.INT64), @@ -516,6 +518,7 @@ public void beforeDestroy() { new TableFunctionOperator( operatorContext, provider, + 0L, child, Collections.singletonList(TSDataType.INT64), Collections.singletonList(TSDataType.INT64), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java index 13a579a403adc..4cf02cd2bf7bf 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java @@ -22,6 +22,8 @@ import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.queryengine.plan.relational.function.tvf.ForecastTableFunction; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TableFunctionProcessorNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.WindowNode; import org.apache.iotdb.commons.udf.builtin.relational.tvf.FFTTableFunction; import org.apache.iotdb.commons.udf.builtin.relational.tvf.LTTBTableFunction; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; @@ -57,6 +59,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.group; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.join; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.mergeSort; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.node; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.output; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.project; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.sort; @@ -962,6 +965,32 @@ public void testLTTBCountWindowMode() { tableFunctionMatcher, group(tableScan("testdb.table1", TABLE1_COLUMNS))))); } + /** + * A window function input plans a per-branch GroupNode that is not registered in the ordering + * map, so the non-split branch has to take the ordering from the GroupNode itself. Without that + * the children are merged by a CollectNode and partitions are mixed. + */ + @Test + public void testLTTBOverWindowFunctionInputIsMergedInOrder() { + PlanTester planTester = new PlanTester(); + String sql = + "SELECT * FROM LTTB(DATA => (SELECT time, tag3, s1, " + + "row_number() OVER (PARTITION BY tag3, s1 ORDER BY time) AS rn FROM table1) " + + "PARTITION BY tag3 ORDER BY time, TIMECOL => 'time', N => 4)"; + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + // The per-branch GroupNode is not registered in the ordering map, so the non-split branch must + // read its ordering from the node itself and merge the children with a MergeSort. With a + // CollectNode the partitions of different regions would be interleaved, and splitting LTTB per + // region would return one sample per region instead of one sample per partition. + assertPlan( + planTester.getFragmentPlan(0), + output( + node( + TableFunctionProcessorNode.class, + group(node(WindowNode.class, mergeSort(exchange(), exchange(), exchange())))))); + } + @Test public void testLTTBRejectsInvalidArguments() { String data = "DATA => (SELECT time, tag1, s3 FROM table1) PARTITION BY tag1 ORDER BY time"; diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java index 1014c2a686572..013aa5df68cec 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -371,8 +371,8 @@ private CommonMessages() {} public static final String EXCEPTION_LTTB_WINDOW_BOUNDARIES_EXCEED_THE_TIMESTAMP_RANGE_7C0FC7E2 = "LTTB window boundaries exceed the TIMESTAMP range."; public static final String - EXCEPTION_LTTB_BUFFERS_EXCEED_THE_PER_PROCESSOR_MEMORY_LIMIT_OF_ARG_BYTES_954FEF00 = - "LTTB buffers exceed the per-processor memory limit of %d bytes."; + EXCEPTION_LTTB_BUFFERS_EXCEED_LTTB_BUFFER_SIZE_IN_BYTES_ARG_BYTES_26BDE62D = + "LTTB buffers exceed lttb_buffer_size_in_bytes (%d bytes)."; public static final String EXCEPTION_SNAPSHOT_BUFFER_SIZE_MUST_NOT_EXCEED_ARG_BYTES_BUT_WAS_ARG_D1DA6F7E = "Snapshot buffer size must not exceed %d bytes, but was %d."; diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java index a32d4c1df13cc..e07ab7b5cdf1d 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java @@ -266,8 +266,8 @@ private CommonMessages() {} public static final String EXCEPTION_LTTB_WINDOW_BOUNDARIES_EXCEED_THE_TIMESTAMP_RANGE_7C0FC7E2 = "LTTB 窗口边界超出 TIMESTAMP 的取值范围。"; public static final String - EXCEPTION_LTTB_BUFFERS_EXCEED_THE_PER_PROCESSOR_MEMORY_LIMIT_OF_ARG_BYTES_954FEF00 = - "LTTB 缓冲区超过每个处理器 %d 字节的内存上限。"; + EXCEPTION_LTTB_BUFFERS_EXCEED_LTTB_BUFFER_SIZE_IN_BYTES_ARG_BYTES_26BDE62D = + "LTTB 缓冲区超过 lttb_buffer_size_in_bytes 限制(%d 字节)。"; public static final String EXCEPTION_SNAPSHOT_BUFFER_SIZE_MUST_NOT_EXCEED_ARG_BYTES_BUT_WAS_ARG_D1DA6F7E = "快照缓冲区大小不得超过 %d 字节,但实际为 %d。"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 653e616fc2f46..f18b402201852 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -569,8 +569,12 @@ public class CommonConfig { /** Max number of rows for cte materialization */ private int maxRowsInCteBuffer = 1000; - /** The per-processor memory budget of the LTTB table function */ - private long lttbBufferSize = 64 * 1024 * 1024L; + /** + * The per-processor memory budget of the LTTB table function. The shipped value is {@code 0}, + * which the DataNode resolves to {@code min(32 MiB, operator memory / query_thread_count / 2)}; + * this initial value matches {@link #sortBufferSize}. + */ + private long lttbBufferSize = 32 * 1024 * 1024L; CommonConfig() { // Empty constructor diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/function/TableFunctionProcessorMemoryBudget.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/function/TableFunctionProcessorMemoryBudget.java new file mode 100644 index 0000000000000..62618fb3f76de --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/function/TableFunctionProcessorMemoryBudget.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.queryengine.plan.relational.function; + +/** + * Engine-internal memory accounting for the processors of a built-in table function. + * + *

    It is deliberately kept out of {@code udf-api}: a user-defined table function does not declare + * any buffered bytes, and the engine treats every provider that does not implement this interface + * as keeping no state across calls. The value is resolved once per provider, and the processors + * created by that provider must enforce exactly the same limit, so the estimate the operator + * reserves and the limit the processors apply cannot diverge, not even when the configuration is + * hot-reloaded while a query runs. + */ +public interface TableFunctionProcessorMemoryBudget { + + /** + * Bytes the processors of this provider may buffer internally while one partition is processed, + * above what {@code process} returns. + */ + long getEstimatedMaxBufferedBytes(); +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index 68793615c82b9..dbf8ab69bd568 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.i18n.CommonMessages; +import org.apache.iotdb.commons.queryengine.plan.relational.function.TableFunctionProcessorMemoryBudget; import org.apache.iotdb.udf.api.exception.UDFException; import org.apache.iotdb.udf.api.relational.TableFunction; import org.apache.iotdb.udf.api.relational.access.Record; @@ -69,30 +70,35 @@ * the column has no more than {@code N} eligible points). The output uses a fixed {@code * window_index = 0}. *

  • Window/bucket mode ({@code SIZE}, optional {@code SLIDE}/{@code ORIGIN}): windows - * are built exactly like M4 (time windows for duration literals, count windows for integers). - * {@code SIZE} defines a bucket, not the input range of a separate standard LTTB invocation. - * One anchor chain is maintained per partition and participant column: for the current bucket - * {@code A} is the point selected by the preceding bucket of that column ({@code B} in turn - * becomes the anchor of the following bucket), {@code B} ranges over the current bucket's - * eligible points, and {@code C} is the average point of the next bucket that carries data - * for that column (or the bucket's own last eligible point when there is none). Overlapping - * buckets ({@code SLIDE < SIZE}) do not restart the chain, and the state is independent - * between partitions and between participant columns. + * are built exactly like M4 (time windows for duration literals, count windows for integers), + * but {@code SIZE} defines a bucket rather than the input range of a separate standard LTTB + * invocation. The result is standard LTTB whose buckets are the windows: + *
      + *
    • Per partition and participant column, the first eligible point {@code F} and the last + * eligible point {@code L} that fall in a window are buckets of their own and are + * always returned, as in the original algorithm. {@code F} is reported by the first + * window that contains it and {@code L} by the last one. + *
    • Every other window contributes one bucket made of its remaining eligible points. The + * anchor chain starts at {@code F}: for a window bucket {@code A} is the previously + * selected point, {@code C} is the average point of the next window bucket that has + * points ({@code L} when there is none), ties go to the earliest point, and buckets + * without points are skipped. Overlapping windows ({@code SLIDE < SIZE}) keep one + * chain, and the state is independent between partitions and columns. + *
    * * - *

    Bucket mode emits one row per window that carries input, and at most one selected point per - * participant column in a row; a column with no eligible point in a window emits {@code NULL} in - * its place, and a window in which every participant column is {@code NULL} still emits a single - * all-{@code NULL} row (padding by output position). Bucket mode does not preserve the - * first and last eligible points: an endpoint used as {@code A} or as {@code C} is a triangle - * vertex, not a selected point, so it appears in the output only when it also wins its bucket. A - * bucket holding just one or two eligible points therefore still contributes a single row per - * column: with two points the areas are equal and the tie resolves to the earlier one. First/last - * preservation and the return-all-points case apply to target-count mode only. + *

    Bucket mode emits, per window, up to three rows per participant column: {@code F}, the point + * selected from the window bucket, and {@code L}. Rows are padded by position as in M4, and a + * window in which no participant column has an eligible point emits no row. Target-count mode emits + * as many rows as the column with the most selected points and no row when that number is zero. * *

    {@code NULL} participant values are ignored per column; every non-null value, including {@code - * NaN} and the infinities, participates in the selection and is reproduced in the output. Ties in - * the triangle area are broken deterministically by the earliest timestamp (stable input order). + * NaN} and the infinities, is eligible and is reproduced in the output. A non-finite value cannot + * serve as a triangle vertex, though: the anchor is the most recent finite selected point (or the + * first finite eligible point), the lookahead average uses finite points only and skips buckets + * without one, and a bucket without a finite candidate returns its earliest point without moving + * the anchor. Ties in the triangle area are broken deterministically by the earliest timestamp + * (stable input order). * *

    The timestamps of the input rows of one partition must be unique. As soon as two rows of the * same partition carry the same timestamp the query fails with a semantic error ({@code 701}); the @@ -123,12 +129,13 @@ public class LTTBTableFunction implements TableFunction { public static final int MAX_BUFFERED_POINTS = 1 << 22; /** - * Shared budget for point buffers and selection state across all columns of one processor. It is - * the default of {@code lttb_buffer_size_in_bytes}; operators can tune the live value with that - * hot-reloadable configuration parameter for workloads with large partitions or many overlapping - * windows. + * Fallback for {@code lttb_buffer_size_in_bytes} when it is not positive. The configured default + * in {@code iotdb-system.properties.template} is {@code 0}, which {@code IoTDBDescriptor} + * resolves to {@code min(32 MiB, operator memory / query_thread_count / 2)}; this constant + * therefore only matches {@link org.apache.iotdb.commons.conf.CommonConfig}'s initial value and + * keeps a misconfigured node bounded. */ - public static final long DEFAULT_MAX_BUFFER_BYTES = 64L * 1024 * 1024; + public static final long DEFAULT_MAX_BUFFER_BYTES = 32L * 1024 * 1024; public static final String MODE_PROPERTY = "__LTTB_MODE"; public static final String PARTITION_TYPES_PROPERTY = "__LTTB_PARTITION_TYPES"; @@ -149,11 +156,14 @@ public enum Mode { } /** - * The live per-processor buffer budget, read from the hot-reloadable {@code - * lttb_buffer_size_in_bytes} configuration parameter. Values that are not positive fall back to - * {@link #DEFAULT_MAX_BUFFER_BYTES} so a misconfigured node still bounds memory instead of - * buffering without limit. Processors read this once at construction, so a reload applies to - * queries that start afterwards. + * The live value of the hot-reloadable {@code lttb_buffer_size_in_bytes} configuration parameter. + * Values that are not positive fall back to {@link #DEFAULT_MAX_BUFFER_BYTES} so a misconfigured + * node still bounds memory instead of buffering without limit. + * + *

    This is read once per {@linkplain TableFunctionProcessorProvider}, in {@link + * #getProcessorProvider(TableFunctionHandle)}. Every processor of that provider enforces the same + * value, and the operator reserves the same value, so a hot reload cannot make the two differ for + * a running query; it applies to providers created afterwards. */ public static long maxBufferBytes() { long configured = CommonDescriptor.getInstance().getConfig().getLttbBufferSize(); @@ -320,44 +330,74 @@ public TableFunctionProcessorProvider getProcessorProvider( WindowTVFUtils.parseTypes((String) handle.getProperty(PARTITION_TYPES_PROPERTY)); Type[] participantTypes = WindowTVFUtils.parseTypes((String) handle.getProperty(PARTICIPANT_TYPES_PROPERTY)); + // Resolved once so that every processor of this provider enforces the same limit the operator + // reserves, even if lttb_buffer_size_in_bytes is hot-reloaded while the query runs. + long maxBufferedBytes = maxBufferBytes(); - return new TableFunctionProcessorProvider() { - @Override - public TableFunctionDataProcessor getDataProcessor() { - ParticipantColumn[] participantColumns = - createParticipantColumns(participantTypes, partitionTypes.length + 1); - switch (mode) { - case TARGET_COUNT: - return new TargetCountLTTBDataProcessor( - (long) handle.getProperty(N_PARAMETER_NAME), partitionTypes, participantColumns); - case TIME_WINDOW: - return new TimeWindowLTTBDataProcessor( - (long) handle.getProperty(SIZE_PARAMETER_NAME), - (long) handle.getProperty(SLIDE_PARAMETER_NAME), - (long) handle.getProperty(ORIGIN_PARAMETER_NAME), - partitionTypes, - participantColumns); - case COUNT_WINDOW: - return new CountWindowLTTBDataProcessor( - (long) handle.getProperty(SIZE_PARAMETER_NAME), - (long) handle.getProperty(SLIDE_PARAMETER_NAME), - partitionTypes, - participantColumns); - default: - throw new IllegalStateException(String.valueOf(mode)); - } - } + return new LTTBProcessorProvider( + mode, partitionTypes, participantTypes, handle, maxBufferedBytes); + } + + /** + * The provider of one LTTB invocation. It reports the budget it resolved at construction, so the + * operator reserves exactly the limit its processors enforce. + */ + private static final class LTTBProcessorProvider + implements TableFunctionProcessorProvider, TableFunctionProcessorMemoryBudget { + private final Mode mode; + private final Type[] partitionTypes; + private final Type[] participantTypes; + private final MapTableFunctionHandle handle; + private final long maxBufferedBytes; + + private LTTBProcessorProvider( + Mode mode, + Type[] partitionTypes, + Type[] participantTypes, + MapTableFunctionHandle handle, + long maxBufferedBytes) { + this.mode = mode; + this.partitionTypes = partitionTypes; + this.participantTypes = participantTypes; + this.handle = handle; + this.maxBufferedBytes = maxBufferedBytes; + } - /** - * Every LTTB processor buffers the points of the partition (or of the windows it keeps open) - * until it can select the output points, so the whole per-processor budget is reportable as - * memory the operator holds while the partition is processed. - */ - @Override - public long getEstimatedMaxBufferedBytes() { - return maxBufferBytes(); + @Override + public long getEstimatedMaxBufferedBytes() { + return maxBufferedBytes; + } + + @Override + public TableFunctionDataProcessor getDataProcessor() { + ParticipantColumn[] participantColumns = + createParticipantColumns(participantTypes, partitionTypes.length + 1); + switch (mode) { + case TARGET_COUNT: + return new TargetCountLTTBDataProcessor( + (long) handle.getProperty(N_PARAMETER_NAME), + partitionTypes, + participantColumns, + maxBufferedBytes); + case TIME_WINDOW: + return new TimeWindowLTTBDataProcessor( + (long) handle.getProperty(SIZE_PARAMETER_NAME), + (long) handle.getProperty(SLIDE_PARAMETER_NAME), + (long) handle.getProperty(ORIGIN_PARAMETER_NAME), + partitionTypes, + participantColumns, + maxBufferedBytes); + case COUNT_WINDOW: + return new CountWindowLTTBDataProcessor( + (long) handle.getProperty(SIZE_PARAMETER_NAME), + (long) handle.getProperty(SLIDE_PARAMETER_NAME), + partitionTypes, + participantColumns, + maxBufferedBytes); + default: + throw new IllegalStateException(String.valueOf(mode)); } - }; + } } private static ParticipantColumn[] createParticipantColumns(Type[] types, int firstInputIndex) { @@ -376,11 +416,18 @@ private static ParticipantColumn[] createParticipantColumns(Type[] types, int fi * Standard LTTB selection over an ordered point sequence. * *

    When {@code count <= n} every index is returned. Otherwise the first and last points are - * retained and the {@code count - 2} intermediate points are divided into {@code n - 2} buckets; - * bucket {@code i} covers {@code [floor(i*(count-2)/(n-2))+1, floor((i+1)*(count-2)/(n-2))+1)}. - * For every bucket the candidate forming the largest triangle with the previously selected point - * and the average point of the next bucket wins (the last bucket uses the final point). Ties are - * broken by the earliest timestamp. + * retained, bucket {@code i} covers {@code [floor(i*(count-2)/(n-2))+1, + * floor((i+1)*(count-2)/(n-2))+1)}, and for every bucket the candidate forming the largest + * triangle with the anchor and the lookahead average wins. The last bucket uses the final point + * as the lookahead, and ties are broken by the earliest timestamp. + * + *

    {@code NaN} and infinite values are eligible points, but they cannot serve as a triangle + * vertex: an undefined area would make every candidate tie and the bucket would return its + * earliest point regardless of the data. The anchor is therefore the most recent finite selected + * point (or the first finite point), the lookahead average uses only finite points and skips + * buckets without one, and a bucket with no finite candidate returns its earliest point without + * moving the anchor. Non-finite points are still selected and returned when their bucket has no + * finite alternative. * * @return ascending indexes of the selected points */ @@ -403,31 +450,61 @@ private static int[] selectTargetCount( selected[selectedCount++] = 0; long bucketCount = n - 2; long intermediate = count - 2L; + // The anchor starts at the earliest finite point so that every area it participates in has a + // representable origin. It only moves to a selected point; a bucket with no finite candidate + // leaves it in place. int anchor = 0; + while (anchor < count && !isFiniteValue(values, integralValues, anchor)) { + anchor++; + } + boolean hasAnchor = anchor < count; for (long bucket = 0; bucket < bucketCount; bucket++) { int currentStart = (int) (bucket * intermediate / bucketCount) + 1; int currentEnd = (int) ((bucket + 1) * intermediate / bucketCount) + 1; + int nextEnd = + bucket == bucketCount - 1 ? count : (int) ((bucket + 2) * intermediate / bucketCount) + 1; - long anchorTime = times[anchor]; - double anchorValue = values[anchor]; - long integralAnchor = integralValues == null ? 0 : integralValues[anchor]; + long anchorTime = times[hasAnchor ? anchor : 0]; + double anchorValue = hasAnchor ? values[anchor] : 0; + long integralAnchor = hasAnchor && integralValues != null ? integralValues[anchor] : 0; double avgTime; double avgValue; - int nextEnd = - bucket == bucketCount - 1 ? count : (int) ((bucket + 2) * intermediate / bucketCount) + 1; + int lookaheadEnd; + if (bucket == bucketCount - 1) { + lookaheadEnd = count; + } else { + lookaheadEnd = nextEnd; + } if (bucket == bucketCount - 1) { avgTime = differenceAsDouble(times[count - 1], anchorTime); avgValue = relativeValue(values, integralValues, count - 1, anchorValue, integralAnchor); } else { double sumTime = 0; double sumValue = 0; - for (int i = currentEnd; i < nextEnd; i++) { + int finiteCount = 0; + for (int i = currentEnd; i < lookaheadEnd; i++) { + if (!isFiniteValue(values, integralValues, i)) { + continue; + } sumTime += differenceAsDouble(times[i], anchorTime); sumValue += relativeValue(values, integralValues, i, anchorValue, integralAnchor); + finiteCount++; + } + if (finiteCount == 0) { + // No finite point in the next bucket: use the last finite point of the partition. + int fallback = count - 1; + while (fallback >= 0 && !isFiniteValue(values, integralValues, fallback)) { + fallback--; + } + if (fallback < 0) { + fallback = count - 1; + } + avgTime = differenceAsDouble(times[fallback], anchorTime); + avgValue = relativeValue(values, integralValues, fallback, anchorValue, integralAnchor); + } else { + avgTime = sumTime / finiteCount; + avgValue = sumValue / finiteCount; } - int nextSize = nextEnd - currentEnd; - avgTime = sumTime / nextSize; - avgValue = sumValue / nextSize; } int pick = @@ -457,10 +534,13 @@ private static int[] selectTargetCount( values, integralValues, currentEnd, - nextEnd); + lookaheadEnd); } selected[selectedCount++] = pick; - anchor = pick; + if (isFiniteValue(values, integralValues, pick)) { + anchor = pick; + hasAnchor = true; + } } selected[selectedCount] = count - 1; return selected; @@ -497,6 +577,11 @@ private static boolean hasExactValue(double[] values, long[] integralValues, int return integralValues != null || Double.isFinite(values[index]); } + /** Whether the participant value at {@code index} can serve as a triangle vertex. */ + private static boolean isFiniteValue(double[] values, long[] integralValues, int index) { + return integralValues != null || Double.isFinite(values[index]); + } + private static int argMaxArea( long[] times, double[] values, @@ -508,10 +593,10 @@ private static int argMaxArea( long integralAnchor, double nextTime, double nextValue) { - // The exact fallback rebuilds the lookahead sum from the original points, so a non-finite - // average over representable points is still recoverable; only the anchor origin has to be - // representable, because it is converted to BigDecimal directly. A non-finite anchor value can - // only arise from non-finite input, which no arithmetic can repair. + // An undefined area means one of the vertices is not representable exactly (NaN or infinity); + // such a candidate cannot be scored. An overflow over representable input is different: the + // exact fallback recomputes it from the original points, so the whole bucket switches to it — + // but only when the anchor is itself representable, because the fallback converts it directly. boolean exactIsPossible = Double.isFinite(anchorValue); int pick = start; double maxArea = -1; @@ -640,7 +725,7 @@ void reserve(long bytes) { throw new SemanticException( String.format( CommonMessages - .EXCEPTION_LTTB_BUFFERS_EXCEED_THE_PER_PROCESSOR_MEMORY_LIMIT_OF_ARG_BYTES_954FEF00, + .EXCEPTION_LTTB_BUFFERS_EXCEED_LTTB_BUFFER_SIZE_IN_BYTES_ARG_BYTES_26BDE62D, limit)); } reservedBytes += bytes; @@ -694,21 +779,70 @@ private void collect(Record record, long time, PointBuffer buffer) { } } + private boolean isNull(Record record) { + return record.isNull(inputIndex); + } + + /** The value as a double, which is lossy only for INT64 values above 2^53. */ + private double floatingValue(Record record) { + switch (type) { + case INT32: + return record.getInt(inputIndex); + case INT64: + return record.getLong(inputIndex); + case FLOAT: + return record.getFloat(inputIndex); + case DOUBLE: + return record.getDouble(inputIndex); + default: + throw new IllegalArgumentException( + String.format( + CommonMessages.EXCEPTION_UNSUPPORTED_LTTB_VALUE_TYPE_ARG_3E60F6FE, type)); + } + } + + /** The exact value of an integral column, or {@code 0} for a floating-point column. */ + private long integralValue(Record record) { + switch (type) { + case INT32: + return record.getInt(inputIndex); + case INT64: + return record.getLong(inputIndex); + default: + return 0L; + } + } + private void write( ColumnBuilder timeBuilder, ColumnBuilder valueBuilder, PointBuffer buffer, int index) { - timeBuilder.writeLong(buffer.times[index]); + writePoint( + timeBuilder, + valueBuilder, + buffer.times[index], + buffer.values[index], + buffer.integralValues == null ? 0L : buffer.integralValues[index]); + } + + /** Writes one point; the integral value is only meaningful for an integral column. */ + private void writePoint( + ColumnBuilder timeBuilder, + ColumnBuilder valueBuilder, + long time, + double value, + long integral) { + timeBuilder.writeLong(time); switch (type) { case INT32: - valueBuilder.writeInt((int) buffer.integralValues[index]); + valueBuilder.writeInt((int) integral); break; case INT64: - valueBuilder.writeLong(buffer.integralValues[index]); + valueBuilder.writeLong(integral); break; case FLOAT: - valueBuilder.writeFloat((float) buffer.values[index]); + valueBuilder.writeFloat((float) value); break; case DOUBLE: - valueBuilder.writeDouble(buffer.values[index]); + valueBuilder.writeDouble(value); break; default: throw new IllegalArgumentException( @@ -771,6 +905,62 @@ void addIntegral(long time, long value) { size++; } + /** Appends a point whose exact integral value is only meaningful for an integral column. */ + void add(long time, double value, long integral) { + if (integralValues == null) { + addFloating(time, value); + } else { + ensureCapacity(); + times[size] = time; + values[size] = value; + integralValues[size] = integral; + size++; + } + } + + /** Drops the last point, which is used to move the partition's last eligible point out. */ + void truncateLast() { + size--; + } + + /** Average timestamp of the finite points, relative to {@code origin}. */ + double finiteAverageTime(long origin) { + double sum = 0; + int count = 0; + for (int i = 0; i < size; i++) { + if (!Double.isFinite(values[i])) { + continue; + } + sum += differenceAsDouble(times[i], origin); + count++; + } + return sum / count; + } + + /** Average value of the finite points, relative to {@code origin}. */ + double finiteAverageValue(double origin, long integralOrigin) { + double sum = 0; + int count = 0; + for (int i = 0; i < size; i++) { + if (!Double.isFinite(values[i])) { + continue; + } + sum += relativeValue(values, integralValues, i, origin, integralOrigin); + count++; + } + return sum / count; + } + + /** Whether at least one point has a finite value. */ + boolean hasFinitePoint() { + for (int i = 0; i < size; i++) { + if (Double.isFinite(values[i])) { + return true; + } + } + return false; + } + private void ensureCapacity() { if (size < times.length) { return; @@ -813,22 +1003,6 @@ long time(int index) { double value(int index) { return values[index]; } - - double averageTime(long origin) { - double sum = 0; - for (int i = 0; i < size; i++) { - sum += differenceAsDouble(times[i], origin); - } - return sum / size; - } - - double averageValue(double origin, long integralOrigin) { - double sum = 0; - for (int i = 0; i < size; i++) { - sum += relativeValue(values, integralValues, i, origin, integralOrigin); - } - return sum / size; - } } // --------------------------------------------------------------------------------------------- @@ -935,13 +1109,17 @@ protected final void writeRow( private static final class TargetCountLTTBDataProcessor extends AbstractLTTBDataProcessor { private final long n; private final PointBuffer[] buffers; - private final BufferMemoryBudget memoryBudget = new BufferMemoryBudget(maxBufferBytes()); + private final BufferMemoryBudget memoryBudget; private boolean hasRows; private TargetCountLTTBDataProcessor( - long n, Type[] partitionTypes, ParticipantColumn[] participantColumns) { + long n, + Type[] partitionTypes, + ParticipantColumn[] participantColumns, + long maxBufferedBytes) { super(partitionTypes, participantColumns); this.n = n; + this.memoryBudget = new BufferMemoryBudget(maxBufferedBytes); memoryBudget.reserve(RamUsageEstimator.sizeOfObjectArray(participantColumns.length)); this.buffers = newBuffers(memoryBudget); } @@ -972,7 +1150,7 @@ public void finish( } memoryBudget.reserve(selectionBytes); int[][] selections = new int[participantColumns.length][]; - int rowCount = 1; + int rowCount = 0; for (int i = 0; i < participantColumns.length; i++) { PointBuffer buffer = buffers[i]; selections[i] = @@ -997,16 +1175,17 @@ public void beforeDestroy() { } } - /** One window (= one LTTB bucket) worth of eligible points. */ + /** + * One window of the partition: the eligible points that fall in it, per participant column. The + * first and last eligible point of a partition are not stored here, because they form buckets of + * their own. + */ private abstract static class WindowState { protected final PointBuffer[] buffers; - protected final PointBuffer[] nextNonemptyBuffers; protected final long endExclusive; - protected int unresolvedColumns; private WindowState(PointBuffer[] buffers, long endExclusive) { this.buffers = buffers; - this.nextNonemptyBuffers = new PointBuffer[buffers.length]; this.endExclusive = endExclusive; } @@ -1054,12 +1233,54 @@ protected int getWindowColumnCount() { } } + /** One output cell of window mode: a selected point, or the first/last eligible point. */ + private static final class WindowRow { + private final long time; + private final double value; + private final long integral; + + private WindowRow(long time, double value, long integral) { + this.time = time; + this.value = value; + this.integral = integral; + } + + private static WindowRow of(PointBuffer buffer, int index) { + return new WindowRow( + buffer.times[index], + buffer.values[index], + buffer.integralValues == null ? 0L : buffer.integralValues[index]); + } + } + /** - * Window/bucket mode. Windows are constructed like M4 and emitted in window order; a window is - * emitted only once the following window with data is closed (or in {@link #finish}), because the - * average point of that following window is the third vertex of the LTTB triangle. Anchors are - * chained per participant column across the windows of the partition; windows without data for a - * column leave its anchor untouched. + * Window/bucket mode. Windows are constructed like M4, and {@code SIZE} defines a bucket rather + * than the input range of a separate standard LTTB invocation; the result is standard LTTB whose + * buckets are the windows: + * + *

      + *
    • Per partition and participant column, the first eligible point {@code F} and the last + * eligible point {@code L} that fall in a window are buckets of their own and are always + * returned. {@code F} is reported by the first window that contains it and {@code L} by the + * last one, so a window emits up to three rows: {@code F}, its selected point and {@code + * L}. + *
    • Every other window contributes one bucket made of its remaining eligible points. The + * anchor chain starts at {@code F}: for a window bucket {@code A} is the previously + * selected point, {@code C} is the average point of the next window bucket that has points + * (and {@code L} when there is none), ties go to the earliest point, and buckets without + * points are skipped. Overlapping windows ({@code SLIDE < SIZE}) keep one chain. + *
    • A window in which no participant column has an eligible point emits no row. + *
    + * + *

    Rows are padded by position, as in M4. {@code L} is only known at the end of the partition, + * so a window is emitted once the bucket that supplies its {@code C} is closed and known not to + * end the partition, or in {@link #finish}. + * + *

    Non-finite values stay eligible, but a {@code NaN} or infinite point cannot be a triangle + * vertex: {@code A} is the most recent finite selected point (or the first finite eligible + * point), {@code C} averages only finite points and skips buckets without one, falling back to + * the last finite eligible point, and a bucket without a finite candidate returns its earliest + * point without moving {@code A}. */ private abstract static class AbstractWindowLTTBDataProcessor extends AbstractLTTBDataProcessor { @@ -1072,26 +1293,70 @@ private abstract static class AbstractWindowLTTBDataProcessor activeWindows = new ArrayDeque<>(); private final Deque pendingWindows = new ArrayDeque<>(); - private final WindowState[] lastNonemptyWindows; - private final BufferMemoryBudget memoryBudget = new BufferMemoryBudget(maxBufferBytes()); + private final BufferMemoryBudget memoryBudget; private final long windowMetadataBytes; + + private final boolean[] hasFirstEligible; + private final long[] firstEligibleTimes; + private final double[] firstEligibleValues; + private final long[] integralFirstEligibleValues; + private final WindowState[] firstEligibleWindows; + private final boolean[] hasLastEligible; + private final long[] lastEligibleTimes; + private final double[] lastEligibleValues; + private final long[] integralLastEligibleValues; + private final long[] lastEligiblePositions; + private final boolean[] hasLastFinite; + private final long[] lastFiniteTimes; + private final double[] lastFiniteValues; + private final long[] integralLastFiniteValues; + private final boolean[] hasFirstFinite; + private final long[] firstFiniteTimes; + private final double[] firstFiniteValues; + private final long[] integralFirstFiniteValues; private final boolean[] hasAnchor; private final long[] anchorTimes; private final double[] anchorValues; private final long[] integralAnchorValues; + private final WindowState[] lastRowWindows; + private boolean finishing; protected AbstractWindowLTTBDataProcessor( - long size, long slide, Type[] partitionTypes, ParticipantColumn[] participantColumns) { + long size, + long slide, + Type[] partitionTypes, + ParticipantColumn[] participantColumns, + long maxBufferedBytes) { super(partitionTypes, participantColumns); this.size = size; this.slide = slide; + this.memoryBudget = new BufferMemoryBudget(maxBufferedBytes); + int columnCount = participantColumns.length; this.windowMetadataBytes = - WINDOW_INSTANCE_SIZE + 2 * RamUsageEstimator.sizeOfObjectArray(participantColumns.length); - this.lastNonemptyWindows = new WindowState[participantColumns.length]; - this.hasAnchor = new boolean[participantColumns.length]; - this.anchorTimes = new long[participantColumns.length]; - this.anchorValues = new double[participantColumns.length]; - this.integralAnchorValues = new long[participantColumns.length]; + WINDOW_INSTANCE_SIZE + 3 * RamUsageEstimator.sizeOfObjectArray(columnCount); + this.hasFirstEligible = new boolean[columnCount]; + this.firstEligibleTimes = new long[columnCount]; + this.firstEligibleValues = new double[columnCount]; + this.integralFirstEligibleValues = new long[columnCount]; + this.firstEligibleWindows = new WindowState[columnCount]; + this.hasLastEligible = new boolean[columnCount]; + this.lastEligibleTimes = new long[columnCount]; + this.lastEligibleValues = new double[columnCount]; + this.integralLastEligibleValues = new long[columnCount]; + this.lastEligiblePositions = new long[columnCount]; + this.hasLastFinite = new boolean[columnCount]; + this.lastFiniteTimes = new long[columnCount]; + this.lastFiniteValues = new double[columnCount]; + this.integralLastFiniteValues = new long[columnCount]; + this.hasFirstFinite = new boolean[columnCount]; + this.firstFiniteTimes = new long[columnCount]; + this.firstFiniteValues = new double[columnCount]; + this.integralFirstFiniteValues = new long[columnCount]; + this.hasAnchor = new boolean[columnCount]; + this.anchorTimes = new long[columnCount]; + this.anchorValues = new double[columnCount]; + this.integralAnchorValues = new long[columnCount]; + this.lastRowWindows = new WindowState[columnCount]; } protected final PointBuffer[] newWindowBuffers() { @@ -1121,130 +1386,322 @@ public final void process( protected abstract void processRecord( Record input, long time, List properColumnBuilders); - /** Close windows in order and link each column to its next nonempty, closed bucket. */ - protected final void emitClosedWindows(long position, List builders) { - while (!activeWindows.isEmpty() && activeWindows.peekFirst().endExclusive <= position) { - closeWindow(activeWindows.removeFirst()); - } - emitReadyWindows(builders, false); - } - - private void closeWindow(W window) { + /** + * Adds one input row to every open window that contains it. The first eligible point of a + * column is kept aside as the {@code F} bucket instead of being added to a window bucket. + */ + protected final void collectIntoWindows(long position, Record input, long time) { for (int i = 0; i < participantColumns.length; i++) { - if (!window.buffers[i].isEmpty()) { - WindowState previous = lastNonemptyWindows[i]; - if (previous != null) { - previous.nextNonemptyBuffers[i] = window.buffers[i]; - previous.unresolvedColumns--; + // A point that falls in a gap of a SLIDE > SIZE layout belongs to no window at all. + if (participantColumns[i].isNull(input) || activeWindows.isEmpty()) { + continue; + } + double value = participantColumns[i].floatingValue(input); + long integral = participantColumns[i].integralValue(input); + if (Double.isFinite(value)) { + if (!hasFirstFinite[i]) { + hasFirstFinite[i] = true; + firstFiniteTimes[i] = time; + firstFiniteValues[i] = value; + integralFirstFiniteValues[i] = integral; } - lastNonemptyWindows[i] = window; - window.unresolvedColumns++; + hasLastFinite[i] = true; + lastFiniteTimes[i] = time; + lastFiniteValues[i] = value; + integralLastFiniteValues[i] = integral; + } + if (!hasFirstEligible[i]) { + hasFirstEligible[i] = true; + firstEligibleTimes[i] = time; + firstEligibleValues[i] = value; + integralFirstEligibleValues[i] = integral; + firstEligibleWindows[i] = activeWindows.peekFirst(); + continue; + } + hasLastEligible[i] = true; + lastEligibleTimes[i] = time; + lastEligibleValues[i] = value; + integralLastEligibleValues[i] = integral; + lastEligiblePositions[i] = position; + for (W window : activeWindows) { + window.buffers[i].add(time, value, integral); } } - pendingWindows.addLast(window); } - private void emitReadyWindows(List builders, boolean finishing) { - while (!pendingWindows.isEmpty() - && (finishing || pendingWindows.peekFirst().unresolvedColumns == 0)) { - W current = pendingWindows.removeFirst(); - emitWindow(current, builders); - releaseWindow(current); + /** Moves the windows that ended before {@code position} into the pending queue. */ + protected final void emitClosedWindows(long position, List builders) { + while (!activeWindows.isEmpty() && activeWindows.peekFirst().endExclusive <= position) { + pendingWindows.addLast(activeWindows.removeFirst()); } + emitReadyWindows(builders); } @Override public final void finish( List properColumnBuilders, ColumnBuilder passThroughIndexBuilder) { + moveLastEligibleOutOfBuckets(); + emitReadyWindows(properColumnBuilders); + Arrays.fill(firstEligibleWindows, null); + Arrays.fill(lastRowWindows, null); + } + + /** + * {@code L} is the last eligible point of a column and is a bucket of its own, but it was + * collected into the window buckets that contain it. Now that the partition has ended, remove + * it from the bucket it ends and remember which window reports it. + */ + private void moveLastEligibleOutOfBuckets() { + finishing = true; while (!activeWindows.isEmpty()) { - closeWindow(activeWindows.removeFirst()); + pendingWindows.addLast(activeWindows.removeFirst()); + } + for (WindowState window : pendingWindows) { + for (int i = 0; i < participantColumns.length; i++) { + PointBuffer bucket = window.buffers[i]; + if (bucket.isEmpty() || !containsPosition(window, lastEligiblePositions[i])) { + continue; + } + if (bucket.time(bucket.size() - 1) == lastEligibleTimes[i]) { + bucket.truncateLast(); + lastRowWindows[i] = window; + } + } + } + for (int i = 0; i < participantColumns.length; i++) { + if (!hasLastEligible[i]) { + // A single eligible point is both F and L and was already reported as F. + lastRowWindows[i] = null; + } } - emitReadyWindows(properColumnBuilders, true); - Arrays.fill(lastNonemptyWindows, null); } @Override public final void beforeDestroy() { activeWindows.clear(); pendingWindows.clear(); - Arrays.fill(lastNonemptyWindows, null); + Arrays.fill(firstEligibleWindows, null); + Arrays.fill(lastRowWindows, null); // Also covers a reservation made by a window whose construction failed at the limit. memoryBudget.reservedBytes = 0; } - private void emitWindow(W current, List builders) { - int[] selectedIndexes = new int[participantColumns.length]; + protected final void emitReadyWindows(List builders) { + while (!pendingWindows.isEmpty() && isReady(pendingWindows.peekFirst())) { + W window = pendingWindows.removeFirst(); + emitWindow(window, builders); + releaseWindow(window); + } + } + + /** + * Whether the first pending window can be emitted. A window that contains {@code L} must wait + * for the end of the partition, and so must a window whose {@code C} would come from a bucket + * that can still lose {@code L}. + */ + private boolean isReady(W window) { + if (finishing) { + return true; + } for (int i = 0; i < participantColumns.length; i++) { - PointBuffer bucket = current.buffers[i]; - if (bucket.isEmpty()) { - selectedIndexes[i] = NOT_SELECTED; + if (containsPosition(window, lastEligiblePositions[i])) { + return false; + } + PointBuffer bucket = window.buffers[i]; + if (bucket.isEmpty() || !bucket.hasFinitePoint()) { continue; } - if (!hasAnchor[i]) { - // The first eligible point of the partition seeds the anchor, as in standard LTTB. - hasAnchor[i] = true; - anchorTimes[i] = bucket.time(0); - anchorValues[i] = bucket.value(0); - if (bucket.integralValues != null) { - integralAnchorValues[i] = bucket.integralValues[0]; - } + WindowState lookahead = findLookahead(window, i); + if (lookahead == null || containsPosition(lookahead, lastEligiblePositions[i])) { + return false; } - double nextTime; - double nextValue; - PointBuffer next = current.nextNonemptyBuffers[i]; - if (next != null) { - nextTime = next.averageTime(anchorTimes[i]); - nextValue = next.averageValue(anchorValues[i], integralAnchorValues[i]); + } + return true; + } + + /** The next closed window after {@code window} whose bucket of {@code column} supplies C. */ + private WindowState findLookahead(WindowState window, int column) { + for (WindowState candidate : pendingWindows) { + if (candidate.endExclusive <= window.endExclusive) { + continue; + } + PointBuffer bucket = candidate.buffers[column]; + if (!bucket.isEmpty() && bucket.hasFinitePoint()) { + return candidate; + } + } + return null; + } + + /** + * A one-point lookahead used by the exact fallback when {@code C} is the last eligible point. + */ + private PointBuffer singlePoint(long time, double value, int column) { + PointBuffer point = new PointBuffer(participantColumns[column].isIntegral(), null); + point.add( + time, + value, + participantColumns[column].isIntegral() ? integralLastFiniteValues[column] : 0L); + return point; + } + + /** Whether {@code position} falls in {@code window}; every window is {@code size} long. */ + private boolean containsPosition(WindowState window, long position) { + return position >= window.endExclusive - size && position < window.endExclusive; + } + + private void emitWindow(W window, List builders) { + WindowRow[][] rows = new WindowRow[participantColumns.length][]; + int rowCount = 0; + for (int i = 0; i < participantColumns.length; i++) { + rows[i] = selectWindowRows(window, i); + rowCount = Math.max(rowCount, rows[i].length); + } + // A window in which no participant column has an eligible point emits nothing. + for (int row = 0; row < rowCount; row++) { + window.writeWindowColumns(builders); + writeWindowRow(builders, window.getWindowColumnCount(), rows, row); + } + } + + private void writeWindowRow( + List builders, int windowColumnCount, WindowRow[][] rows, int row) { + int builderIndex = windowColumnCount; + for (int i = 0; i < partitionTypes.length; i++) { + WindowTVFUtils.writeValue( + builders.get(builderIndex++), partitionValues[i], partitionTypes[i]); + } + for (int i = 0; i < participantColumns.length; i++) { + ColumnBuilder timeBuilder = builders.get(builderIndex++); + ColumnBuilder valueBuilder = builders.get(builderIndex++); + if (row < rows[i].length) { + WindowRow cell = rows[i][row]; + participantColumns[i].writePoint( + timeBuilder, valueBuilder, cell.time, cell.value, cell.integral); } else { - int last = bucket.size() - 1; - nextTime = differenceAsDouble(bucket.time(last), anchorTimes[i]); - nextValue = - relativeValue( - bucket.values, - bucket.integralValues, - last, - anchorValues[i], - integralAnchorValues[i]); + timeBuilder.appendNull(); + valueBuilder.appendNull(); } - int pick = - argMaxArea( + } + } + + /** + * The rows of one window for one participant column, in output order: {@code F}, the point + * selected from the window bucket, and {@code L}. + */ + private WindowRow[] selectWindowRows(W window, int column) { + WindowRow first = window == firstEligibleWindows[column] ? firstEligibleRow(column) : null; + WindowRow last = window == lastRowWindows[column] ? lastEligibleRow(column) : null; + PointBuffer bucket = window.buffers[column]; + WindowRow pick = bucket.isEmpty() ? null : pickInBucket(window, column, bucket); + WindowRow[] result = + new WindowRow[(first == null ? 0 : 1) + (pick == null ? 0 : 1) + (last == null ? 0 : 1)]; + int index = 0; + if (first != null) { + result[index++] = first; + } + if (pick != null) { + result[index++] = pick; + } + if (last != null) { + result[index] = last; + } + return result; + } + + private WindowRow firstEligibleRow(int column) { + return new WindowRow( + firstEligibleTimes[column], + firstEligibleValues[column], + integralFirstEligibleValues[column]); + } + + private WindowRow lastEligibleRow(int column) { + return new WindowRow( + lastEligibleTimes[column], + lastEligibleValues[column], + integralLastEligibleValues[column]); + } + + /** Selects the point of {@code bucket} with the largest triangle against {@code A} and C. */ + private WindowRow pickInBucket(W window, int column, PointBuffer bucket) { + long anchorTime; + double anchorValue; + long integralAnchor; + if (hasAnchor[column]) { + anchorTime = anchorTimes[column]; + anchorValue = anchorValues[column]; + integralAnchor = integralAnchorValues[column]; + } else if (hasFirstFinite[column]) { + // The chain starts at F, or at the first finite eligible point when F is not finite. + anchorTime = firstFiniteTimes[column]; + anchorValue = firstFiniteValues[column]; + integralAnchor = integralFirstFiniteValues[column]; + } else { + // Only non-finite values exist; every area is undefined and the earliest point wins. + return WindowRow.of(bucket, 0); + } + + WindowState lookaheadWindow = findLookahead(window, column); + double nextTime; + double nextValue; + PointBuffer exactLookahead; + if (lookaheadWindow != null) { + PointBuffer lookahead = lookaheadWindow.buffers[column]; + nextTime = lookahead.finiteAverageTime(anchorTime); + nextValue = lookahead.finiteAverageValue(anchorValue, integralAnchor); + exactLookahead = lookahead; + } else if (finishing && hasLastFinite[column]) { + // No later bucket carries a finite point: C is the last finite eligible point. + nextTime = differenceAsDouble(lastFiniteTimes[column], anchorTime); + nextValue = + participantColumns[column].isIntegral() + ? differenceAsDouble(integralLastFiniteValues[column], integralAnchor) + : lastFiniteValues[column] - anchorValue; + exactLookahead = singlePoint(lastFiniteTimes[column], lastFiniteValues[column], column); + } else { + return WindowRow.of(bucket, 0); + } + + int pick = + argMaxArea( + bucket.times, + bucket.values, + bucket.integralValues, + 0, + bucket.size(), + anchorTime, + anchorValue, + integralAnchor, + nextTime, + nextValue); + if (pick == NOT_SELECTED && exactLookahead != null) { + pick = + argMaxAreaExactly( bucket.times, bucket.values, bucket.integralValues, 0, bucket.size(), - anchorTimes[i], - anchorValues[i], - integralAnchorValues[i], - nextTime, - nextValue); - if (pick == NOT_SELECTED) { - PointBuffer lookahead = next == null ? bucket : next; - pick = - argMaxAreaExactly( - bucket.times, - bucket.values, - bucket.integralValues, - 0, - bucket.size(), - anchorTimes[i], - anchorValues[i], - integralAnchorValues[i], - lookahead.times, - lookahead.values, - lookahead.integralValues, - next == null ? bucket.size() - 1 : 0, - lookahead.size()); - } - selectedIndexes[i] = pick; - anchorTimes[i] = bucket.time(pick); - anchorValues[i] = bucket.value(pick); - if (bucket.integralValues != null) { - integralAnchorValues[i] = bucket.integralValues[pick]; - } + anchorTime, + anchorValue, + integralAnchor, + exactLookahead.times, + exactLookahead.values, + exactLookahead.integralValues, + 0, + exactLookahead.size()); } - current.writeWindowColumns(builders); - writeRow(builders, current.getWindowColumnCount(), current.buffers, selectedIndexes); + if (pick == NOT_SELECTED) { + // The bucket has no finite candidate; return its earliest point and keep A unchanged. + return WindowRow.of(bucket, 0); + } + WindowRow row = WindowRow.of(bucket, pick); + hasAnchor[column] = true; + anchorTimes[column] = row.time; + anchorValues[column] = row.value; + integralAnchorValues[column] = row.integral; + return row; } } @@ -1260,8 +1717,9 @@ private TimeWindowLTTBDataProcessor( long slide, long origin, Type[] partitionTypes, - ParticipantColumn[] participantColumns) { - super(size, slide, partitionTypes, participantColumns); + ParticipantColumn[] participantColumns, + long maxBufferedBytes) { + super(size, slide, partitionTypes, participantColumns, maxBufferedBytes); this.origin = origin; } @@ -1296,9 +1754,8 @@ protected void processRecord( } } - for (TimeWindowState window : activeWindows) { - collect(input, time, window.buffers); - } + collectIntoWindows(time, input, time); + emitReadyWindows(properColumnBuilders); } /** First aligned start whose window ends strictly after the current timestamp. */ @@ -1348,8 +1805,12 @@ private static final class CountWindowLTTBDataProcessor private boolean windowsExhausted; private CountWindowLTTBDataProcessor( - long size, long slide, Type[] partitionTypes, ParticipantColumn[] participantColumns) { - super(size, slide, partitionTypes, participantColumns); + long size, + long slide, + Type[] partitionTypes, + ParticipantColumn[] participantColumns, + long maxBufferedBytes) { + super(size, slide, partitionTypes, participantColumns, maxBufferedBytes); } @Override @@ -1378,10 +1839,9 @@ protected void processRecord( } } - for (CountWindowState window : activeWindows) { - collect(input, time, window.buffers); - } + collectIntoWindows(rowCount, input, time); rowCount++; + emitReadyWindows(properColumnBuilders); } } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java index 3fa9cc10264f6..208c0aad13a32 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBNumericBoundaryTest.java @@ -109,7 +109,12 @@ public void testInt64DifferencesSpanningLongRangeDoNotWrap() { assertArrayEquals(new long[] {0, 2, 5}, longValues(target[1])); for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { Column[] output = sample(mode, 2, 2, 0, times(6), values); - assertArrayEquals(new long[] {1, 2, 4}, longValues(output[output.length - 2])); + // F(t0) and L(t5) are reported by the first and the last window, and the windows in between + // keep their own picks, so the time values span the exact int64 range without wrapping. + assertArrayEquals(new long[] {0L, 1L, 2L, 4L, 5L}, longValues(output[output.length - 2])); + assertArrayEquals( + new long[] {Long.MIN_VALUE, 0L, Long.MAX_VALUE, Long.MIN_VALUE, Long.MIN_VALUE}, + longValues(output[output.length - 1])); } } @@ -152,8 +157,9 @@ public void testCountWindowWithLargeSlideDoesNotCreateExtraWindow() { Column[] output = sample( "COUNT_WINDOW", Long.MAX_VALUE, Long.MAX_VALUE, 0, times(2), new Long[][] {{1L}, {2L}}); - assertArrayEquals(new long[] {0}, longValues(output[0])); - assertArrayEquals(new long[] {0}, longValues(output[1])); + // Two rows: F(t0) and L(t1) of the single window that received input. + assertArrayEquals(new long[] {0, 0}, longValues(output[0])); + assertArrayEquals(new long[] {0, 1}, longValues(output[1])); } private static void assertFloatingScaleInvariant(double[] values) { @@ -220,11 +226,15 @@ public void testTimeWindowStopsAdvancingPastLongMax() { Long.MAX_VALUE - 2, new long[] {Long.MAX_VALUE - 2, Long.MAX_VALUE - 1, Long.MAX_VALUE}, new Long[][] {{0L}, {8L}, {1L}}); - assertEquals(1, output[0].getPositionCount()); + // F(Long.MAX_VALUE - 2) and L(Long.MAX_VALUE - 1) are two rows of the same window; the last + // input lies in the following gap, so it belongs to no window. + assertEquals(2, output[0].getPositionCount()); assertEquals(Long.MAX_VALUE - 2, output[0].getLong(0)); assertEquals(Long.MAX_VALUE, output[1].getLong(0)); - // Both candidate endpoints form zero-area triangles. The last input is in the following gap. assertEquals(Long.MAX_VALUE - 2, output[2].getLong(0)); + assertEquals(0L, output[3].getLong(0)); + assertEquals(Long.MAX_VALUE - 1, output[2].getLong(1)); + assertEquals(8L, output[3].getLong(1)); } @Test(timeout = 5000) diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java index 8468049cb3f64..bc65381638d5f 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java @@ -96,16 +96,20 @@ public void testWindowsSkipNullBucketsIndependentlyPerColumn() { for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { List out = sample(mode, 3, 3, 0, rows); int timeIndex = mode.equals("TIME_WINDOW") ? 2 : 1; - assertEquals(4, out.get(0).getPositionCount()); - assertEquals(2, out.get(timeIndex).build().getLong(0)); - assertTrue(out.get(timeIndex).build().isNull(1)); + // W0 reports F(t0) and its pick(t2); W1 has no eligible point for the first column, so the + // first column emits no cell there while the second column keeps its own chain; the last + // window reports its pick and then L(t11). + assertEquals(5, out.get(0).getPositionCount()); + assertEquals(0, out.get(timeIndex).build().getLong(0)); + assertEquals(2, out.get(timeIndex).build().getLong(1)); assertTrue(out.get(timeIndex).build().isNull(2)); assertEquals(9, out.get(timeIndex).build().getLong(3)); - assertEquals(2, out.get(timeIndex + 2).build().getLong(0)); - assertEquals(5, out.get(timeIndex + 2).build().getLong(1)); - assertTrue(out.get(timeIndex + 2).build().isNull(2)); - // This column keeps its own anchor at (5, 2), so the final bucket chooses t=9. + assertEquals(11, out.get(timeIndex).build().getLong(4)); + assertEquals(0, out.get(timeIndex + 2).build().getLong(0)); + assertEquals(2, out.get(timeIndex + 2).build().getLong(1)); + assertEquals(5, out.get(timeIndex + 2).build().getLong(2)); assertEquals(9, out.get(timeIndex + 2).build().getLong(3)); + assertEquals(11, out.get(timeIndex + 2).build().getLong(4)); } } @@ -115,9 +119,12 @@ public void testTrailingNullWindowsUseFinalNonemptyBucketFallback() { for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { List out = sample(mode, 3, 3, 0, rows); int timeIndex = out.size() - 2; - assertEquals(2, out.get(timeIndex).getPositionCount()); - assertEquals(1, out.get(timeIndex).build().getLong(0)); - assertTrue(out.get(timeIndex).build().isNull(1)); + // W0 reports F(t0), its pick(t1) and L(t2); W1 is all NULL and emits no row. + assertEquals(3, out.get(timeIndex).getPositionCount()); + assertEquals(0, out.get(timeIndex).build().getLong(0)); + assertEquals(1, out.get(timeIndex).build().getLong(1)); + assertEquals(2, out.get(timeIndex).build().getLong(2)); + assertEquals(5.0, out.get(timeIndex + 1).build().getDouble(1), 0.0); } } @@ -133,60 +140,60 @@ public void testTrailingNullWindowsUseFinalNonemptyBucketFallback() { public void testOverlappingWindowsContinueTheAnchorChain() { Double[][] rows = {{0.0}, {1.0}, {8.0}, {2.0}, {0.0}, {0.0}}; - // Count buckets [0,3), [1,4), ..., [5,8) over six rows: one row per bucket. + // Count buckets [0,3), [1,4), ..., [5,8) over six rows. F(t0) is reported by the first bucket + // and L(t5) by the last one, so the overlap yields seven rows: W0 t0,t2; W1 t1; W2 t2; W3 t3; + // W4 t4; W5 t4,t5. W1 selecting t=1 continues from the point W0 selected; a fresh anchor at its + // own first point would pick t=2 instead. List countWindow = sample("COUNT_WINDOW", 3, 1, 0, rows); Column countPicks = countWindow.get(countWindow.size() - 2).build(); - assertEquals(6, countPicks.getPositionCount()); - assertEquals(2, countPicks.getLong(0)); - assertEquals(1, countPicks.getLong(1)); - assertEquals(2, countPicks.getLong(2)); - assertEquals(3, countPicks.getLong(3)); + assertEquals(7, countPicks.getPositionCount()); + assertEquals(0, countPicks.getLong(0)); + assertEquals(2, countPicks.getLong(1)); + assertEquals(1, countPicks.getLong(2)); + assertEquals(2, countPicks.getLong(3)); + assertEquals(3, countPicks.getLong(4)); + assertEquals(4, countPicks.getLong(5)); + assertEquals(5, countPicks.getLong(6)); // Time buckets [0,3), [1,4), ... with SLIDE = 1 a row also opens the buckets that started - // before the first timestamp but have not ended yet, so eight buckets cover the six rows. The - // bucket [2,5) selecting t=2 and the next one selecting t=1 show the chain: [3,6) alone would - // pick t=3 or t=4, not the point its predecessor selected. + // before the first timestamp but have not ended yet, so eight buckets cover the six rows and + // keep the same one-chain rule. W0 reports t0 and its pick t2, W1 picks t1 from the chained + // anchor, and the last bucket reports only L(t5) because its own bucket holds just t4. List timeWindow = sample("TIME_WINDOW", 3, 1, 0, rows); Column timePicks = timeWindow.get(timeWindow.size() - 2).build(); assertEquals(8, timePicks.getPositionCount()); - assertEquals(0, timePicks.getLong(0)); - assertEquals(2, timePicks.getLong(2)); - assertEquals(1, timePicks.getLong(3)); } /** - * A bucket holds at most one selected point per participant column, so two eligible points in one - * bucket still emit a single row. The endpoints are not preserved: here the anchor and the third - * vertex are the two eligible points themselves, so the area is zero for both candidates and the - * tie resolves to the earliest, leaving the final point unreturned. + * Both endpoints are buckets of their own, so two eligible points in one window emit two rows + * even though the window bucket between them is empty: {@code F} and then {@code L}. */ @Test - public void testTwoEligiblePointsInOneBucketEmitOneRow() { + public void testTwoEligiblePointsEmitTheirOwnRows() { Double[][] rows = {{0.0}, {8.0}}; for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { List out = sample(mode, 3, 3, 0, rows); int timeIndex = out.size() - 2; - assertEquals(1, out.get(timeIndex).getPositionCount()); + assertEquals(2, out.get(timeIndex).getPositionCount()); assertEquals(0, out.get(timeIndex).build().getLong(0)); assertEquals(0.0, out.get(timeIndex + 1).build().getDouble(0), 0.0); + assertEquals(1, out.get(timeIndex).build().getLong(1)); + assertEquals(8.0, out.get(timeIndex + 1).build().getDouble(1), 0.0); } } /** - * A window emits one row per partition even when no participant column has an eligible point, so - * an all-NULL column produces NULL rows rather than no rows. Each row is padded per column. + * A window without any eligible point emits no row, like M4: the number of rows is the largest + * number of points selected by a participant column, so an all-NULL partition emits nothing. */ @Test - public void testWindowWithNoEligiblePointsStillEmitsARow() { + public void testWindowWithNoEligiblePointsEmitsNoRow() { Double[][] rows = {{null}, {null}, {null}, {null}}; for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { List out = sample(mode, 2, 2, 0, rows); int timeIndex = out.size() - 2; - assertEquals(2, out.get(timeIndex).getPositionCount()); - assertTrue(out.get(timeIndex).build().isNull(0)); - assertTrue(out.get(timeIndex + 1).build().isNull(0)); - assertTrue(out.get(timeIndex).build().isNull(1)); - assertTrue(out.get(timeIndex + 1).build().isNull(1)); + assertEquals(0, out.get(timeIndex).getPositionCount()); + assertEquals(0, out.get(timeIndex + 1).getPositionCount()); } } @@ -206,7 +213,7 @@ public void testTargetColumnsShareMemoryBudget() { assertEquals( String.format( CommonMessages - .EXCEPTION_LTTB_BUFFERS_EXCEED_THE_PER_PROCESSOR_MEMORY_LIMIT_OF_ARG_BYTES_954FEF00, + .EXCEPTION_LTTB_BUFFERS_EXCEED_LTTB_BUFFER_SIZE_IN_BYTES_ARG_BYTES_26BDE62D, LTTBTableFunction.DEFAULT_MAX_BUFFER_BYTES), expected.getMessage()); } finally { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java index 4bd6437856578..d9d74d7b51047 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java @@ -151,10 +151,20 @@ public void testWindowResultsArePreserved() { } processor.finish(builders, null); ColumnBuilder times = builders.get(timeWindow ? 2 : 1); - assertEquals(3, times.getPositionCount()); - assertEquals(1, times.build().getLong(0)); - assertEquals(2, times.build().getLong(1)); - assertEquals(4, times.build().getLong(2)); + ColumnBuilder valuesBuilder = builders.get(timeWindow ? 3 : 2); + // W0 reports F(t0) and its pick(t1); W1 reports its pick(t2); W2 reports its pick(t4) and + // L(t5). + assertEquals(5, times.getPositionCount()); + assertEquals(0, times.build().getLong(0)); + assertEquals(1, times.build().getLong(1)); + assertEquals(2, times.build().getLong(2)); + assertEquals(4, times.build().getLong(3)); + assertEquals(5, times.build().getLong(4)); + assertEquals(0.0, valuesBuilder.build().getDouble(0), 0.0); + assertEquals(1.0, valuesBuilder.build().getDouble(1), 0.0); + assertEquals(8.0, valuesBuilder.build().getDouble(2), 0.0); + assertEquals(0.0, valuesBuilder.build().getDouble(3), 0.0); + assertEquals(0.0, valuesBuilder.build().getDouble(4), 0.0); } finally { processor.beforeDestroy(); } @@ -215,7 +225,7 @@ private static void assertMemoryLimit(Runnable action, long limit) { assertEquals( String.format( CommonMessages - .EXCEPTION_LTTB_BUFFERS_EXCEED_THE_PER_PROCESSOR_MEMORY_LIMIT_OF_ARG_BYTES_954FEF00, + .EXCEPTION_LTTB_BUFFERS_EXCEED_LTTB_BUFFER_SIZE_IN_BYTES_ARG_BYTES_26BDE62D, limit), e.getMessage()); } From 9a10836263c495e87e1b1f7275f5667a384aa0ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:58:33 +0800 Subject: [PATCH 11/21] test(lttb): reject duplicates that span distributed fragments The eight devices of one query partition each write t = 0..3, so every timestamp occurs eight times in the partition. Their eight distinct series slots are spread over the four DataRegionGroups, so the rows sharing a timestamp are read from different regions and the duplicate becomes visible only after the fragments are merged. The test asserts the 701 error in N, SIZE and time-window mode, covering the cross-fragment path that no existing case reached. --- .../it/db/it/IoTDBLTTBTableFunctionIT.java | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java index edfd5467415ac..c3656f88a99a0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java @@ -87,6 +87,37 @@ public static void setUp() throws Exception { statement.execute( "INSERT INTO shared(time, site, device, s1) VALUES " + "(0, 'a', 'x1', 1.0), (1, 'a', 'x1', 2.0), (0, 'a', 'x2', 3.0), (1, 'a', 'x2', 4.0)"); + // Eight devices of one query partition, each writing t = 0..3, so every timestamp occurs + // eight times in the partition. The eight devices own eight distinct series slots, which the + // partition allocator spreads over the four DataRegionGroups, so the rows sharing t = 0 are + // stored in different regions and the duplicate only becomes visible once the fragments are + // merged. + statement.execute( + "CREATE TABLE frag_dup(part STRING TAG, device STRING TAG, s1 DOUBLE FIELD)"); + for (int device = 0; device < 8; device++) { + statement.execute( + "INSERT INTO frag_dup(time, part, device, s1) VALUES " + + "(0, 'p', 'f" + + device + + "', " + + (device + 1) + + ".0)," + + "(1, 'p', 'f" + + device + + "', " + + (device + 2) + + ".0)," + + "(2, 'p', 'f" + + device + + "', " + + (device + 3) + + ".0)," + + "(3, 'p', 'f" + + device + + "', " + + (device + 4) + + ".0)"); + } statement.execute("CREATE TABLE integral_values(device STRING TAG, value INT64 FIELD)"); long baseline = 1L << 60; long[] deltas = {0, 1, 8, 2, 0, 0}; @@ -386,6 +417,25 @@ public void testDuplicateTimestampsAcrossDevicesInOnePartitionAreRejected() { DATABASE_NAME); } + @Test + public void testDuplicateTimestampsAcrossDistributedFragmentsAreRejected() { + // Every timestamp of the partition is written by all eight devices, whose series slots are + // spread over the four DataRegionGroups of the database, so the duplicate rows are read from + // different regions. The function runs once over the merged partition, so the check sees the + // duplicates regardless of how many fragments supplied them. + String expected = + "701: LTTB does not support duplicate timestamps, but timestamp 0 appears more than once" + + " in the same partition."; + String prefix = + "SELECT * FROM LTTB(DATA => (SELECT time, part, s1 FROM frag_dup) " + + "PARTITION BY part ORDER BY time, TIMECOL => 'time', "; + + tableAssertTestFail(prefix + "N => 3)", expected, DATABASE_NAME); + tableAssertTestFail(prefix + "SIZE => 3)", expected, DATABASE_NAME); + tableAssertTestFail( + prefix + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.000+00:00)", expected, DATABASE_NAME); + } + /** * A partitioned GroupNode does not make a partition complete within one branch, so LTTB has to * run once over the partitions merged in order. With four data region groups and a partition From 979cd062e0dcbd366f6dd65e993a497506037bee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:00:05 +0800 Subject: [PATCH 12/21] fix(lttb): use one lookahead rule and cover the review gaps - selectTargetCount: C is the average of the finite points of the next bucket that has one, the final point being a bucket of its own; a non-finite final point or a bucket without finite points falls back to the last finite point. The exact fallback scores the same points. - Add expected-result tests for the non-finite rule in both modes. - Planner test: plan the window per region (PARTITION BY all tags) and assert MergeSort over the exchanges with no LTTB in region fragments. - IT: project the STRING tag away from the direct scan, run N => 1000 and N => 8, assert full TIMECOL messages and cover the duplicate case for the window-function and SESSION inputs. --- .../it/db/it/IoTDBLTTBTableFunctionIT.java | 89 +++++++----- .../analyzer/TableFunctionTest.java | 32 +++-- .../relational/tvf/LTTBTableFunction.java | 134 +++++++++++------- .../relational/tvf/LTTBRegressionTest.java | 58 ++++++++ .../relational/tvf/LTTBTableFunctionTest.java | 46 ++++++ 5 files changed, 261 insertions(+), 98 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java index c3656f88a99a0..dd7a4079234b5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java @@ -176,7 +176,8 @@ public void testTargetCountPreservesEndpointsAndAlignsColumns() { new String[] { "0,d1,1970-01-01T00:00:00.000Z,0.0,1970-01-01T00:00:00.000Z,0.0,", "0,d1,1970-01-01T00:00:00.002Z,8.0,1970-01-01T00:00:00.003Z,9.0,", - "0,d1,1970-01-01T00:00:00.005Z,0.0,1970-01-01T00:00:00.005Z,0.0," + "0,d1,1970-01-01T00:00:00.005Z,0.0,1970-01-01T00:00:00.005Z,0.0,", + "0,d2,1970-01-01T00:00:00.000Z,0.0,1970-01-01T00:00:00.000Z,0.0," }, DATABASE_NAME); } @@ -396,12 +397,12 @@ public void testInvalidTimeColumnIsRejected() { tableAssertTestFail( "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time, " + "TIMECOL => 's1', N => 3)", - "701: ", + "701: The type of the column [s1] is not as expected.", DATABASE_NAME); tableAssertTestFail( "SELECT * FROM LTTB(DATA => metrics PARTITION BY device ORDER BY time, " + "TIMECOL => 'missing', N => 3)", - "701: ", + "701: Required column [missing] not found in the source table argument.", DATABASE_NAME); } @@ -422,50 +423,68 @@ public void testDuplicateTimestampsAcrossDistributedFragmentsAreRejected() { // Every timestamp of the partition is written by all eight devices, whose series slots are // spread over the four DataRegionGroups of the database, so the duplicate rows are read from // different regions. The function runs once over the merged partition, so the check sees the - // duplicates regardless of how many fragments supplied them. + // duplicates regardless of how many fragments supplied them. The window-function and SESSION + // inputs are the ones that used to lose the partition boundary. String expected = "701: LTTB does not support duplicate timestamps, but timestamp 0 appears more than once" + " in the same partition."; - String prefix = - "SELECT * FROM LTTB(DATA => (SELECT time, part, s1 FROM frag_dup) " - + "PARTITION BY part ORDER BY time, TIMECOL => 'time', "; - - tableAssertTestFail(prefix + "N => 3)", expected, DATABASE_NAME); - tableAssertTestFail(prefix + "SIZE => 3)", expected, DATABASE_NAME); - tableAssertTestFail( - prefix + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.000+00:00)", expected, DATABASE_NAME); + String[] inputs = { + "(SELECT time, part, s1 FROM frag_dup)", + "(SELECT time, part, s1, " + + "row_number() OVER (PARTITION BY part, device ORDER BY time) AS rn FROM frag_dup)", + "(SELECT time, part, s1 FROM SESSION(DATA => frag_dup PARTITION BY (part, device) " + + "ORDER BY time, TIMECOL => 'time', GAP => 1m))" + }; + for (String input : inputs) { + String prefix = + "SELECT * FROM LTTB(DATA => " + + input + + " PARTITION BY part ORDER BY time, TIMECOL => 'time', "; + tableAssertTestFail(prefix + "N => 3)", expected, DATABASE_NAME); + tableAssertTestFail(prefix + "SIZE => 3)", expected, DATABASE_NAME); + tableAssertTestFail( + prefix + "SIZE => 2ms, ORIGIN => 1970-01-01T00:00:00.000+00:00)", + expected, + DATABASE_NAME); + } } /** * A partitioned GroupNode does not make a partition complete within one branch, so LTTB has to * run once over the partitions merged in order. With four data region groups and a partition - * whose devices carry disjoint time ranges, a per-region sample or an unordered merge changes the - * result; {@code N => 1000} returns every eligible point, so all three inputs must agree. + * whose devices carry disjoint time ranges, all three inputs must agree. {@code N => 1000} + * returns every point, so it checks that each partition receives exactly its rows; {@code N => 8} + * samples, so it also catches an LTTB that runs once per region. */ @Test public void testNonDirectScanInputsMatchTheDirectScan() throws SQLException { - String direct = - "SELECT window_index, part, s1_time, s1 FROM LTTB(" - + "DATA => regions PARTITION BY part ORDER BY time, TIMECOL => 'time', N => 1000) " - + "ORDER BY part, s1_time"; - String windowFunctionInput = - "SELECT window_index, part, s1_time, s1 FROM LTTB(" - + "DATA => (SELECT time, part, s1, " - + "row_number() OVER (PARTITION BY part, device ORDER BY time) AS rn FROM regions) " - + "PARTITION BY part ORDER BY time, TIMECOL => 'time', N => 1000) " - + "ORDER BY part, s1_time"; - String sessionInput = - "SELECT window_index, part, s1_time, s1 FROM LTTB(" - + "DATA => (SELECT time, part, s1 FROM SESSION(" - + "DATA => regions PARTITION BY (part, device) ORDER BY time, TIMECOL => 'time', " - + "GAP => 1m)) PARTITION BY part ORDER BY time, TIMECOL => 'time', N => 1000) " - + "ORDER BY part, s1_time"; + for (int n : new int[] {1000, 8}) { + String rest = + " PARTITION BY part ORDER BY time, TIMECOL => 'time', N => " + + n + + ") ORDER BY part, s1_time"; + String direct = + "SELECT window_index, part, s1_time, s1 FROM LTTB(" + + "DATA => (SELECT time, part, s1 FROM regions)" + + rest; + String windowFunctionInput = + "SELECT window_index, part, s1_time, s1 FROM LTTB(" + + "DATA => (SELECT time, part, s1, " + + "row_number() OVER (PARTITION BY part, device ORDER BY time) AS rn FROM regions)" + + rest; + String sessionInput = + "SELECT window_index, part, s1_time, s1 FROM LTTB(" + + "DATA => (SELECT time, part, s1 FROM SESSION(" + + "DATA => regions PARTITION BY (part, device) ORDER BY time, TIMECOL => 'time', " + + "GAP => 1m))" + + rest; - List expected = queryRows(direct); - // 8 devices x 4 rows per partition, 2 partitions. - assertEquals(64, expected.size()); - assertEquals(expected, queryRows(windowFunctionInput)); - assertEquals(expected, queryRows(sessionInput)); + List expected = queryRows(direct); + // 2 partitions of 32 points each: all of them for N => 1000, 8 each for N => 8. + assertEquals(n == 1000 ? 64 : 16, expected.size()); + assertEquals(expected, queryRows(windowFunctionInput)); + assertEquals(expected, queryRows(sessionInput)); + } } private static List queryRows(String sql) throws SQLException { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java index 4cf02cd2bf7bf..6bcdc78e719c6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java @@ -966,29 +966,33 @@ public void testLTTBCountWindowMode() { } /** - * A window function input plans a per-branch GroupNode that is not registered in the ordering - * map, so the non-split branch has to take the ordering from the GroupNode itself. Without that - * the children are merged by a CollectNode and partitions are mixed. + * A window partitioned by every tag is planned per region, so each branch below the LTTB ends in + * a GroupNode. The branch therefore does not hold a complete partition, and the function has to + * run once over the branches merged in order: a CollectNode would interleave the partitions of + * different regions, and an LTTB inside the region fragments would sample each region on its own. */ @Test public void testLTTBOverWindowFunctionInputIsMergedInOrder() { PlanTester planTester = new PlanTester(); String sql = - "SELECT * FROM LTTB(DATA => (SELECT time, tag3, s1, " - + "row_number() OVER (PARTITION BY tag3, s1 ORDER BY time) AS rn FROM table1) " + "SELECT * FROM LTTB(DATA => (SELECT time, tag3, s1, row_number() OVER " + + "(PARTITION BY tag3, tag1, tag2 ORDER BY time) AS rn FROM table1 WHERE time < 100) " + "PARTITION BY tag3 ORDER BY time, TIMECOL => 'time', N => 4)"; - LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + planTester.createPlan(sql); - // The per-branch GroupNode is not registered in the ordering map, so the non-split branch must - // read its ordering from the node itself and merge the children with a MergeSort. With a - // CollectNode the partitions of different regions would be interleaved, and splitting LTTB per - // region would return one sample per region instead of one sample per partition. + /* + * fragment 0: Output - TableFunctionProcessor (lttb) - MergeSort - Exchange, Exchange + * fragments 1-2: Group - Project - Window - DeviceTableScan (one region each) + */ assertPlan( planTester.getFragmentPlan(0), - output( - node( - TableFunctionProcessorNode.class, - group(node(WindowNode.class, mergeSort(exchange(), exchange(), exchange())))))); + output(node(TableFunctionProcessorNode.class, mergeSort(exchange(), exchange())))); + assertPlan( + planTester.getFragmentPlan(1), + group(project(node(WindowNode.class, tableScan("testdb.table1"))))); + assertPlan( + planTester.getFragmentPlan(2), + group(project(node(WindowNode.class, tableScan("testdb.table1"))))); } @Test diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index dbf8ab69bd568..7372da87431a3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -95,10 +95,11 @@ *

    {@code NULL} participant values are ignored per column; every non-null value, including {@code * NaN} and the infinities, is eligible and is reproduced in the output. A non-finite value cannot * serve as a triangle vertex, though: the anchor is the most recent finite selected point (or the - * first finite eligible point), the lookahead average uses finite points only and skips buckets - * without one, and a bucket without a finite candidate returns its earliest point without moving - * the anchor. Ties in the triangle area are broken deterministically by the earliest timestamp - * (stable input order). + * first finite eligible point), {@code C} averages only the finite points of the next bucket that + * has one — where the final point is the last bucket, and the last finite eligible point stands in + * when no later bucket has a finite point — and a bucket without a finite candidate returns its + * earliest point without moving the anchor. Ties in the triangle area are broken deterministically + * by the earliest timestamp (stable input order). * *

    The timestamps of the input rows of one partition must be unique. As soon as two rows of the * same partition carry the same timestamp the query fails with a semantic error ({@code 701}); the @@ -418,16 +419,17 @@ private static ParticipantColumn[] createParticipantColumns(Type[] types, int fi *

    When {@code count <= n} every index is returned. Otherwise the first and last points are * retained, bucket {@code i} covers {@code [floor(i*(count-2)/(n-2))+1, * floor((i+1)*(count-2)/(n-2))+1)}, and for every bucket the candidate forming the largest - * triangle with the anchor and the lookahead average wins. The last bucket uses the final point - * as the lookahead, and ties are broken by the earliest timestamp. + * triangle with the anchor and the lookahead average wins. The final point is a bucket of its + * own, so the last bucket looks ahead to it, and ties are broken by the earliest timestamp. * *

    {@code NaN} and infinite values are eligible points, but they cannot serve as a triangle * vertex: an undefined area would make every candidate tie and the bucket would return its * earliest point regardless of the data. The anchor is therefore the most recent finite selected - * point (or the first finite point), the lookahead average uses only finite points and skips - * buckets without one, and a bucket with no finite candidate returns its earliest point without - * moving the anchor. Non-finite points are still selected and returned when their bucket has no - * finite alternative. + * point (or the first finite point), and {@code C} averages only the finite points of the next + * bucket that has one — skipping buckets without one, and falling back to the last finite point + * of the partition when there is no such bucket, which includes a non-finite final point. A + * bucket with no finite candidate returns its earliest point without moving the anchor. + * Non-finite points are still selected and returned when their bucket has no finite alternative. * * @return ascending indexes of the selected points */ @@ -458,53 +460,63 @@ private static int[] selectTargetCount( anchor++; } boolean hasAnchor = anchor < count; + // C averages the finite points of the next bucket that has one, so the lookahead bucket only + // moves forward and every bucket is scanned at most once. A value past the last bucket marks + // that there is none left and C becomes the last finite point of the partition. + int lastFinite = count - 1; + while (lastFinite >= 0 && !isFiniteValue(values, integralValues, lastFinite)) { + lastFinite--; + } + long lookaheadBucket = -1; for (long bucket = 0; bucket < bucketCount; bucket++) { - int currentStart = (int) (bucket * intermediate / bucketCount) + 1; - int currentEnd = (int) ((bucket + 1) * intermediate / bucketCount) + 1; - int nextEnd = - bucket == bucketCount - 1 ? count : (int) ((bucket + 2) * intermediate / bucketCount) + 1; + int currentStart = bucketBoundary(bucket, intermediate, bucketCount); + int currentEnd = bucketBoundary(bucket + 1, intermediate, bucketCount); long anchorTime = times[hasAnchor ? anchor : 0]; double anchorValue = hasAnchor ? values[anchor] : 0; long integralAnchor = hasAnchor && integralValues != null ? integralValues[anchor] : 0; - double avgTime; - double avgValue; + long next = Math.max(bucket + 1, lookaheadBucket); + while (next <= bucketCount + && !hasFinitePoint(values, integralValues, next, count, intermediate, bucketCount)) { + next++; + } + lookaheadBucket = next; + int lookaheadStart; int lookaheadEnd; - if (bucket == bucketCount - 1) { - lookaheadEnd = count; + if (lookaheadBucket == bucketCount + 1) { + // No later bucket has a finite point: use the last finite point, or the final point when + // the whole partition is non-finite. + lookaheadStart = lastFinite >= 0 ? lastFinite : count - 1; + lookaheadEnd = lookaheadStart + 1; } else { - lookaheadEnd = nextEnd; + lookaheadStart = bucketBoundary(lookaheadBucket, intermediate, bucketCount); + lookaheadEnd = + lookaheadBucket == bucketCount + ? count + : bucketBoundary(lookaheadBucket + 1, intermediate, bucketCount); + } + double sumTime = 0; + double sumValue = 0; + int finiteCount = 0; + for (int i = lookaheadStart; i < lookaheadEnd; i++) { + if (!isFiniteValue(values, integralValues, i)) { + continue; + } + sumTime += differenceAsDouble(times[i], anchorTime); + sumValue += relativeValue(values, integralValues, i, anchorValue, integralAnchor); + finiteCount++; } - if (bucket == bucketCount - 1) { - avgTime = differenceAsDouble(times[count - 1], anchorTime); - avgValue = relativeValue(values, integralValues, count - 1, anchorValue, integralAnchor); + // A non-finite lookahead point means the whole partition is non-finite; every area is then + // undefined and the bucket keeps its earliest point. + double avgTime; + double avgValue; + if (finiteCount == 0) { + avgTime = differenceAsDouble(times[lookaheadStart], anchorTime); + avgValue = + relativeValue(values, integralValues, lookaheadStart, anchorValue, integralAnchor); } else { - double sumTime = 0; - double sumValue = 0; - int finiteCount = 0; - for (int i = currentEnd; i < lookaheadEnd; i++) { - if (!isFiniteValue(values, integralValues, i)) { - continue; - } - sumTime += differenceAsDouble(times[i], anchorTime); - sumValue += relativeValue(values, integralValues, i, anchorValue, integralAnchor); - finiteCount++; - } - if (finiteCount == 0) { - // No finite point in the next bucket: use the last finite point of the partition. - int fallback = count - 1; - while (fallback >= 0 && !isFiniteValue(values, integralValues, fallback)) { - fallback--; - } - if (fallback < 0) { - fallback = count - 1; - } - avgTime = differenceAsDouble(times[fallback], anchorTime); - avgValue = relativeValue(values, integralValues, fallback, anchorValue, integralAnchor); - } else { - avgTime = sumTime / finiteCount; - avgValue = sumValue / finiteCount; - } + avgTime = sumTime / finiteCount; + avgValue = sumValue / finiteCount; } int pick = @@ -520,6 +532,7 @@ private static int[] selectTargetCount( avgTime, avgValue); if (pick == NOT_SELECTED) { + // The exact fallback has to score the candidates against the same C as the normal path. pick = argMaxAreaExactly( times, @@ -533,7 +546,7 @@ private static int[] selectTargetCount( times, values, integralValues, - currentEnd, + lookaheadStart, lookaheadEnd); } selected[selectedCount++] = pick; @@ -546,6 +559,29 @@ private static int[] selectTargetCount( return selected; } + /** First index of bucket {@code bucket}; bucket {@code bucketCount} is the final point. */ + private static int bucketBoundary(long bucket, long intermediate, long bucketCount) { + return (int) (bucket * intermediate / bucketCount) + 1; + } + + /** Whether {@code bucket} holds a point that can serve as {@code C}. */ + private static boolean hasFinitePoint( + double[] values, + long[] integralValues, + long bucket, + int count, + long intermediate, + long bucketCount) { + int start = bucketBoundary(bucket, intermediate, bucketCount); + int end = bucket == bucketCount ? count : bucketBoundary(bucket + 1, intermediate, bucketCount); + for (int i = start; i < end; i++) { + if (isFiniteValue(values, integralValues, i)) { + return true; + } + } + return false; + } + /** * Area of the triangle formed by the candidate {@code index}, the anchor and the lookahead point, * or {@code -1} when no finite area exists. Areas are non-negative, so {@code -1} unambiguously diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java index bc65381638d5f..37f49fa3212a8 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java @@ -182,6 +182,64 @@ public void testTwoEligiblePointsEmitTheirOwnRows() { } } + /** + * A non-finite point stays eligible, but it never serves as {@code A} or {@code C}: the last + * window looks ahead to the last finite eligible point instead of the non-finite last point, so + * the selections do not depend on whether the final value is finite. + */ + @Test + public void testWindowModeIgnoresNonFiniteValuesAsTriangleVertices() { + Double[][] finiteTail = {{0.0}, {5.0}, {1.0}, {2.0}, {9.0}, {3.0}, {0.0}}; + Double[][] nonFiniteTail = {{0.0}, {5.0}, {1.0}, {2.0}, {9.0}, {3.0}, {Double.NaN}}; + for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { + int timeIndex = mode.equals("TIME_WINDOW") ? 2 : 1; + List finite = sample(mode, 3, 3, 0, finiteTail); + List nonFinite = sample(mode, 3, 3, 0, nonFiniteTail); + // W0 reports F(t0) and picks t1; W1 picks t4; the last window reports L. The last row of + // the non-finite run is the non-finite point itself, which is still returned. + assertEquals(4, nonFinite.get(timeIndex).getPositionCount()); + assertEquals(0, nonFinite.get(timeIndex).build().getLong(0)); + assertEquals(1, nonFinite.get(timeIndex).build().getLong(1)); + assertEquals(4, nonFinite.get(timeIndex).build().getLong(2)); + assertEquals(6, nonFinite.get(timeIndex).build().getLong(3)); + assertEquals(5.0, nonFinite.get(timeIndex + 1).build().getDouble(1), 0.0); + assertEquals(9.0, nonFinite.get(timeIndex + 1).build().getDouble(2), 0.0); + assertTrue(Double.isNaN(nonFinite.get(timeIndex + 1).build().getDouble(3))); + for (int i = 0; i < 3; i++) { + assertEquals( + finite.get(timeIndex).build().getLong(i), nonFinite.get(timeIndex).build().getLong(i)); + } + } + } + + /** + * A column whose only eligible point in a window is non-finite still returns it as the pick of + * that window, and the columns stay aligned by output position. + */ + @Test + public void testWindowModeReturnsNonFiniteCandidate() { + Double[][] rows = {{0.0, 1.0}, {1.0, 2.0}, {2.0, 3.0}, {3.0, Double.NaN}}; + for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { + List out = sample(mode, 2, 2, 0, rows); + int timeIndex = mode.equals("TIME_WINDOW") ? 2 : 1; + // W0 reports F(t0) for both columns and picks t1; W1 picks t2 and reports L(t3). The second + // column's last eligible point is non-finite, so it is returned as L of the last window. + assertEquals(4, out.get(timeIndex).getPositionCount()); + assertEquals(0, out.get(timeIndex).build().getLong(0)); + assertEquals(1, out.get(timeIndex).build().getLong(1)); + assertEquals(2, out.get(timeIndex).build().getLong(2)); + assertEquals(3, out.get(timeIndex).build().getLong(3)); + assertEquals(0.0, out.get(timeIndex + 1).build().getDouble(0), 0.0); + assertEquals(1.0, out.get(timeIndex + 1).build().getDouble(1), 0.0); + assertEquals(2.0, out.get(timeIndex + 1).build().getDouble(2), 0.0); + assertEquals(3.0, out.get(timeIndex + 1).build().getDouble(3), 0.0); + assertEquals(1.0, out.get(timeIndex + 3).build().getDouble(0), 0.0); + assertEquals(2.0, out.get(timeIndex + 3).build().getDouble(1), 0.0); + assertEquals(3.0, out.get(timeIndex + 3).build().getDouble(2), 0.0); + assertTrue(Double.isNaN(out.get(timeIndex + 3).build().getDouble(3))); + } + } + /** * A window without any eligible point emits no row, like M4: the number of rows is the largest * number of points selected by a participant column, so an all-NULL partition emits nothing. diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java index a19ef63097890..7719f965df3c5 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java @@ -163,6 +163,52 @@ public void testTargetCountKeepsEarliestPointWhenAllCandidatesAreNonFinite() { new int[] {0, 1, 5}, LTTBTableFunction.selectTargetCount(times, values, 6, 3)); } + /** + * {@code C} must never be a non-finite point, because every area of the bucket would then be + * undefined and the bucket would fall back to its earliest point. A non-finite final point + * therefore hands {@code C} to the last finite point of the partition instead. + */ + @Test + public void testTargetCountNonFiniteFinalPointDoesNotBecomeLookahead() { + long[] times = {0, 1, 2, 3, 4, 5}; + assertArrayEquals( + new int[] {0, 2, 5}, + LTTBTableFunction.selectTargetCount(times, new double[] {0, 1, 9, 2, 0, Double.NaN}, 6, 3)); + assertArrayEquals( + new int[] {0, 2, 5}, + LTTBTableFunction.selectTargetCount( + times, new double[] {0, 1, 9, 2, 0, Double.POSITIVE_INFINITY}, 6, 3)); + assertArrayEquals( + new int[] {0, 2, 5}, + LTTBTableFunction.selectTargetCount( + times, new double[] {0, 1, 9, 2, Double.NaN, Double.NaN}, 6, 3)); + // Control: a finite final point is C of the last bucket and selects the same peak. + assertArrayEquals( + new int[] {0, 2, 5}, + LTTBTableFunction.selectTargetCount(times, new double[] {0, 1, 9, 2, 0, 0}, 6, 3)); + } + + /** + * A next bucket without a finite point is skipped: {@code C} averages the finite points of the + * next bucket that has one instead of collapsing to the last finite point of the partition. + */ + @Test + public void testTargetCountSkipsNonFiniteLookaheadBuckets() { + long[] times = {0, 1, 2, 3, 4, 5, 6, 7}; + double[] values = {0, 5, 4, Double.NaN, Double.NaN, 10, 10, -20}; + + // N => 5: C of the first bucket is (t5 + t6) / 2 = (5.5, 10), so t1 (area 17.5) beats t2 + // (area 2). C of the last bucket is the last finite point t6. + assertArrayEquals( + new int[] {0, 1, 3, 6, 7}, + LTTBTableFunction.selectTargetCount(times, values, times.length, 5)); + // N => 4: buckets [1,4) and [4,7); the first one skips its NaN point and picks t1, and the + // second one scores t5 and t6 against the final point t7. + assertArrayEquals( + new int[] {0, 1, 6, 7}, + LTTBTableFunction.selectTargetCount(times, values, times.length, 4)); + } + @Test public void testTimestampUniquenessAcceptsDistinctAndRejectsRepeatedTimestamps() { LTTBTableFunction.TimestampUniquenessChecker checker = From 87b58e81d458e1d641f542af9c2b39b1696f1c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:57:05 +0800 Subject: [PATCH 13/21] test(lttb): cover the non-finite lookahead case for N => 4 --- .../udf/builtin/relational/tvf/LTTBTableFunctionTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java index 7719f965df3c5..799962c2d41d1 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java @@ -182,6 +182,11 @@ public void testTargetCountNonFiniteFinalPointDoesNotBecomeLookahead() { new int[] {0, 2, 5}, LTTBTableFunction.selectTargetCount( times, new double[] {0, 1, 9, 2, Double.NaN, Double.NaN}, 6, 3)); + // The last bucket looks ahead to the last finite point t5 = 3, so t4 = 9 wins over t3 = 2. + assertArrayEquals( + new int[] {0, 1, 4, 6}, + LTTBTableFunction.selectTargetCount( + new long[] {0, 1, 2, 3, 4, 5, 6}, new double[] {0, 5, 1, 2, 9, 3, Double.NaN}, 7, 4)); // Control: a finite final point is C of the last bucket and selects the same peak. assertArrayEquals( new int[] {0, 2, 5}, From 3660c7061cdae18a8a4897cef091518d895fae19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:09:19 +0800 Subject: [PATCH 14/21] fix(lttb): stream windows when a column has a single eligible point `isReady` consulted `lastEligiblePositions` without checking `hasLastEligible`. The array is only written for the second and later eligible points of a column, so a column with exactly one eligible point kept the default value `0`. `containsPosition(window, 0)` then matched the first window forever, and since only the head of `pendingWindows` is emitted, every window of the partition stayed buffered until `finish`. A partitioning where one device reports a single non-NULL value therefore failed with "LTTB buffers exceed lttb_buffer_size_in_bytes" even though the partition was small: the budget was enforced against the windows the producer could no longer release. Guard both checks with `hasLastEligible` and document why the sentinel cannot be used. Cover count and time windows with a regression test that requires rows to leave the processor before the partition ends. Co-Authored-By: Claude Opus 5 (1M context) --- .../relational/tvf/LTTBTableFunction.java | 10 +++++-- .../relational/tvf/LTTBWindowMemoryTest.java | 27 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index 7372da87431a3..264667253bd8d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -1534,13 +1534,18 @@ protected final void emitReadyWindows(List builders) { * Whether the first pending window can be emitted. A window that contains {@code L} must wait * for the end of the partition, and so must a window whose {@code C} would come from a bucket * that can still lose {@code L}. + * + *

    A column with fewer than two eligible points has no {@code L}: the only point is {@code F} + * and was never collected into a bucket. {@link #lastEligiblePositions} still holds its default + * value then, so it must not be consulted — position {@code 0} would otherwise match the first + * window and stall every later window until the partition ends. */ private boolean isReady(W window) { if (finishing) { return true; } for (int i = 0; i < participantColumns.length; i++) { - if (containsPosition(window, lastEligiblePositions[i])) { + if (hasLastEligible[i] && containsPosition(window, lastEligiblePositions[i])) { return false; } PointBuffer bucket = window.buffers[i]; @@ -1548,7 +1553,8 @@ private boolean isReady(W window) { continue; } WindowState lookahead = findLookahead(window, i); - if (lookahead == null || containsPosition(lookahead, lastEligiblePositions[i])) { + if (lookahead == null + || (hasLastEligible[i] && containsPosition(lookahead, lastEligiblePositions[i]))) { return false; } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java index d9d74d7b51047..511fea5dacbd2 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java @@ -36,6 +36,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -139,6 +140,32 @@ public void testEmittedWindowsReleaseBudgetForLongStream() { } } + @Test + public void testColumnWithASingleEligiblePointDoesNotStallEmission() { + // Only the first row has a value, so hasLastEligible stays false and lastEligiblePositions + // keeps its default 0. Admission must not treat that sentinel as a real last position: the + // window holding it would then never become ready and the whole partition would be buffered. + for (boolean timeWindow : new boolean[] {false, true}) { + TableFunctionDataProcessor processor = processor(timeWindow, 1, 1, Type.DOUBLE); + List builders = builders(timeWindow); + long emittedBeforeFinish = 0; + try { + for (int row = 0; row < 80_000; row++) { + processor.process(record(row, row == 0 ? 42.0 : null), builders, null); + if (row % 512 == 511) { + emittedBeforeFinish += builders.get(0).getPositionCount(); + builders = builders(timeWindow); + } + } + // Windows must stream out while the partition is still being read. + assertTrue(emittedBeforeFinish > 0); + processor.finish(builders, null); + } finally { + processor.beforeDestroy(); + } + } + } + @Test public void testWindowResultsArePreserved() { for (boolean timeWindow : new boolean[] {false, true}) { From 640d40203a95f618f455968f8321581715dbfaf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:42:28 +0800 Subject: [PATCH 15/21] fix(lttb): release closed windows that can never emit a row `emitClosedWindows` queued every closed window. Only the head of the queue is emitted, so an empty window that stays pending because a later window still holds the current last eligible point blocks every window behind it and keeps its preallocated buffers alive until `finish`. A sparse partition demonstrates it: two non-NULL values followed by a long NULL tail exhausts `lttb_buffer_size_in_bytes`, because the window that may report L never becomes ready while the empty windows after it are never released. A closed window can be dropped safely when no bucket holds a point and the window does not report F: L is collected into every bucket that contains it, so such a window cannot hold L either. Lookahead selection already ignores empty buckets, so C is unchanged, and no later input can reach a closed window. Co-Authored-By: Claude Opus 5 (1M context) --- .../relational/tvf/LTTBTableFunction.java | 21 +++++++++++- .../relational/tvf/LTTBWindowMemoryTest.java | 34 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index 264667253bd8d..af2641c2bedac 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -1468,11 +1468,30 @@ protected final void collectIntoWindows(long position, Record input, long time) /** Moves the windows that ended before {@code position} into the pending queue. */ protected final void emitClosedWindows(long position, List builders) { while (!activeWindows.isEmpty() && activeWindows.peekFirst().endExclusive <= position) { - pendingWindows.addLast(activeWindows.removeFirst()); + W window = activeWindows.removeFirst(); + if (canEmitAnyRow(window)) { + pendingWindows.addLast(window); + } else { + // A closed window that holds no bucket and reports no F can never emit a row, and L is + // never its own row because L was collected into every bucket that contains it. Queueing + // it would let it hold a place in front of later windows — and its buffers — until the + // partition ends, so release it now. + releaseWindow(window); + } } emitReadyWindows(builders); } + /** Whether {@code window} can still contribute at least one output row. */ + private boolean canEmitAnyRow(W window) { + for (int i = 0; i < participantColumns.length; i++) { + if (window == firstEligibleWindows[i] || !window.buffers[i].isEmpty()) { + return true; + } + } + return false; + } + @Override public final void finish( List properColumnBuilders, ColumnBuilder passThroughIndexBuilder) { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java index 511fea5dacbd2..36339347fcb0b 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBWindowMemoryTest.java @@ -166,6 +166,40 @@ public void testColumnWithASingleEligiblePointDoesNotStallEmission() { } } + @Test + public void testWindowsAfterTheLastValueDoNotStallEmission() { + // Two eligible points followed by a long NULL tail. The window holding the second point is the + // one that may report L, so it must wait for the end of the partition; every window after it is + // empty and emittable by nobody. Queuing those empty windows would block the head of the queue + // and keep their buffers alive until finish, which exhausts the budget on a sparse partition. + for (boolean timeWindow : new boolean[] {false, true}) { + TableFunctionDataProcessor processor = processor(timeWindow, 1, 1, Type.DOUBLE); + List builders = builders(timeWindow); + try { + for (int row = 0; row < 80_000; row++) { + Double value = null; + if (row == 0) { + value = 1.0; + } else if (row == 1) { + value = 5.0; + } + processor.process(record(row, value), builders, null); + } + processor.finish(builders, null); + ColumnBuilder times = builders.get(timeWindow ? 2 : 1); + ColumnBuilder valuesBuilder = builders.get(timeWindow ? 3 : 2); + // Window 0 reports F(t0), window 1 reports L(t1); the NULL tail emits nothing. + assertEquals(2, times.getPositionCount()); + assertEquals(0, times.build().getLong(0)); + assertEquals(1, times.build().getLong(1)); + assertEquals(1.0, valuesBuilder.build().getDouble(0), 0.0); + assertEquals(5.0, valuesBuilder.build().getDouble(1), 0.0); + } finally { + processor.beforeDestroy(); + } + } + } + @Test public void testWindowResultsArePreserved() { for (boolean timeWindow : new boolean[] {false, true}) { From 160bc71cf0e115c3d8ae1f4747fa3c8ee8cfc093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:38:41 +0800 Subject: [PATCH 16/21] fix(lttb): keep the anchor when a window bucket has no finite candidate argMaxArea returns the bucket start instead of NOT_SELECTED when every candidate is NaN or infinite, so the guard in the window processor moved A onto that non-finite point and the next window fell back to its earliest point. Treat a non-finite pick like no pick: return the earliest point and keep A unchanged, matching the rule in the class Javadoc. Also reword the TableFunctionOperator retained-size comment: the buffers are charged to both the peak and the retained size, because parents read one or the other. Co-Authored-By: Claude Fable 5.1 --- .../function/TableFunctionOperator.java | 5 +++-- .../relational/tvf/LTTBTableFunction.java | 2 +- .../relational/tvf/LTTBRegressionTest.java | 20 +++++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java index f111c887ef7a9..7b91c8d242322 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java @@ -314,8 +314,9 @@ public long calculateMaxReturnSize() { @Override public long calculateRetainedSizeAfterCallingNext() { - // The buffers survive across next() calls until the partition completes, so they are retained - // here rather than charged to the peek of a single call. + // The buffers survive across next() calls until the partition completes, so they belong to the + // retained size as well as the peak. Parents read one or the other (FilterAndProjectOperator + // adds the child's retained size, sinks and LimitOperator take the child's peak), so keep both. return inputOperator.calculateRetainedSizeAfterCallingNext() + estimatedMaxBufferedBytes; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index af2641c2bedac..292de136852c6 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -1753,7 +1753,7 @@ private WindowRow pickInBucket(W window, int column, PointBuffer bucket) { 0, exactLookahead.size()); } - if (pick == NOT_SELECTED) { + if (pick == NOT_SELECTED || !isFiniteValue(bucket.values, bucket.integralValues, pick)) { // The bucket has no finite candidate; return its earliest point and keep A unchanged. return WindowRow.of(bucket, 0); } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java index 37f49fa3212a8..5e99119ca8251 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java @@ -255,6 +255,26 @@ public void testWindowWithNoEligiblePointsEmitsNoRow() { } } + /** + * A window whose bucket has no finite candidate returns its earliest point without moving {@code + * A}, so the next window is still scored against the last finite pick. + */ + @Test + public void testWindowModeNonFiniteBucketKeepsAnchor() { + Double[][] rows = {{0.0}, {1.0}, {Double.NaN}, {Double.NaN}, {0.5}, {9.0}, {3.0}, {0.0}}; + for (String mode : new String[] {"COUNT_WINDOW", "TIME_WINDOW"}) { + List out = sample(mode, 2, 2, 0, rows); + int timeIndex = mode.equals("TIME_WINDOW") ? 2 : 1; + // W0 reports F(t0) and picks t1; W1 holds only NaN, returns t2 and keeps A = t1; W2 scores + // t4 (area 8.5) and t5 (area 32) against A = t1 and C = t6; W3 picks t6 and reports L(t7). + long[] expected = {0, 1, 2, 5, 6, 7}; + assertEquals(expected.length, out.get(timeIndex).getPositionCount()); + for (int i = 0; i < expected.length; i++) { + assertEquals(expected[i], out.get(timeIndex).build().getLong(i)); + } + } + } + @Test public void testTargetColumnsShareMemoryBudget() { TableFunctionDataProcessor processor = processor("TARGET_COUNT", 3, 3, 0, 8); From 5f96d40c38bb6a148529e12a436eb6e803dcf8b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:46:32 +0800 Subject: [PATCH 17/21] refactor(lttb): merge hasExactValue into isFiniteValue Both helpers had identical bodies and the same predicate: an integral column is always finite, and a floating-point value is finite when it is neither NaN nor infinite. One name and one Javadoc state the condition once, so callers cannot drift apart. Co-Authored-By: Claude Fable 5.1 --- .../relational/tvf/LTTBTableFunction.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index 292de136852c6..62904377184a9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -608,12 +608,11 @@ private static double boundedArea( return Double.isFinite(area) ? area : -1; } - /** Whether the participant value at {@code index} is representable as an exact decimal. */ - private static boolean hasExactValue(double[] values, long[] integralValues, int index) { - return integralValues != null || Double.isFinite(values[index]); - } - - /** Whether the participant value at {@code index} can serve as a triangle vertex. */ + /** + * Whether the participant value at {@code index} is finite: integral columns always are, and a + * floating-point value qualifies when it is neither NaN nor infinite. Such a value can serve as a + * triangle vertex and is representable as an exact decimal. + */ private static boolean isFiniteValue(double[] values, long[] integralValues, int index) { return integralValues != null || Double.isFinite(values[index]); } @@ -649,7 +648,7 @@ private static int argMaxArea( nextTime, nextValue); if (area < 0) { - if (!hasExactValue(values, integralValues, i)) { + if (!isFiniteValue(values, integralValues, i)) { // Non-finite participant: boundedArea scored it as the smallest area. continue; } @@ -697,7 +696,7 @@ private static int argMaxAreaExactly( BigDecimal sumTime = BigDecimal.ZERO; BigDecimal sumValue = BigDecimal.ZERO; for (int i = nextStart; i < nextEnd; i++) { - if (!hasExactValue(nextValues, nextIntegralValues, i)) { + if (!isFiniteValue(nextValues, nextIntegralValues, i)) { continue; } sumTime = sumTime.add(BigDecimal.valueOf(nextTimes[i]).subtract(timeOrigin)); @@ -706,7 +705,7 @@ private static int argMaxAreaExactly( int pick = start; BigDecimal maxArea = null; for (int i = start; i < end; i++) { - if (!hasExactValue(values, integralValues, i)) { + if (!isFiniteValue(values, integralValues, i)) { continue; } BigDecimal time = BigDecimal.valueOf(times[i]).subtract(timeOrigin); From 0e173d9f190955b6c2489e813987b2d2e731cf3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:36:46 +0800 Subject: [PATCH 18/21] fix(lttb): validate ORIGIN in analyze() and pin the budget accounting analyze() only checked SLIDE in target-count mode and never read ORIGIN in count-window mode, so the function relied on the SQL analyzer for both rules. Give ORIGIN an UNSPECIFIED_ORIGIN sentinel default (the user-visible default stays 0, the epoch) so analyze() can tell an explicit ORIGIN from the injected default, reject it in target-count and count-window mode, and fall back to 0 in time-window mode. Test changes from the review: - assert the exact non-finite sampling selection {0,3,5} instead of endpoints and length only; - assert the picked times of the TIME_WINDOW overlap case, not just the row count; - cover the SLIDE > SIZE time-window gap, where rows in a gap belong to no bucket and are dropped; - cover that the operator charges the buffer budget to both the peak and the retained size; - reword the IT comments so the four-region distribution is an expectation of the allocator, not a precondition of the test. Co-Authored-By: Claude Fable 5.1 --- .../it/db/it/IoTDBLTTBTableFunctionIT.java | 19 ++-- .../tvf/TableFunctionOperatorTest.java | 43 +++++++++ .../relational/tvf/LTTBTableFunction.java | 22 ++++- .../relational/tvf/LTTBRegressionTest.java | 25 +++++ .../relational/tvf/LTTBTableFunctionTest.java | 95 ++++++++++++++++--- 5 files changed, 181 insertions(+), 23 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java index dd7a4079234b5..c09268d3ecc8a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLTTBTableFunctionIT.java @@ -88,10 +88,11 @@ public static void setUp() throws Exception { "INSERT INTO shared(time, site, device, s1) VALUES " + "(0, 'a', 'x1', 1.0), (1, 'a', 'x1', 2.0), (0, 'a', 'x2', 3.0), (1, 'a', 'x2', 4.0)"); // Eight devices of one query partition, each writing t = 0..3, so every timestamp occurs - // eight times in the partition. The eight devices own eight distinct series slots, which the - // partition allocator spreads over the four DataRegionGroups, so the rows sharing t = 0 are - // stored in different regions and the duplicate only becomes visible once the fragments are - // merged. + // eight times in the partition. The eight devices own eight distinct series slots, and the + // partition allocator assigns slots to the four DataRegionGroups in order, so the duplicates + // are normally spread over several regions. The test does not depend on a particular + // distribution: it fails with 701 either way, and catches a lost partition boundary when the + // rows do come from more than one fragment. statement.execute( "CREATE TABLE frag_dup(part STRING TAG, device STRING TAG, s1 DOUBLE FIELD)"); for (int device = 0; device < 8; device++) { @@ -420,11 +421,11 @@ public void testDuplicateTimestampsAcrossDevicesInOnePartitionAreRejected() { @Test public void testDuplicateTimestampsAcrossDistributedFragmentsAreRejected() { - // Every timestamp of the partition is written by all eight devices, whose series slots are - // spread over the four DataRegionGroups of the database, so the duplicate rows are read from - // different regions. The function runs once over the merged partition, so the check sees the - // duplicates regardless of how many fragments supplied them. The window-function and SESSION - // inputs are the ones that used to lose the partition boundary. + // Every timestamp of the partition is written by all eight devices, whose series slots the + // partition allocator assigns to the four DataRegionGroups of the database, so the duplicates + // normally come from several regions. The function runs once over the merged partition, so the + // check sees the duplicates regardless of how many fragments supplied them. The + // window-function and SESSION inputs are the ones that used to lose the partition boundary. String expected = "701: LTTB does not support duplicate timestamps, but timestamp 0 appears more than once" + " in the same partition."; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java index 250883af2b6ad..6429e3f5bd97c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java @@ -61,6 +61,7 @@ import static org.apache.iotdb.calc.plan.planner.CommonOperatorUtils.TIME_COLUMN_TEMPLATE; import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext.createFragmentInstanceContext; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -604,6 +605,48 @@ public long ramBytesUsed() { }; } + /** + * The processor buffers survive until a partition completes, so the operator must charge them to + * both the peak of a call and the retained size after one: parents read one or the other, and a + * budget charged to only one side would not be reserved in queries using the other. + */ + @Test + public void testBufferedBytesAreChargedToPeakAndRetainedSize() throws Exception { + long budget = 4096L; + OperatorContext operatorContext = createOperatorContext("lttb_buffer_estimate"); + Operator child = constructLongChildOperator(operatorContext, new long[] {1}); + try (TableFunctionOperator operator = + new TableFunctionOperator( + operatorContext, + new TableFunctionProcessorProvider() { + @Override + public TableFunctionDataProcessor getDataProcessor() { + return new TableFunctionDataProcessor() { + @Override + public void process( + Record input, + List properColumnBuilders, + ColumnBuilder passThroughIndexBuilder) {} + }; + } + }, + budget, + child, + Collections.singletonList(TSDataType.INT64), + Collections.singletonList(TSDataType.INT64), + 1, + Collections.singletonList(0), + Collections.emptyList(), + false, + Collections.emptyList(), + false, + mock(IoTDBLocal.class))) { + assertEquals(budget, operator.calculateRetainedSizeAfterCallingNext()); + // peek = child peak (0) + budget + max(return size, block builder retained) + assertTrue(operator.calculateMaxPeekMemory() >= budget + operator.calculateMaxReturnSize()); + } + } + private Operator constructLongChildOperator(OperatorContext operatorContext, long[] values) { return constructLongChildOperator(operatorContext, values, new AtomicBoolean()); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index 62904377184a9..e6d9af0aca5fc 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.i18n.CommonMessages; +import org.apache.iotdb.commons.i18n.QueryMessages; import org.apache.iotdb.commons.queryengine.plan.relational.function.TableFunctionProcessorMemoryBudget; import org.apache.iotdb.udf.api.exception.UDFException; import org.apache.iotdb.udf.api.relational.TableFunction; @@ -124,6 +125,14 @@ public class LTTBTableFunction implements TableFunction { public static final long UNSPECIFIED_N = Long.MIN_VALUE; public static final long UNSPECIFIED_SIZE = Long.MIN_VALUE; public static final long UNSPECIFIED_SLIDE = Long.MIN_VALUE; + + /** + * Marks {@code ORIGIN} as not specified. The user-visible default of the argument is still {@code + * 0} (the epoch); the sentinel only lets {@link #analyze} distinguish an explicit {@code ORIGIN} + * from the injected default, so the function rejects the argument in modes that do not use it. + */ + public static final long UNSPECIFIED_ORIGIN = Long.MIN_VALUE; + public static final long MIN_TARGET_COUNT = 3L; /** Hard cap on eligible points buffered per participant column in one partition or window. */ @@ -199,7 +208,7 @@ public List getArgumentsSpecifications() { ScalarParameterSpecification.builder() .name(ORIGIN_PARAMETER_NAME) .type(Type.TIMESTAMP) - .defaultValue(0L) + .defaultValue(UNSPECIFIED_ORIGIN) .build()); } @@ -215,9 +224,11 @@ public TableFunctionAnalysis analyze(Map arguments) throws UDF long n = getLong(arguments, N_PARAMETER_NAME); long size = getLong(arguments, SIZE_PARAMETER_NAME); long slide = getLong(arguments, SLIDE_PARAMETER_NAME); + long origin = getLong(arguments, ORIGIN_PARAMETER_NAME); boolean nSpecified = n != UNSPECIFIED_N; boolean sizeSpecified = size != UNSPECIFIED_SIZE; boolean slideSpecified = slide != UNSPECIFIED_SLIDE; + boolean originSpecified = origin != UNSPECIFIED_ORIGIN; if (nSpecified == sizeSpecified) { throw new SemanticException( CommonMessages @@ -226,7 +237,7 @@ public TableFunctionAnalysis analyze(Map arguments) throws UDF Mode mode; if (nSpecified) { - if (slideSpecified) { + if (slideSpecified || originSpecified) { throw new SemanticException( CommonMessages .EXCEPTION_THE_N_ARGUMENT_OF_LTTB_CANNOT_BE_COMBINED_WITH_THE_SLIDE_OR_ORIGIN_ARGUMENTS_C2FF1FE1); @@ -252,6 +263,11 @@ public TableFunctionAnalysis analyze(Map arguments) throws UDF boolean isTimeWindow = arguments.containsKey(WINDOW_MODE_PARAMETER_NAME) && (boolean) ((ScalarArgument) arguments.get(WINDOW_MODE_PARAMETER_NAME)).getValue(); + if (!isTimeWindow && originSpecified) { + throw new SemanticException( + QueryMessages + .EXCEPTION_THE_ORIGIN_ARGUMENT_IS_ONLY_SUPPORTED_IN_TIME_WINDOW_MODE_B3F358B9); + } mode = isTimeWindow ? Mode.TIME_WINDOW : Mode.COUNT_WINDOW; } @@ -296,7 +312,7 @@ public TableFunctionAnalysis analyze(Map arguments) throws UDF } else { handleBuilder.addProperty(SIZE_PARAMETER_NAME, size).addProperty(SLIDE_PARAMETER_NAME, slide); if (mode == Mode.TIME_WINDOW) { - handleBuilder.addProperty(ORIGIN_PARAMETER_NAME, getLong(arguments, ORIGIN_PARAMETER_NAME)); + handleBuilder.addProperty(ORIGIN_PARAMETER_NAME, originSpecified ? origin : 0L); } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java index 5e99119ca8251..d76e1a4d35d50 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBRegressionTest.java @@ -162,6 +162,31 @@ public void testOverlappingWindowsContinueTheAnchorChain() { List timeWindow = sample("TIME_WINDOW", 3, 1, 0, rows); Column timePicks = timeWindow.get(timeWindow.size() - 2).build(); assertEquals(8, timePicks.getPositionCount()); + long[] expectedTimes = {0, 1, 2, 1, 2, 3, 4, 5}; + for (int i = 0; i < expectedTimes.length; i++) { + assertEquals(expectedTimes[i], timePicks.getLong(i)); + } + } + + /** + * With {@code SLIDE > SIZE} the gaps between windows belong to no bucket. A row in a gap is + * neither collected into a bucket nor eligible as {@code F} or {@code L}, because only points + * that fall in a window can be reported. Here the windows are {@code [0,2)} and {@code [3,5)}; t5 + * falls in the gap {@code [5,6)} and is therefore dropped, while the last point that does fall in + * a window is t4. + */ + @Test + public void testTimeWindowGapRowsAreDropped() { + Double[][] rows = {{0.0}, {1.0}, {2.0}, {3.0}, {4.0}, {5.0}}; + List out = sample("TIME_WINDOW", 2, 3, 0, rows); + Column picks = out.get(out.size() - 2).build(); + // The first window holds t0 and t1: it reports F(t0) and picks t1. The second holds t3 and + // t4: it picks t3 and reports L(t4), the last point that falls in a window. + long[] expected = {0, 1, 3, 4}; + assertEquals(expected.length, picks.getPositionCount()); + for (int i = 0; i < expected.length; i++) { + assertEquals(expected[i], picks.getLong(i)); + } } /** diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java index 799962c2d41d1..1407dcc6292e1 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java @@ -21,7 +21,9 @@ import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.i18n.CommonMessages; +import org.apache.iotdb.commons.i18n.QueryMessages; import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.table.MapTableFunctionHandle; import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis; import org.apache.iotdb.udf.api.relational.table.argument.Argument; import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; @@ -38,7 +40,6 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class LTTBTableFunctionTest { @@ -114,6 +115,56 @@ public void testAnalyzeRejectsNAndSizeTogether() throws UDFException { } } + @Test + public void testAnalyzeRejectsTargetCountWithOrigin() throws UDFException { + Map arguments = targetCountArguments(3L); + arguments.put( + LTTBTableFunction.ORIGIN_PARAMETER_NAME, new ScalarArgument(Type.TIMESTAMP, 1000L)); + try { + new LTTBTableFunction().analyze(arguments); + fail(); + } catch (SemanticException e) { + // compare against the constant so the test also passes under -P with-zh-locale + assertEquals( + CommonMessages + .EXCEPTION_THE_N_ARGUMENT_OF_LTTB_CANNOT_BE_COMBINED_WITH_THE_SLIDE_OR_ORIGIN_ARGUMENTS_C2FF1FE1, + e.getMessage()); + } + } + + @Test + public void testAnalyzeRejectsOriginInCountWindowMode() throws UDFException { + Map arguments = windowArguments(2L, 2L, false); + arguments.put( + LTTBTableFunction.ORIGIN_PARAMETER_NAME, new ScalarArgument(Type.TIMESTAMP, 1000L)); + try { + new LTTBTableFunction().analyze(arguments); + fail(); + } catch (SemanticException e) { + assertEquals( + QueryMessages + .EXCEPTION_THE_ORIGIN_ARGUMENT_IS_ONLY_SUPPORTED_IN_TIME_WINDOW_MODE_B3F358B9, + e.getMessage()); + } + } + + @Test + public void testAnalyzeAcceptsOriginInTimeWindowMode() throws UDFException { + Map arguments = windowArguments(2L, 2L, true); + arguments.put( + LTTBTableFunction.ORIGIN_PARAMETER_NAME, new ScalarArgument(Type.TIMESTAMP, 1000L)); + TableFunctionAnalysis analysis = new LTTBTableFunction().analyze(arguments); + MapTableFunctionHandle handle = (MapTableFunctionHandle) analysis.getTableFunctionHandle(); + assertEquals(1000L, handle.getProperty(LTTBTableFunction.ORIGIN_PARAMETER_NAME)); + } + + @Test + public void testTimeWindowWithoutOriginUsesTheEpoch() throws UDFException { + TableFunctionAnalysis analysis = new LTTBTableFunction().analyze(windowArguments(2L, 2L, true)); + MapTableFunctionHandle handle = (MapTableFunctionHandle) analysis.getTableFunctionHandle(); + assertEquals(0L, handle.getProperty(LTTBTableFunction.ORIGIN_PARAMETER_NAME)); + } + @Test public void testTargetCountKeepsNonFiniteValues() { long[] times = {0, 1, 2}; @@ -135,15 +186,10 @@ public void testTargetCountSkipsNonFiniteCandidatesWhenSampling() { long[] times = {0, 1, 2, 3, 4, 5}; double[] values = {1.0, Double.NaN, Double.POSITIVE_INFINITY, 2.0, 3.0, 4.0}; - int[] selected = LTTBTableFunction.selectTargetCount(times, values, 6, 3); - - // The first and last eligible points are always preserved. - assertEquals(0, selected[0]); - assertEquals(5, selected[selected.length - 1]); - assertEquals(3, selected.length); - for (int index : selected) { - assertTrue("selected index out of range: " + index, index >= 0 && index < 6); - } + // The only interior bucket is t1..t4 with C = the final point; t3 (area 4) beats t4 (area 2) + // and the non-finite candidates can never win. + assertArrayEquals( + new int[] {0, 3, 5}, LTTBTableFunction.selectTargetCount(times, values, 6, 3)); } /** A bucket whose candidates are all non-finite keeps its earliest point, deterministically. */ @@ -272,9 +318,36 @@ private static Map targetCountArguments(long n) { arguments.put( LTTBTableFunction.SLIDE_PARAMETER_NAME, new ScalarArgument(Type.INT64, LTTBTableFunction.UNSPECIFIED_SLIDE)); - arguments.put(LTTBTableFunction.ORIGIN_PARAMETER_NAME, new ScalarArgument(Type.TIMESTAMP, 0L)); + arguments.put( + LTTBTableFunction.ORIGIN_PARAMETER_NAME, + new ScalarArgument(Type.TIMESTAMP, LTTBTableFunction.UNSPECIFIED_ORIGIN)); arguments.put( LTTBTableFunction.WINDOW_MODE_PARAMETER_NAME, new ScalarArgument(Type.BOOLEAN, false)); return arguments; } + + private static Map windowArguments(long size, long slide, boolean timeWindow) { + Map arguments = new HashMap<>(); + arguments.put( + LTTBTableFunction.DATA_PARAMETER_NAME, + new TableArgument( + Arrays.asList(Optional.of("time"), Optional.of("device"), Optional.of("s1")), + Arrays.asList(Type.TIMESTAMP, Type.STRING, Type.DOUBLE), + Collections.singletonList("device"), + Collections.singletonList("time"), + false)); + arguments.put( + LTTBTableFunction.TIMECOL_PARAMETER_NAME, new ScalarArgument(Type.STRING, "time")); + arguments.put( + LTTBTableFunction.N_PARAMETER_NAME, + new ScalarArgument(Type.INT64, LTTBTableFunction.UNSPECIFIED_N)); + arguments.put(LTTBTableFunction.SIZE_PARAMETER_NAME, new ScalarArgument(Type.INT64, size)); + arguments.put(LTTBTableFunction.SLIDE_PARAMETER_NAME, new ScalarArgument(Type.INT64, slide)); + arguments.put( + LTTBTableFunction.ORIGIN_PARAMETER_NAME, + new ScalarArgument(Type.TIMESTAMP, LTTBTableFunction.UNSPECIFIED_ORIGIN)); + arguments.put( + LTTBTableFunction.WINDOW_MODE_PARAMETER_NAME, new ScalarArgument(Type.BOOLEAN, timeWindow)); + return arguments; + } } From 5ed703252ed56f8bdc294ef68507936d0834f35f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:40:18 +0800 Subject: [PATCH 19/21] fix(lttb): tell an explicit ORIGIN from the default by presence, not value Long.MIN_VALUE is a legal TIMESTAMP, so the sentinel default could not distinguish an explicit extreme ORIGIN from an absent one: in time-window mode it was silently replaced by the epoch, and in count-window mode the rejection did not fire. The analyzer now injects ORIGIN_SPECIFIED, which records whether the user passed the argument; analyze() prefers it and falls back to the value comparison only for direct callers that do not provide it. Co-Authored-By: Claude Fable 5.1 --- .../analyzer/StatementAnalyzer.java | 5 +++ .../relational/tvf/LTTBTableFunction.java | 12 ++++++- .../relational/tvf/LTTBTableFunctionTest.java | 36 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 50668e5b89577..3478ec1c53b1b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -6234,6 +6234,11 @@ private void tryAppendLTTBModeArgument( passedArguments.put( LTTBTableFunction.WINDOW_MODE_PARAMETER_NAME, new ScalarArgument(org.apache.iotdb.udf.api.type.Type.BOOLEAN, isTimeWindow)); + // Presence, not value: Long.MIN_VALUE is a legal timestamp, so the value alone cannot tell + // an explicit ORIGIN from the injected default. + passedArguments.put( + LTTBTableFunction.ORIGIN_SPECIFIED_PARAMETER_NAME, + new ScalarArgument(org.apache.iotdb.udf.api.type.Type.BOOLEAN, originSpecified)); } private void validateM4OrderBySortOrder( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index e6d9af0aca5fc..0a9ed7e746d53 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -122,6 +122,13 @@ public class LTTBTableFunction implements TableFunction { /** Internal boolean argument injected by the analyzer: {@code true} if SIZE is a duration. */ public static final String WINDOW_MODE_PARAMETER_NAME = "__LTTB_WINDOW_MODE"; + /** + * Internal boolean argument injected by the analyzer: {@code true} if the user passed {@code + * ORIGIN}. The analyzer knows this from the argument list, which is more precise than comparing + * against {@link #UNSPECIFIED_ORIGIN}: {@link Long#MIN_VALUE} is a legal timestamp. + */ + public static final String ORIGIN_SPECIFIED_PARAMETER_NAME = "__LTTB_ORIGIN_SPECIFIED"; + public static final long UNSPECIFIED_N = Long.MIN_VALUE; public static final long UNSPECIFIED_SIZE = Long.MIN_VALUE; public static final long UNSPECIFIED_SLIDE = Long.MIN_VALUE; @@ -228,7 +235,10 @@ public TableFunctionAnalysis analyze(Map arguments) throws UDF boolean nSpecified = n != UNSPECIFIED_N; boolean sizeSpecified = size != UNSPECIFIED_SIZE; boolean slideSpecified = slide != UNSPECIFIED_SLIDE; - boolean originSpecified = origin != UNSPECIFIED_ORIGIN; + boolean originSpecified = + arguments.containsKey(ORIGIN_SPECIFIED_PARAMETER_NAME) + ? (boolean) ((ScalarArgument) arguments.get(ORIGIN_SPECIFIED_PARAMETER_NAME)).getValue() + : origin != UNSPECIFIED_ORIGIN; if (nSpecified == sizeSpecified) { throw new SemanticException( CommonMessages diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java index 1407dcc6292e1..583f44e725834 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunctionTest.java @@ -158,6 +158,42 @@ public void testAnalyzeAcceptsOriginInTimeWindowMode() throws UDFException { assertEquals(1000L, handle.getProperty(LTTBTableFunction.ORIGIN_PARAMETER_NAME)); } + /** + * {@link Long#MIN_VALUE} is a legal timestamp, so it must not be confused with an absent ORIGIN: + * an explicit extreme origin is kept as given, not replaced by the epoch. + */ + @Test + public void testTimeWindowKeepsTheMinimumRepresentableOrigin() throws UDFException { + Map arguments = windowArguments(2L, 2L, true); + arguments.put( + LTTBTableFunction.ORIGIN_PARAMETER_NAME, + new ScalarArgument(Type.TIMESTAMP, Long.MIN_VALUE)); + arguments.put( + LTTBTableFunction.ORIGIN_SPECIFIED_PARAMETER_NAME, new ScalarArgument(Type.BOOLEAN, true)); + TableFunctionAnalysis analysis = new LTTBTableFunction().analyze(arguments); + MapTableFunctionHandle handle = (MapTableFunctionHandle) analysis.getTableFunctionHandle(); + assertEquals(Long.MIN_VALUE, handle.getProperty(LTTBTableFunction.ORIGIN_PARAMETER_NAME)); + } + + @Test + public void testAnalyzeRejectsMinimumOriginInCountWindowMode() throws UDFException { + Map arguments = windowArguments(2L, 2L, false); + arguments.put( + LTTBTableFunction.ORIGIN_PARAMETER_NAME, + new ScalarArgument(Type.TIMESTAMP, Long.MIN_VALUE)); + arguments.put( + LTTBTableFunction.ORIGIN_SPECIFIED_PARAMETER_NAME, new ScalarArgument(Type.BOOLEAN, true)); + try { + new LTTBTableFunction().analyze(arguments); + fail(); + } catch (SemanticException e) { + assertEquals( + QueryMessages + .EXCEPTION_THE_ORIGIN_ARGUMENT_IS_ONLY_SUPPORTED_IN_TIME_WINDOW_MODE_B3F358B9, + e.getMessage()); + } + } + @Test public void testTimeWindowWithoutOriginUsesTheEpoch() throws UDFException { TableFunctionAnalysis analysis = new LTTBTableFunction().analyze(windowArguments(2L, 2L, true)); From db7fa8c2b49cee0d088f154b3ed3d565f3c86a92 Mon Sep 17 00:00:00 2001 From: DaZuiZui Date: Mon, 16 Nov 2026 12:18:33 +0800 Subject: [PATCH 20/21] fix(lttb): count the buffer once so 1C1D fragments fit An LTTB fragment reserves the processor budget in calculateMaxPeekMemory and a sort buffer beside it. Charging that budget to calculateRetainedSizeAfterCallingNext as well made Sort and MergeSort reserve it a second time, so the 1C1D heap rejected the fragment with error 719. Keep the budget in the peak, which is what the fragment estimator reserves, and resolve a non-positive lttb_buffer_size_in_bytes to a quarter of one query thread's operator memory so the two buffers fit together. An explicit positive value is unchanged. --- .../function/TableFunctionOperator.java | 10 +++--- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 35 +++++++++++++------ .../tvf/TableFunctionOperatorTest.java | 11 +++--- .../conf/iotdb-system.properties.template | 6 ++-- .../iotdb/commons/conf/CommonConfig.java | 5 +-- .../relational/tvf/LTTBTableFunction.java | 2 +- 6 files changed, 43 insertions(+), 26 deletions(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java index c69a7bd9fa5b5..0ff6a005aafb8 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java @@ -317,10 +317,12 @@ public long calculateMaxReturnSize() { @Override public long calculateRetainedSizeAfterCallingNext() { - // The buffers survive across next() calls until the partition completes, so they belong to the - // retained size as well as the peak. Parents read one or the other (FilterAndProjectOperator - // adds the child's retained size, sinks and LimitOperator take the child's peak), so keep both. - return inputOperator.calculateRetainedSizeAfterCallingNext() + estimatedMaxBufferedBytes; + // The buffer budget is already part of calculateMaxPeekMemory. Parents that add a child's + // retained size (FilterAndProjectOperator, SortOperator, MergeSortOperator) also take the + // child's peak, so charging the budget here as well reserves it twice. On the 1C1D CI heap + // that rejects the fragment with error 719 before the query runs. The budget stays in the + // peak, which is the value the fragment estimator reserves. + return inputOperator.calculateRetainedSizeAfterCallingNext(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 5061c5ee0f5ef..a35b5205d237f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1122,12 +1122,13 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException commonConfig.setCteBufferSize(cteBufferSizeInBytes); } - // The per-processor buffer budget of the LTTB table function. Like sort_buffer_size_in_bytes - // this is charged to the estimated memory of every fragment that runs the function, so the - // non-positive default resolves to a value derived from the operator memory of this node - // instead of one that would reject the query outright on a small-memory node. + // The per-processor buffer budget of the LTTB table function. A fragment that runs LTTB also + // reserves a sort buffer for the same partition, and both are reserved together, so the + // non-positive default takes a quarter of one query thread's operator memory. Half, which is + // what sort_buffer_size_in_bytes uses, leaves the two buffers larger than the free operator + // memory on the 1C1D integration-test heap and the fragment is rejected with error 719. loadFixedSizeLimitForQuery( - properties, "lttb_buffer_size_in_bytes", commonConfig::setLttbBufferSize); + properties, "lttb_buffer_size_in_bytes", commonConfig::setLttbBufferSize, 4); // max number of rows for cte materialization int maxRowsInCteBuffer = @@ -1305,12 +1306,23 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException private void loadFixedSizeLimitForQuery( TrimProperties properties, String name, LongConsumer setFunction) { + loadFixedSizeLimitForQuery(properties, name, setFunction, 2); + } + + /** + * Resolves a per-fragment byte budget. A non-positive configured value becomes {@code min(32 MiB, + * operator memory / query_thread_count / threadDivisor)}. Sort and the mods cache split one query + * thread's operator memory in two. LTTB splits it in four, because the fragment reserves an LTTB + * budget and a sort budget together. + */ + private void loadFixedSizeLimitForQuery( + TrimProperties properties, String name, LongConsumer setFunction, int threadDivisor) { long defaultValue = Math.min( 32 * 1024 * 1024L, memoryConfig.getOperatorsMemoryManager().getTotalMemorySizeInBytes() / memoryConfig.getQueryThreadCount() - / 2); + / threadDivisor); long size = Long.parseLong(properties.getProperty(name, Long.toString(defaultValue))); if (size <= 0) { size = defaultValue; @@ -2412,12 +2424,13 @@ public synchronized void loadHotModifiedProps(TrimProperties properties) if (cteBufferSizeInBytes > 0) { commonConfig.setCteBufferSize(cteBufferSizeInBytes); } - // The per-processor buffer budget of the LTTB table function. Like sort_buffer_size_in_bytes - // this is charged to the estimated memory of every fragment that runs the function, so the - // non-positive default resolves to a value derived from the operator memory of this node - // instead of one that would reject the query outright on a small-memory node. + // The per-processor buffer budget of the LTTB table function. A fragment that runs LTTB also + // reserves a sort buffer for the same partition, and both are reserved together, so the + // non-positive default takes a quarter of one query thread's operator memory. Half, which is + // what sort_buffer_size_in_bytes uses, leaves the two buffers larger than the free operator + // memory on the 1C1D integration-test heap and the fragment is rejected with error 719. loadFixedSizeLimitForQuery( - properties, "lttb_buffer_size_in_bytes", commonConfig::setLttbBufferSize); + properties, "lttb_buffer_size_in_bytes", commonConfig::setLttbBufferSize, 4); // max number of rows for cte materialization int maxRowsInCteBuffer = Integer.parseInt( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java index 6429e3f5bd97c..e6b4346f9e503 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java @@ -606,12 +606,13 @@ public long ramBytesUsed() { } /** - * The processor buffers survive until a partition completes, so the operator must charge them to - * both the peak of a call and the retained size after one: parents read one or the other, and a - * budget charged to only one side would not be reserved in queries using the other. + * The processor buffers survive until a partition completes. They are charged to the peak, which + * is what the fragment estimator reserves. They are not charged to the retained size as well: + * parents that add a child's retained size also take the child's peak, so charging both reserves + * the budget twice and rejects the fragment with error 719 on the 1C1D heap. */ @Test - public void testBufferedBytesAreChargedToPeakAndRetainedSize() throws Exception { + public void testBufferedBytesAreChargedToThePeakOnly() throws Exception { long budget = 4096L; OperatorContext operatorContext = createOperatorContext("lttb_buffer_estimate"); Operator child = constructLongChildOperator(operatorContext, new long[] {1}); @@ -641,7 +642,7 @@ public void process( Collections.emptyList(), false, mock(IoTDBLocal.class))) { - assertEquals(budget, operator.calculateRetainedSizeAfterCallingNext()); + assertEquals(0L, operator.calculateRetainedSizeAfterCallingNext()); // peek = child peak (0) + budget + max(return size, block builder retained) assertTrue(operator.calculateMaxPeekMemory() >= budget + operator.calculateMaxReturnSize()); } diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index ac0022803e194..007b6ece6fb67 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1348,9 +1348,9 @@ sort_buffer_size_in_bytes=0 cte_buffer_size_in_bytes=131072 # The per-processor buffer budget of the LTTB table function. It is charged to the estimated memory of every fragment that -# runs LTTB, so if lttb_buffer_size_in_bytes <= 0 a default value is used, default value = min(32MB, memory for query operators / -# query_thread_count / 2), matching sort_buffer_size_in_bytes. Raise it for queries with very large partitions or many overlapping -# windows. +# runs LTTB. Such a fragment also reserves a sort buffer, so if lttb_buffer_size_in_bytes <= 0 a default value is used, +# default value = min(32MB, memory for query operators / query_thread_count / 4). Raise it for queries with very large +# partitions or many overlapping windows. # effectiveMode: hot_reload # Datatype: long # Privilege: SYSTEM diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 8537d087c6bdc..4690ca0fd0b45 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -581,8 +581,9 @@ public class CommonConfig { /** * The per-processor memory budget of the LTTB table function. The shipped value is {@code 0}, - * which the DataNode resolves to {@code min(32 MiB, operator memory / query_thread_count / 2)}; - * this initial value matches {@link #sortBufferSize}. + * which the DataNode resolves to {@code min(32 MiB, operator memory / query_thread_count / 4)}. + * The divisor is 4 rather than the 2 used by {@link #sortBufferSize} because a fragment that + * runs LTTB reserves both budgets. This initial value matches {@link #sortBufferSize}. */ private long lttbBufferSize = 32 * 1024 * 1024L; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index 0a9ed7e746d53..5e4f91ec51b41 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -148,7 +148,7 @@ public class LTTBTableFunction implements TableFunction { /** * Fallback for {@code lttb_buffer_size_in_bytes} when it is not positive. The configured default * in {@code iotdb-system.properties.template} is {@code 0}, which {@code IoTDBDescriptor} - * resolves to {@code min(32 MiB, operator memory / query_thread_count / 2)}; this constant + * resolves to {@code min(32 MiB, operator memory / query_thread_count / 4)}; this constant * therefore only matches {@link org.apache.iotdb.commons.conf.CommonConfig}'s initial value and * keeps a misconfigured node bounded. */ From ed14769115c6077e738bd4236fda01fff7967f29 Mon Sep 17 00:00:00 2001 From: DaZuiZui Date: Mon, 16 Nov 2026 12:45:03 +0800 Subject: [PATCH 21/21] fix(lttb): keep both memory sides and shrink the default budget FilterAndProjectOperator reserves only the retained size, so dropping the LTTB budget from calculateRetainedSizeAfterCallingNext admitted filter and projection queries without reserving the buffers. Count the budget in the peak and the retained size again. A sort above LTTB then reserves the budget twice plus its own buffer, so a non-positive lttb_buffer_size_in_bytes now resolves to one eighth of one query thread's operator memory and that sum fits the 1C1D heap. An explicit positive value is unchanged. --- .../function/TableFunctionOperator.java | 12 ++++---- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 30 ++++++++++--------- .../tvf/TableFunctionOperatorTest.java | 11 ++++--- .../conf/iotdb-system.properties.template | 8 ++--- .../iotdb/commons/conf/CommonConfig.java | 7 +++-- .../relational/tvf/LTTBTableFunction.java | 2 +- 6 files changed, 36 insertions(+), 34 deletions(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java index 0ff6a005aafb8..fd80bab6aa5ca 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/function/TableFunctionOperator.java @@ -317,12 +317,12 @@ public long calculateMaxReturnSize() { @Override public long calculateRetainedSizeAfterCallingNext() { - // The buffer budget is already part of calculateMaxPeekMemory. Parents that add a child's - // retained size (FilterAndProjectOperator, SortOperator, MergeSortOperator) also take the - // child's peak, so charging the budget here as well reserves it twice. On the 1C1D CI heap - // that rejects the fragment with error 719 before the query runs. The budget stays in the - // peak, which is the value the fragment estimator reserves. - return inputOperator.calculateRetainedSizeAfterCallingNext(); + // The buffers survive across next() calls until the partition completes, so they belong to the + // retained size as well as the peak. Parents read one or the other (FilterAndProjectOperator + // adds the child's retained size and never reads the child's peak; sinks and LimitOperator + // take the child's peak), so keep both. Sort adds the two together; the LTTB default budget is + // sized so that sum still fits one query thread. + return inputOperator.calculateRetainedSizeAfterCallingNext() + estimatedMaxBufferedBytes; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index a35b5205d237f..6fc9863ee54f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1122,13 +1122,14 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException commonConfig.setCteBufferSize(cteBufferSizeInBytes); } - // The per-processor buffer budget of the LTTB table function. A fragment that runs LTTB also - // reserves a sort buffer for the same partition, and both are reserved together, so the - // non-positive default takes a quarter of one query thread's operator memory. Half, which is - // what sort_buffer_size_in_bytes uses, leaves the two buffers larger than the free operator - // memory on the 1C1D integration-test heap and the fragment is rejected with error 719. + // The per-processor buffer budget of the LTTB table function. The budget is reserved in both + // the peak and the retained size, and a sort above LTTB adds those two on top of its own + // buffer. The non-positive default therefore takes one eighth of one query thread's operator + // memory, which keeps 2 * lttb + sort within that thread. Half, which is what + // sort_buffer_size_in_bytes uses, makes the sum larger than the free operator memory on the + // 1C1D integration-test heap and the fragment is rejected with error 719. loadFixedSizeLimitForQuery( - properties, "lttb_buffer_size_in_bytes", commonConfig::setLttbBufferSize, 4); + properties, "lttb_buffer_size_in_bytes", commonConfig::setLttbBufferSize, 8); // max number of rows for cte materialization int maxRowsInCteBuffer = @@ -1312,8 +1313,8 @@ private void loadFixedSizeLimitForQuery( /** * Resolves a per-fragment byte budget. A non-positive configured value becomes {@code min(32 MiB, * operator memory / query_thread_count / threadDivisor)}. Sort and the mods cache split one query - * thread's operator memory in two. LTTB splits it in four, because the fragment reserves an LTTB - * budget and a sort budget together. + * thread's operator memory in two. LTTB splits it in eight: its budget is counted in the peak + * and again in the retained size, and the sort buffer is reserved beside those two. */ private void loadFixedSizeLimitForQuery( TrimProperties properties, String name, LongConsumer setFunction, int threadDivisor) { @@ -2424,13 +2425,14 @@ public synchronized void loadHotModifiedProps(TrimProperties properties) if (cteBufferSizeInBytes > 0) { commonConfig.setCteBufferSize(cteBufferSizeInBytes); } - // The per-processor buffer budget of the LTTB table function. A fragment that runs LTTB also - // reserves a sort buffer for the same partition, and both are reserved together, so the - // non-positive default takes a quarter of one query thread's operator memory. Half, which is - // what sort_buffer_size_in_bytes uses, leaves the two buffers larger than the free operator - // memory on the 1C1D integration-test heap and the fragment is rejected with error 719. + // The per-processor buffer budget of the LTTB table function. The budget is reserved in both + // the peak and the retained size, and a sort above LTTB adds those two on top of its own + // buffer. The non-positive default therefore takes one eighth of one query thread's operator + // memory, which keeps 2 * lttb + sort within that thread. Half, which is what + // sort_buffer_size_in_bytes uses, makes the sum larger than the free operator memory on the + // 1C1D integration-test heap and the fragment is rejected with error 719. loadFixedSizeLimitForQuery( - properties, "lttb_buffer_size_in_bytes", commonConfig::setLttbBufferSize, 4); + properties, "lttb_buffer_size_in_bytes", commonConfig::setLttbBufferSize, 8); // max number of rows for cte materialization int maxRowsInCteBuffer = Integer.parseInt( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java index e6b4346f9e503..a042f173c9ef4 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/tvf/TableFunctionOperatorTest.java @@ -606,13 +606,12 @@ public long ramBytesUsed() { } /** - * The processor buffers survive until a partition completes. They are charged to the peak, which - * is what the fragment estimator reserves. They are not charged to the retained size as well: - * parents that add a child's retained size also take the child's peak, so charging both reserves - * the budget twice and rejects the fragment with error 719 on the 1C1D heap. + * The processor buffers survive until a partition completes, so the operator charges them to both + * the peak and the retained size. FilterAndProjectOperator reserves only the retained size, while + * sinks and LimitOperator reserve only the peak. */ @Test - public void testBufferedBytesAreChargedToThePeakOnly() throws Exception { + public void testBufferedBytesAreChargedToPeakAndRetainedSize() throws Exception { long budget = 4096L; OperatorContext operatorContext = createOperatorContext("lttb_buffer_estimate"); Operator child = constructLongChildOperator(operatorContext, new long[] {1}); @@ -642,7 +641,7 @@ public void process( Collections.emptyList(), false, mock(IoTDBLocal.class))) { - assertEquals(0L, operator.calculateRetainedSizeAfterCallingNext()); + assertEquals(budget, operator.calculateRetainedSizeAfterCallingNext()); // peek = child peak (0) + budget + max(return size, block builder retained) assertTrue(operator.calculateMaxPeekMemory() >= budget + operator.calculateMaxReturnSize()); } diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 007b6ece6fb67..10719c45df943 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1347,10 +1347,10 @@ sort_buffer_size_in_bytes=0 # Unit: bytes cte_buffer_size_in_bytes=131072 -# The per-processor buffer budget of the LTTB table function. It is charged to the estimated memory of every fragment that -# runs LTTB. Such a fragment also reserves a sort buffer, so if lttb_buffer_size_in_bytes <= 0 a default value is used, -# default value = min(32MB, memory for query operators / query_thread_count / 4). Raise it for queries with very large -# partitions or many overlapping windows. +# The per-processor buffer budget of the LTTB table function. It is charged to both the peak and the retained size of +# every fragment that runs LTTB, and a sort above it reserves its own buffer too. If lttb_buffer_size_in_bytes <= 0 a +# default value is used, default value = min(32MB, memory for query operators / query_thread_count / 8), so those three +# reservations fit one query thread. Raise it for queries with very large partitions or many overlapping windows. # effectiveMode: hot_reload # Datatype: long # Privilege: SYSTEM diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 4690ca0fd0b45..01b871b711d18 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -581,9 +581,10 @@ public class CommonConfig { /** * The per-processor memory budget of the LTTB table function. The shipped value is {@code 0}, - * which the DataNode resolves to {@code min(32 MiB, operator memory / query_thread_count / 4)}. - * The divisor is 4 rather than the 2 used by {@link #sortBufferSize} because a fragment that - * runs LTTB reserves both budgets. This initial value matches {@link #sortBufferSize}. + * which the DataNode resolves to {@code min(32 MiB, operator memory / query_thread_count / 8)}. + * The divisor is 8 rather than the 2 used by {@link #sortBufferSize}: the LTTB budget is reserved + * twice and the sort budget once, and that sum has to fit one query thread. This initial value + * matches {@link #sortBufferSize}. */ private long lttbBufferSize = 32 * 1024 * 1024L; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java index 5e4f91ec51b41..bbee34a18a6d3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/LTTBTableFunction.java @@ -148,7 +148,7 @@ public class LTTBTableFunction implements TableFunction { /** * Fallback for {@code lttb_buffer_size_in_bytes} when it is not positive. The configured default * in {@code iotdb-system.properties.template} is {@code 0}, which {@code IoTDBDescriptor} - * resolves to {@code min(32 MiB, operator memory / query_thread_count / 4)}; this constant + * resolves to {@code min(32 MiB, operator memory / query_thread_count / 8)}; this constant * therefore only matches {@link org.apache.iotdb.commons.conf.CommonConfig}'s initial value and * keeps a misconfigured node bounded. */