From 2bca3add55fd7de5773f72af2aa2ac646a5f633d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:52:13 +0800 Subject: [PATCH 1/2] Fix date_bin month bins counting intra-second milliseconds twice DateBinFunctionColumnTransformer.convertToTimestamp added Instant.toEpochMilli() and LocalDateTime.getNano() together. The former already contains the whole milliseconds of the second and the latter is the full nanosecond-of-second, so the milliseconds were counted twice and month/year bins started that many milliseconds late. Convert from getEpochSecond() and getNano() instead so each part is counted once. Co-Authored-By: Claude Fable 5.1 --- .../DateBinFunctionColumnTransformer.java | 16 +++--- .../unary/scalar/DateBinFunctionTest.java | 56 +++++++++++++++++++ 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DateBinFunctionColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DateBinFunctionColumnTransformer.java index 445ca0560adf4..2103eaec5f869 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DateBinFunctionColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DateBinFunctionColumnTransformer.java @@ -90,19 +90,19 @@ private static LocalDateTime convertToLocalDateTime(long timestamp, ZoneId zoneI } private static long convertToTimestamp(LocalDateTime dateTime, ZoneId zoneId) { - // Converting LocalDateTime to Seconds since epoch - long epochMilliSecond = dateTime.atZone(zoneId).toInstant().toEpochMilli(); - // Get the nanoseconds section - long nanoAdjustment = dateTime.getNano(); + // getEpochSecond() gives the whole seconds and getNano() the nanos inside the second, so the + // milliseconds inside the second are counted exactly once + Instant instant = dateTime.atZone(zoneId).toInstant(); + long epochSecond = instant.getEpochSecond(); + long nanoAdjustment = instant.getNano(); switch (CommonDescriptor.getInstance().getConfig().getTimestampPrecision()) { case "ms": - return epochMilliSecond + nanoAdjustment / NANOSECONDS_IN_MILLISECOND; + return TimeUnit.SECONDS.toMillis(epochSecond) + nanoAdjustment / NANOSECONDS_IN_MILLISECOND; case "us": - return TimeUnit.MILLISECONDS.toMicros(epochMilliSecond) - + nanoAdjustment / NANOSECONDS_IN_MICROSECOND; + return TimeUnit.SECONDS.toMicros(epochSecond) + nanoAdjustment / NANOSECONDS_IN_MICROSECOND; case "ns": - return TimeUnit.MILLISECONDS.toNanos(epochMilliSecond) + nanoAdjustment; + return TimeUnit.SECONDS.toNanos(epochSecond) + nanoAdjustment; default: throw new IllegalArgumentException( CalcMessages.EXCEPTION_UNKNOWN_PRECISION_0119BEB0 diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/DateBinFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/DateBinFunctionTest.java index 013e8c8429081..9d82ccad9b782 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/DateBinFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/DateBinFunctionTest.java @@ -19,6 +19,11 @@ package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DateBinFunctionColumnTransformer; +import org.apache.iotdb.commons.conf.CommonDescriptor; + +import org.apache.tsfile.read.common.type.TimestampType; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -28,6 +33,8 @@ import java.util.TimeZone; import static org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DateBinFunctionColumnTransformer.dateBin; +import static org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DateBinFunctionColumnTransformer.nextDateBin; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; public class DateBinFunctionTest { @@ -126,4 +133,53 @@ public void testSourceBeforeUnixEpoch() throws ParseException { zoneId); assertEquals(getTimestamp("1969-12-31 00:00:00.000"), result); } + + @Test + public void testMonthIntervalWithSubSecondOrigin() throws ParseException { + // 500 ms is 1970-01-01T00:00:00.500Z, 2678400500 ms is also exactly one month later + assertEquals(500, dateBin(500, 500, 1, 0, zoneId)); + assertEquals(2678400500L, dateBin(2678400500L, 500, 1, 0, zoneId)); + // a point 200 ms into the next bin must still map to the start of that bin + assertEquals(2678400500L, dateBin(2678400700L, 500, 1, 0, zoneId)); + // an origin far from the epoch must keep its sub-second part too + long origin = getTimestamp("2024-03-15 00:00:00.500"); + assertEquals(origin, dateBin(origin, origin, 1, 0, zoneId)); + assertEquals(origin, dateBin(origin + MILLIS_IN_DAY, origin, 1, 0, zoneId)); + } + + @Test + public void testMonthIntervalStartEndWithSubSecondOrigin() { + DateBinFunctionColumnTransformer transformer = + new DateBinFunctionColumnTransformer(TimestampType.TIMESTAMP, 1, 0, null, 500, zoneId); + assertArrayEquals( + new long[] {2678400500L, 5097600500L}, transformer.dateBinStartEnd(2678400700L)); + } + + @Test + public void testNextDateBinWithSubSecondOrigin() { + assertEquals(2678400500L, nextDateBin(1, zoneId, 500)); + } + + @Test + public void testMonthIntervalWithSubSecondOriginInMicrosecondPrecision() { + setTimestampPrecision("us"); + assertEquals(1500, dateBin(1500, 1500, 1, 0, zoneId)); + // 2678400000000 us is one month after the epoch + assertEquals(2678400001500L, dateBin(2678400001500L, 1500, 1, 0, zoneId)); + } + + @Test + public void testMonthIntervalWithSubSecondOriginInNanosecondPrecision() { + setTimestampPrecision("ns"); + assertEquals(1_500_000L, dateBin(1_500_000L, 1_500_000L, 1, 0, zoneId)); + } + + private static void setTimestampPrecision(String precision) { + CommonDescriptor.getInstance().getConfig().setTimestampPrecision(precision); + } + + @After + public void after() { + setTimestampPrecision("ms"); + } } From 755a66ddcbfc08a73a743e2243cd43f7f05f9015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:41:24 +0800 Subject: [PATCH 2/2] Saturate date_bin month conversion past the long range TimeUnit.SECONDS.toMillis, toMicros and toNanos already saturate, but the sub-second part was added with an unchecked add, so a month bin past the long range wrapped to a large negative value. Use saturatedAdd so the bin end stays at the long bound. Also cover pre-1970 month bins with a sub-second origin, where getEpochSecond() rounds down and getNano() stays non-negative. --- .../scalar/DateBinFunctionColumnTransformer.java | 12 ++++++++---- .../column/unary/scalar/DateBinFunctionTest.java | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DateBinFunctionColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DateBinFunctionColumnTransformer.java index 2103eaec5f869..5eb0d5a528199 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DateBinFunctionColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DateBinFunctionColumnTransformer.java @@ -91,18 +91,22 @@ private static LocalDateTime convertToLocalDateTime(long timestamp, ZoneId zoneI private static long convertToTimestamp(LocalDateTime dateTime, ZoneId zoneId) { // getEpochSecond() gives the whole seconds and getNano() the nanos inside the second, so the - // milliseconds inside the second are counted exactly once + // milliseconds inside the second are counted exactly once. TimeUnit saturates at the long + // bounds; the sub-second part must use the same saturating add, or a bin past the long range + // wraps to the opposite end. Instant instant = dateTime.atZone(zoneId).toInstant(); long epochSecond = instant.getEpochSecond(); long nanoAdjustment = instant.getNano(); switch (CommonDescriptor.getInstance().getConfig().getTimestampPrecision()) { case "ms": - return TimeUnit.SECONDS.toMillis(epochSecond) + nanoAdjustment / NANOSECONDS_IN_MILLISECOND; + return saturatedAdd( + TimeUnit.SECONDS.toMillis(epochSecond), nanoAdjustment / NANOSECONDS_IN_MILLISECOND); case "us": - return TimeUnit.SECONDS.toMicros(epochSecond) + nanoAdjustment / NANOSECONDS_IN_MICROSECOND; + return saturatedAdd( + TimeUnit.SECONDS.toMicros(epochSecond), nanoAdjustment / NANOSECONDS_IN_MICROSECOND); case "ns": - return TimeUnit.SECONDS.toNanos(epochSecond) + nanoAdjustment; + return saturatedAdd(TimeUnit.SECONDS.toNanos(epochSecond), nanoAdjustment); default: throw new IllegalArgumentException( CalcMessages.EXCEPTION_UNKNOWN_PRECISION_0119BEB0 diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/DateBinFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/DateBinFunctionTest.java index 9d82ccad9b782..1548f27970126 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/DateBinFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/DateBinFunctionTest.java @@ -145,6 +145,10 @@ public void testMonthIntervalWithSubSecondOrigin() throws ParseException { long origin = getTimestamp("2024-03-15 00:00:00.500"); assertEquals(origin, dateBin(origin, origin, 1, 0, zoneId)); assertEquals(origin, dateBin(origin + MILLIS_IN_DAY, origin, 1, 0, zoneId)); + // before 1970, getEpochSecond() rounds toward negative infinity and getNano() stays + // non-negative, so the sub-second origin is preserved on both sides of the epoch + assertEquals(-500L, dateBin(-500L, -500L, 1, 0, zoneId)); + assertEquals(-2678400500L, dateBin(-2678399500L, -500L, 1, 0, zoneId)); } @Test @@ -160,12 +164,23 @@ public void testNextDateBinWithSubSecondOrigin() { assertEquals(2678400500L, nextDateBin(1, zoneId, 500)); } + @Test + public void testMonthBinPastLongRangeSaturates() { + DateBinFunctionColumnTransformer transformer = + new DateBinFunctionColumnTransformer(TimestampType.TIMESTAMP, 1, 0, null, 500, zoneId); + long[] startEnd = transformer.dateBinStartEnd(Long.MAX_VALUE - 1000); + assertEquals(9223372035446400500L, startEnd[0]); + assertEquals(Long.MAX_VALUE, startEnd[1]); + assertEquals(Long.MAX_VALUE, nextDateBin(1, zoneId, startEnd[0])); + } + @Test public void testMonthIntervalWithSubSecondOriginInMicrosecondPrecision() { setTimestampPrecision("us"); assertEquals(1500, dateBin(1500, 1500, 1, 0, zoneId)); // 2678400000000 us is one month after the epoch assertEquals(2678400001500L, dateBin(2678400001500L, 1500, 1, 0, zoneId)); + assertEquals(-1500L, dateBin(-1500L, -1500L, 1, 0, zoneId)); } @Test