From 4c2d4e8612c8bb39c4f9b0f984b2142ba5659d00 Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Mon, 16 Nov 2026 09:49:38 +0800 Subject: [PATCH] Fix microsecond conversion of datetimes before 1970 with a fractional second With timestamp_precision=us, DateTimeUtils.getInstantWithPrecision returned a value 999999 us too large for an instant before the epoch that has a fractional second. Like Instant#toEpochMilli, the branch for such instants multiplies (epochSecond + 1), but it then subtracted 1 us instead of the extra second (1_000_000 us). For example, the literal 1969-12-31T23:59:59.5+00:00 was written and queried as 499999 instead of -500000. getInstantWithPrecisionTest did not call getInstantWithPrecision (its body was a copy of convertDurationStrToLongTest3). It now checks the ms/us/ns branches, and two new tests cover fractional seconds before the epoch, directly and through convertDatetimeStrToLong. Backport of apache/iotdb#19284 (issue apache/iotdb#19283) to dev/1.3: - DateTimeUtils is org.apache.iotdb.db.utils.DateTimeUtils (datanode) on this branch, and getInstantWithPrecision keeps its try/catch. The fix is the same one-line change with the same comment. - The two test assertions that pass "microsecond" as the precision are not ported. On dev/1.3, getInstantWithPrecision accepts only "us" (the "microsecond" alias came with the table model on master), so they return milliseconds and fail with or without the fix. The same inputs are asserted with "us". (cherry picked from commit 8e66bf61ae2ee97150255bf742b3cd2e02a0f6ae) --- .../apache/iotdb/db/utils/DateTimeUtils.java | 5 +- .../iotdb/db/utils/DateTimeUtilsTest.java | 88 +++++++++++++++---- 2 files changed, 76 insertions(+), 17 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java index f4cfa58e2b749..7a37393319091 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java @@ -507,9 +507,10 @@ public static long getInstantWithPrecision(String str, String timestampPrecision Instant instant = zonedDateTime.toInstant(); if ("us".equals(timestampPrecision)) { if (instant.getEpochSecond() < 0 && instant.getNano() > 0) { - // adjustment can reduce the loss of the division + // Same as Instant#toEpochMilli: multiply (epochSecond + 1) so that the product cannot + // overflow near Long.MIN_VALUE, then subtract that extra second (1_000_000 us). long millis = Math.multiplyExact(instant.getEpochSecond() + 1, 1000_000L); - long adjustment = instant.getNano() / 1000 - 1L; + long adjustment = instant.getNano() / 1000 - 1_000_000L; return Math.addExact(millis, adjustment); } else { long millis = Math.multiplyExact(instant.getEpochSecond(), 1000_000L); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java index ea8ae9e0a31ef..dcad9c0f34cd5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java @@ -153,21 +153,79 @@ public void convertDurationStrToLongTest3() { @Test public void getInstantWithPrecisionTest() { - Assert.assertEquals(7000000000L, DateTimeUtils.convertDurationStrToLongForTest(7, "s", "ns")); - Assert.assertEquals(420000000000L, DateTimeUtils.convertDurationStrToLongForTest(7, "m", "ns")); - Assert.assertEquals( - 25200000000000L, DateTimeUtils.convertDurationStrToLongForTest(7, "h", "ns")); - Assert.assertEquals( - 604800000000000L, DateTimeUtils.convertDurationStrToLongForTest(7, "d", "ns")); - Assert.assertEquals( - 4233600000000000L, DateTimeUtils.convertDurationStrToLongForTest(7, "w", "ns")); - Assert.assertEquals( - 18144000000000000L, DateTimeUtils.convertDurationStrToLongForTest(7, "mo", "ns")); - Assert.assertEquals( - 220752000000000000L, DateTimeUtils.convertDurationStrToLongForTest(7, "y", "ns")); - Assert.assertEquals(7000000L, DateTimeUtils.convertDurationStrToLongForTest(7, "ms", "ns")); - Assert.assertEquals(7000L, DateTimeUtils.convertDurationStrToLongForTest(7, "us", "ns")); - Assert.assertEquals(7L, DateTimeUtils.convertDurationStrToLongForTest(7, "ns", "ns")); + // the epoch and later + assertEquals(0L, DateTimeUtils.getInstantWithPrecision("1970-01-01T00:00:00+00:00", "ms")); + assertEquals(0L, DateTimeUtils.getInstantWithPrecision("1970-01-01T00:00:00+00:00", "us")); + assertEquals(0L, DateTimeUtils.getInstantWithPrecision("1970-01-01T00:00:00+00:00", "ns")); + assertEquals( + 500_000L, DateTimeUtils.getInstantWithPrecision("1970-01-01T00:00:00.5+00:00", "us")); + assertEquals( + 1_546_413_207_689L, + DateTimeUtils.getInstantWithPrecision("2019-01-02T15:13:27.689123456+08:00", "ms")); + assertEquals( + 1_546_413_207_689_123L, + DateTimeUtils.getInstantWithPrecision("2019-01-02T15:13:27.689123456+08:00", "us")); + assertEquals( + 1_546_413_207_689_123_456L, + DateTimeUtils.getInstantWithPrecision("2019-01-02T15:13:27.689123456+08:00", "ns")); + + // whole seconds before the epoch + assertEquals(-1_000L, DateTimeUtils.getInstantWithPrecision("1969-12-31T23:59:59+00:00", "ms")); + assertEquals( + -1_000_000L, DateTimeUtils.getInstantWithPrecision("1969-12-31T23:59:59+00:00", "us")); + assertEquals( + -1_000_000_000L, DateTimeUtils.getInstantWithPrecision("1969-12-31T23:59:59+00:00", "ns")); + + // the largest instant that microsecond precision can hold, and the next one + assertEquals( + Long.MAX_VALUE, + DateTimeUtils.getInstantWithPrecision("294247-01-10T04:00:54.775807+00:00", "us")); + Assert.assertThrows( + ArithmeticException.class, + () -> DateTimeUtils.getInstantWithPrecision("294247-01-10T04:00:54.775808+00:00", "us")); + } + + @Test + public void getInstantWithPrecisionBeforeEpochWithFractionTest() { + assertEquals(-500L, DateTimeUtils.getInstantWithPrecision("1969-12-31T23:59:59.5+00:00", "ms")); + assertEquals( + -500_000L, DateTimeUtils.getInstantWithPrecision("1969-12-31T23:59:59.5+00:00", "us")); + assertEquals( + -500_000_000L, DateTimeUtils.getInstantWithPrecision("1969-12-31T23:59:59.5+00:00", "ns")); + // the same instant written with a UTC+8 offset + assertEquals( + -500_000L, DateTimeUtils.getInstantWithPrecision("1970-01-01T07:59:59.5+08:00", "us")); + + assertEquals( + -1L, DateTimeUtils.getInstantWithPrecision("1969-12-31T23:59:59.999999+00:00", "us")); + assertEquals( + -1_000_001L, + DateTimeUtils.getInstantWithPrecision("1969-12-31T23:59:58.999999+00:00", "us")); + assertEquals( + -1L, DateTimeUtils.getInstantWithPrecision("1969-12-31T23:59:59.999999999+00:00", "ns")); + + // digits below the precision are rounded down, the same way for ms and us + assertEquals( + -1L, DateTimeUtils.getInstantWithPrecision("1969-12-31T23:59:59.999999+00:00", "ms")); + assertEquals( + -1_000_000L, + DateTimeUtils.getInstantWithPrecision("1969-12-31T23:59:59.000000001+00:00", "us")); + } + + /** Datetime literals in SQL reach getInstantWithPrecision through this method. */ + @Test + public void convertDatetimeStrToLongBeforeEpochWithFractionTest() { + assertEquals( + -500_000L, + DateTimeUtils.convertDatetimeStrToLong("1969-12-31T23:59:59.5Z", ZoneOffset.UTC, 0, "us")); + assertEquals( + -500_000L, + DateTimeUtils.convertDatetimeStrToLong("1969-12-31 23:59:59.5", ZoneOffset.UTC, 0, "us")); + // a 1970-01-01 local time in a UTC+8 session is still before the epoch + assertEquals( + -500_000L, + DateTimeUtils.convertDatetimeStrToLong( + "1970-01-01T07:59:59.500", ZoneOffset.ofHours(8), 0, "us")); } /** Test convert duration including natural month unit. Time includes: 1970-01-01 ~ 1970-12-01 */