diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 700ab47fa44037..c1d8aac409bec0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -377,6 +377,12 @@ public static boolean isSyncWithPartitions(MTMVRefreshContext context, String mt /** * like p_00000101_20170201 * + * The generated name must be deterministic: the same {@link PartitionKeyDesc} must always yield the same + * name, because the name is generated when the MTMV is created and re-generated later when validating + * a partition refresh request. A time-based suffix (e.g. System.currentTimeMillis()) makes the two + * generations diverge for long names (e.g. TIMESTAMPTZ(6) bounded range boundaries), causing + * "partition not exist" even when the user passes the real partition name from SHOW PARTITIONS. + * * @param desc * @return */ @@ -385,8 +391,10 @@ public static String generatePartitionName(PartitionKeyDesc desc) { String prefix = hasNullPartitionValue(desc) ? PARTITION_NAME_NULL_PREFIX : PARTITION_NAME_PREFIX; String partitionName = prefix + matcher.replaceAll("").replaceAll("\\,", "_"); if (partitionName.length() > 50) { - partitionName = partitionName.substring(0, 30) + Math.abs(Objects.hash(partitionName)) - + "_" + System.currentTimeMillis(); + // truncate and append a stable hash of the full name; no time suffix so repeated generation + // (e.g. MTMV creation vs. partition refresh validation) always produces the same name. + // Cast to long before abs so Integer.MIN_VALUE (whose abs stays negative) cannot slip through. + partitionName = partitionName.substring(0, 30) + Math.abs((long) Objects.hash(partitionName)); } return partitionName; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SessionVarGuardRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SessionVarGuardRewriter.java index a10cd23719ac6b..f0262b0a1f2804 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SessionVarGuardRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SessionVarGuardRewriter.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.pattern.Pattern; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher; @@ -29,9 +30,12 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NeedSessionVarGuard; import org.apache.doris.nereids.trees.expressions.SessionVarGuardExpr; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import org.apache.doris.nereids.types.TimeStampTzType; import com.google.common.collect.ImmutableList; @@ -104,7 +108,7 @@ public AddSessionVarGuardRewriter(Map var) { @Override public Expression visit(Expression expr, Boolean insideGuard) { Expression rewritten = rewriteChildren(this, expr, Boolean.FALSE); - if (rewritten instanceof NeedSessionVarGuard && !Boolean.TRUE.equals(insideGuard)) { + if (needsSessionVarGuard(rewritten) && !Boolean.TRUE.equals(insideGuard)) { if (sessionVar == null) { return expr; } @@ -121,6 +125,30 @@ public Expression visitSessionVarGuardExpr(SessionVarGuardExpr expr, Boolean con } return expr; } + + /** + * An expression needs a session variable guard when either + * 1. it implements {@link NeedSessionVarGuard} (its value depends on some session variable), or + * 2. it is time-zone sensitive: it operates on a TIMESTAMPTZ value. TIMESTAMPTZ is stored as UTC and + * any expression that transforms a TIMESTAMPTZ operand (e.g. date_trunc, cast to varchar/datetime, + * floor functions) yields a value that depends on the session time zone. Without a guard, a + * materialized view built in one time zone could be rewritten in a session with a different time + * zone, returning stale materialized values. + */ + private static boolean needsSessionVarGuard(Expression expr) { + return expr instanceof NeedSessionVarGuard || isTimeZoneSensitive(expr); + } + + private static boolean isTimeZoneSensitive(Expression expr) { + if (expr instanceof Slot || expr instanceof Literal) { + return false; + } + try { + return expr.anyMatch(e -> ((Expression) e).getDataType() instanceof TimeStampTzType); + } catch (UnboundException e) { + return false; + } + } } /** rewrite plan tree */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java index 3346f9c388c9c6..b2cd4f1fba65e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java @@ -97,6 +97,13 @@ private void checkPartitionExist(MTMV mtmv) throws org.apache.doris.common.Analy "The partition method of this asynchronous materialized view " + "does not support refreshing by partition"); } + // First validate against the real physical partition names already stored in the MTMV metadata. + // SHOW PARTITIONS returns these names, and MVs created before partition name generation was made + // deterministic may carry a historical time suffix, so regenerating names here could produce a + // different string than the stored one and wrongly reject a valid refresh request. + Set existPartitionNames = mtmv.getPartitionNames(); + // Secondly validate against the partition names that would be generated (and aligned) from the + // related base table partition descs, so that refreshing a not-yet-created partition is allowed. List partitionDescs = MTMVPartitionUtil.getPartitionDescsByRelatedTable( mtmv.getTableProperty().getProperties(), mtmv.getMvPartitionInfo(), mtmv.getMvProperties(), mtmv.getPartitionColumns()); @@ -105,7 +112,8 @@ private void checkPartitionExist(MTMV mtmv) throws org.apache.doris.common.Analy shouldExistPartitionNames.add(((SinglePartitionDesc) desc).getPartitionName()); }); for (String partition : partitions) { - if (!shouldExistPartitionNames.contains(partition)) { + if (!existPartitionNames.contains(partition) + && !shouldExistPartitionNames.contains(partition)) { throw new org.apache.doris.common.AnalysisException("partition not exist: " + partition); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index a1592de2947489..42b45472c1b6ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1340,7 +1340,13 @@ public void checkQuerySlotCount(String slotCnt) { public int netReadTimeout = 600; // The current time zone - @VarAttrDef.VarAttr(name = TIME_ZONE, needForward = true, affectQueryResultInExecution = true) + // affectQueryResultInPlan is required: TIMESTAMPTZ expressions (date_trunc/cast/floor on timestamptz) + // are evaluated in the session time zone, so the time zone must be captured when persisting session + // variables for views / materialized views / generated columns, and must be compared when deciding + // whether a materialized view can be used for rewrite. Otherwise a MV built in one time zone may be + // rewritten in a session with a different time zone and return stale materialized values. + @VarAttrDef.VarAttr(name = TIME_ZONE, needForward = true, affectQueryResultInPlan = true, + affectQueryResultInExecution = true) public String timeZone = TimeUtils.getSystemTimeZone().getID(); @VarAttrDef.VarAttr(name = LC_TIME_NAMES, needForward = true, affectQueryResultInExecution = true, diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java index 80f91de79125cc..e6be7210cf92b6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java @@ -206,6 +206,38 @@ public void testGeneratePartitionName() { Assert.assertEquals("p_1_2", rangeName); } + /** + * TIMESTAMPTZ(6) bounded range boundaries produce a cleaned name longer than 50 chars, which goes + * through the truncate+hash branch of {@link MTMVPartitionUtil#generatePartitionName}. The generated + * name must be deterministic: the same desc must always yield the same name, otherwise the name + * generated when creating the MTMV differs from the one generated when validating a partition refresh, + * causing "partition not exist" for a partition returned by SHOW PARTITIONS. + */ + @Test + public void testGeneratePartitionNameLongTimestamptzDeterministic() { + PartitionKeyDesc tzDesc = PartitionKeyDesc.createFixed( + Lists.newArrayList(new PartitionValue("2024-01-01 00:00:00.000000+00:00")), + Lists.newArrayList(new PartitionValue("2024-01-02 00:00:00.000000+00:00")) + ); + // ensure the desc really goes through the long-name branch (> 50 chars) + String name = MTMVPartitionUtil.generatePartitionName(tzDesc); + Assert.assertTrue("generated name should be shorter than the cleaned long name", + name.length() < tzDesc.toSql().length()); + Assert.assertTrue(name.length() <= 50); + // repeated generation must produce the identical name (no time-based suffix) + for (int i = 0; i < 10; i++) { + Assert.assertEquals("partition name must be deterministic", name, + MTMVPartitionUtil.generatePartitionName(tzDesc)); + } + // two different descs must not collide + PartitionKeyDesc tzDesc2 = PartitionKeyDesc.createFixed( + Lists.newArrayList(new PartitionValue("2024-01-02 00:00:00.000000+00:00")), + Lists.newArrayList(new PartitionValue("2024-01-03 00:00:00.000000+00:00")) + ); + String name2 = MTMVPartitionUtil.generatePartitionName(tzDesc2); + Assert.assertNotEquals(name, name2); + } + @Test public void testIsTableExcluded() { Set excludedTriggerTables = Sets.newHashSet(new TableNameInfo("table1")); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/VariablePersistTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/VariablePersistTest.java index 31bf7e0a6ccfd2..f8c0b1b8b99ea8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/VariablePersistTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/VariablePersistTest.java @@ -24,8 +24,13 @@ import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; import org.apache.doris.nereids.rules.expression.ExpressionRewriteTestHelper; import org.apache.doris.nereids.rules.expression.MergeGuardExpr; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.TimeStampTzType; +import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.util.MemoTestUtils; import com.google.common.collect.ImmutableList; @@ -212,4 +217,47 @@ public void testMergeGuardExprDeepNesting() { Assertions.assertEquals(child, resultGuard.child()); Assertions.assertEquals(sessionVars, resultGuard.getSessionVars()); } + + /** + * A time-zone sensitive expression (an expression that operates on a TIMESTAMPTZ value, such as + * date_trunc on a timestamptz column) must be guarded when the persisted session variables differ + * from the current session. Otherwise a materialized view built in one time zone could be rewritten + * in a session with a different time zone and return stale materialized values. + */ + @Test + public void testTimeZoneSensitiveExprGetsGuard() { + Map persistSessionVars = ImmutableMap.of("time_zone", "+00:00"); + SessionVarGuardRewriter.AddSessionVarGuardRewriter rewriter = + new SessionVarGuardRewriter.AddSessionVarGuardRewriter(persistSessionVars); + SlotReference tzSlot = new SlotReference("ts", TimeStampTzType.of(6)); + Expression dateTruncOnTz = new DateTrunc(tzSlot, new VarcharLiteral("day")); + Expression rewritten = dateTruncOnTz.accept(rewriter, Boolean.FALSE); + Assertions.assertTrue(rewritten instanceof SessionVarGuardExpr, + "date_trunc on TIMESTAMPTZ should be guarded when session vars differ"); + Assertions.assertEquals(dateTruncOnTz, ((SessionVarGuardExpr) rewritten).child()); + + // cast of a timestamptz value is also time-zone sensitive + Expression castOnTz = new Cast(tzSlot, VarcharType.SYSTEM_DEFAULT); + Expression rewrittenCast = castOnTz.accept(rewriter, Boolean.FALSE); + Assertions.assertTrue(rewrittenCast instanceof SessionVarGuardExpr, + "cast of TIMESTAMPTZ should be guarded when session vars differ"); + } + + /** + * The same function applied to a plain DATETIME/DATE column is NOT time-zone sensitive, so it must not + * be guarded; guarding it would unnecessarily disable materialized view rewrite for a value that does + * not depend on the session time zone. + */ + @Test + public void testDateTimeExprNotGuarded() { + Map persistSessionVars = ImmutableMap.of("time_zone", "+00:00"); + SessionVarGuardRewriter.AddSessionVarGuardRewriter rewriter = + new SessionVarGuardRewriter.AddSessionVarGuardRewriter(persistSessionVars); + SlotReference dtSlot = new SlotReference("dt", DateTimeV2Type.SYSTEM_DEFAULT); + Expression dateTruncOnDt = new DateTrunc(dtSlot, new VarcharLiteral("day")); + Expression rewritten = dateTruncOnDt.accept(rewriter, Boolean.FALSE); + Assertions.assertFalse(rewritten instanceof SessionVarGuardExpr, + "date_trunc on DATETIME should not be guarded"); + Assertions.assertEquals(dateTruncOnDt, rewritten); + } } diff --git a/regression-test/suites/mtmv_p0/test_timestamptz_mtmv_partition_refresh_name.groovy b/regression-test/suites/mtmv_p0/test_timestamptz_mtmv_partition_refresh_name.groovy new file mode 100644 index 00000000000000..afa420be9a548d --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_timestamptz_mtmv_partition_refresh_name.groovy @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +/** + * For a TIMESTAMPTZ(6) bounded-range partition table, the MTMV physical partition name auto-generated by + * Doris is longer than the internal threshold and used to be appended with System.currentTimeMillis(). + * That made the name generated when validating a partition refresh differ from the real stored name, so + * refreshing by the real partition name returned by SHOW PARTITIONS failed with "partition not exist". + * The generated name must be deterministic and the refresh validation must accept the real stored name. + */ +suite("test_timestamptz_mtmv_partition_refresh_name","mtmv") { + def dbName = "timestamptz_mtmv_partition_refresh_name" + def tableName = "timestamptz_mtmv_partition_refresh_name_table" + def mvName = "timestamptz_mtmv_partition_refresh_name_mv" + + sql "DROP DATABASE IF EXISTS ${dbName}" + sql "CREATE DATABASE ${dbName}" + sql "USE ${dbName}" + + sql "SET enable_nereids_planner = true" + sql "SET enable_fallback_to_original_planner = false" + sql "SET time_zone = '+00:00'" + + // TIMESTAMPTZ(6) with two bounded range partitions: the generated MTMV partition name is long. + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + id INT, + ts TIMESTAMPTZ(6) NOT NULL + ) + DUPLICATE KEY(id) + PARTITION BY RANGE(ts) ( + PARTITION p1 VALUES [('2024-01-01 00:00:00+00:00'), ('2024-01-02 00:00:00+00:00')), + PARTITION p2 VALUES [('2024-01-02 00:00:00+00:00'), ('2024-01-03 00:00:00+00:00')) + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num' = '1') + """ + sql """ + INSERT INTO ${tableName} VALUES + (1, '2024-01-01 00:30:00+00:00'), + (2, '2024-01-02 00:30:00+00:00') + """ + sql "sync" + + sql "DROP MATERIALIZED VIEW IF EXISTS ${mvName}" + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + PARTITION BY(ts) + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES('replication_num' = '1') + AS SELECT id, ts FROM ${tableName} + """ + + // The real physical partition names returned by SHOW PARTITIONS. + def partitions = sql "SHOW PARTITIONS FROM ${mvName}" + Assert.assertTrue("expected at least one partition, got " + partitions.size(), partitions.size() >= 1) + def realName = partitions[0][1].toString() + Assert.assertFalse(realName.isEmpty()) + + // Refreshing by the real partition name must be accepted (previously "partition not exist"). + sql "REFRESH MATERIALIZED VIEW ${mvName} PARTITIONS(`" + realName + "`)" + waitingMTMVTaskFinishedByMvName(mvName, dbName) + + // Only the refreshed partition carries data (BUILD DEFERRED + partial refresh). + def mvRes = sql "SELECT id, CAST(ts AS STRING) FROM ${mvName} ORDER BY id" + Assert.assertEquals(1, mvRes.size()) + Assert.assertTrue(mvRes[0][1].toString().contains("2024-01-01 00:30:00.000000+00:00")) + + // AUTO refresh fills the remaining partition(s). + sql "REFRESH MATERIALIZED VIEW ${mvName} AUTO" + waitingMTMVTaskFinishedByMvName(mvName, dbName) + def mvResAuto = sql "SELECT id, CAST(ts AS STRING) FROM ${mvName} ORDER BY id" + Assert.assertEquals(2, mvResAuto.size()) + Assert.assertTrue(mvResAuto[0][1].toString().contains("2024-01-01 00:30:00.000000+00:00")) + Assert.assertTrue(mvResAuto[1][1].toString().contains("2024-01-02 00:30:00.000000+00:00")) + + // DATETIME control group: refresh by real partition name also works. + def dtTable = "timestamptz_mtmv_partition_refresh_name_dt_table" + def dtMv = "timestamptz_mtmv_partition_refresh_name_dt_mv" + sql "DROP TABLE IF EXISTS ${dtTable}" + sql """ + CREATE TABLE ${dtTable} ( + id INT, + ts DATETIME(6) NOT NULL + ) + DUPLICATE KEY(id) + PARTITION BY RANGE(ts) ( + PARTITION p1 VALUES [('2024-01-01 00:00:00'), ('2024-01-02 00:00:00')), + PARTITION p2 VALUES [('2024-01-02 00:00:00'), ('2024-01-03 00:00:00')) + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num' = '1') + """ + sql """ + INSERT INTO ${dtTable} VALUES + (1, '2024-01-01 00:30:00'), + (2, '2024-01-02 00:30:00') + """ + sql "sync" + sql "DROP MATERIALIZED VIEW IF EXISTS ${dtMv}" + sql """ + CREATE MATERIALIZED VIEW ${dtMv} + BUILD DEFERRED REFRESH AUTO ON MANUAL + PARTITION BY(ts) + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES('replication_num' = '1') + AS SELECT id, ts FROM ${dtTable} + """ + def dtPartitions = sql "SHOW PARTITIONS FROM ${dtMv}" + def dtRealName = dtPartitions[0][1].toString() + sql "REFRESH MATERIALIZED VIEW ${dtMv} PARTITIONS(`" + dtRealName + "`)" + waitingMTMVTaskFinishedByMvName(dtMv, dbName) + def dtRes = sql "SELECT COUNT(*) FROM ${dtMv}" + Assert.assertEquals(1, dtRes[0][0]) +} diff --git a/regression-test/suites/mtmv_p0/test_timestamptz_partition_mtmv_refresh_timezone.groovy b/regression-test/suites/mtmv_p0/test_timestamptz_partition_mtmv_refresh_timezone.groovy new file mode 100644 index 00000000000000..9f268035bfd0b1 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_timestamptz_partition_mtmv_refresh_timezone.groovy @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +/** + * When an asynchronous partitioned materialized view is created in a session with a non-default time zone + * and its partition key is a time-zone sensitive expression (date_trunc on a TIMESTAMPTZ column), the + * background refresh must run with the SAME session time zone that was used to derive the MV partition + * boundaries. Otherwise the refresh computes a partition key that does not fall into any MV partition + * ("no partition for this tuple") and the MV stays empty. + */ +suite("test_timestamptz_partition_mtmv_refresh_timezone","mtmv") { + def dbName = "timestamptz_partition_mtmv_refresh_timezone" + def tableName = "timestamptz_partition_mtmv_refresh_timezone_table" + def mvName = "timestamptz_partition_mtmv_refresh_timezone_mv" + + sql "DROP DATABASE IF EXISTS ${dbName}" + sql "CREATE DATABASE ${dbName}" + sql "USE ${dbName}" + + sql "SET enable_nereids_planner = true" + sql "SET enable_fallback_to_original_planner = false" + sql "SET time_zone = '+00:00'" + + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + id INT, + ts TIMESTAMPTZ(6), + v INT + ) + DUPLICATE KEY(id) + PARTITION BY RANGE(ts) ( + PARTITION p0 VALUES [('2024-01-01 00:00:00+00:00'), ('2024-01-02 00:00:00+00:00')), + PARTITION p1 VALUES [('2024-01-02 00:00:00+00:00'), ('2024-01-03 00:00:00+00:00')) + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num' = '1') + """ + + sql """ + INSERT INTO ${tableName} VALUES + (1, '2024-01-01 00:30:00+00:00', 10) + """ + sql "sync" + + // The base table computed in the UTC session returns the UTC day boundary. + def baseRes = sql """ + SELECT CAST(ts AS STRING), CAST(date_trunc(ts, 'day') AS STRING), v + FROM ${tableName} + """ + Assert.assertEquals(1, baseRes.size()) + Assert.assertTrue("expected 2024-01-01 00:00:00.000000+00:00, got " + baseRes[0][1], + baseRes[0][1].toString().contains("2024-01-01 00:00:00.000000+00:00")) + + sql "DROP MATERIALIZED VIEW IF EXISTS ${mvName}" + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY(day_ts) + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES('replication_num' = '1') + AS + SELECT date_trunc(ts, 'day') AS day_ts, SUM(v) AS total + FROM ${tableName} + GROUP BY date_trunc(ts, 'day') + """ + def jobName = getJobName(dbName, mvName) + waitingMTMVTaskFinishedByMvName(mvName, dbName) + + // The refresh task must be SUCCESS, not FAILED. + def tasks = sql """ + SELECT Status, MvName, ErrorMsg + FROM tasks('type' = 'mv') + WHERE MvDatabaseName = '${dbName}' AND MvName = '${mvName}' + ORDER BY CreateTime DESC + LIMIT 1 + """ + Assert.assertEquals("SUCCESS", tasks[0][0].toString()) + + // The MTMV must be NORMAL / SUCCESS and contain the row that falls into the UTC day partition. + def mvInfos = sql """ + SELECT Name, State, RefreshState + FROM mv_infos('database' = '${dbName}') + WHERE Name = '${mvName}' + """ + Assert.assertEquals("NORMAL", mvInfos[0][1].toString()) + Assert.assertEquals("SUCCESS", mvInfos[0][2].toString()) + + def mvRes = sql "SELECT CAST(day_ts AS STRING), total FROM ${mvName} ORDER BY 1" + Assert.assertEquals(1, mvRes.size()) + Assert.assertTrue("expected 2024-01-01 00:00:00.000000+00:00, got " + mvRes[0][0], + mvRes[0][0].toString().contains("2024-01-01 00:00:00.000000+00:00")) + Assert.assertEquals(10, mvRes[0][1]) +} diff --git a/regression-test/suites/mtmv_p0/test_timestamptz_sync_mv_rewrite_timezone.groovy b/regression-test/suites/mtmv_p0/test_timestamptz_sync_mv_rewrite_timezone.groovy new file mode 100644 index 00000000000000..624a88a868d113 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_timestamptz_sync_mv_rewrite_timezone.groovy @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +/** + * A synchronous materialized view that materializes a time-zone sensitive TIMESTAMPTZ expression + * (e.g. date_trunc on a timestamptz column) is only valid in the session time zone it was built in. + * The query optimizer must not rewrite a query to such an MV when the query session time zone differs + * from the MV creation session time zone, otherwise the query returns the stale materialized value + * instead of the query-session semantics. + */ +suite("test_timestamptz_sync_mv_rewrite_timezone","mtmv") { + def tableName = "timestamptz_sync_mv_rewrite_timezone_table" + def mvName = "timestamptz_sync_mv_rewrite_timezone_mv" + + sql "SET enable_nereids_planner = true" + sql "SET enable_fallback_to_original_planner = false" + + // A sync MV is an index on the base table: dropping the table drops the MV too. We must NOT run + // `DROP MATERIALIZED VIEW ... ON ` here because that statement requires the table to exist, + // which it may not in a fresh test database. + sql "DROP TABLE IF EXISTS ${tableName}" + + // Build the base table and the sync MV in a UTC session. + sql "SET time_zone = '+00:00'" + sql """ + CREATE TABLE ${tableName} ( + id INT, + ts TIMESTAMPTZ(6), + v INT + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num' = '1') + """ + sql """ + INSERT INTO ${tableName} VALUES + (1, '2024-01-01 00:30:00+00:00', 10) + """ + sql "sync" + + create_sync_mv(context.dbName, tableName, mvName, """ + SELECT date_trunc(ts, 'day') AS day_ts, SUM(v) AS day_sum + FROM ${tableName} + WHERE ts IS NOT NULL + GROUP BY date_trunc(ts, 'day') + """) + + // Query in a different (+08:00) session. + sql "SET time_zone = '+08:00'" + + // Without rewrite, the query computes date_trunc in the query session time zone. + sql "SET enable_materialized_view_rewrite=false" + def resRewriteOff = sql """ + SELECT CAST(date_trunc(ts, 'day') AS STRING) AS day_ts, SUM(v) + FROM ${tableName} + WHERE ts IS NOT NULL + GROUP BY date_trunc(ts, 'day') + """ + // With rewrite enabled, the result must be identical; the UTC-built MV must not be used. + sql "SET enable_materialized_view_rewrite=true" + def resRewriteOn = sql """ + SELECT CAST(date_trunc(ts, 'day') AS STRING) AS day_ts, SUM(v) + FROM ${tableName} + WHERE ts IS NOT NULL + GROUP BY date_trunc(ts, 'day') + """ + Assert.assertEquals(resRewriteOff, resRewriteOn) + Assert.assertEquals(1, resRewriteOn.size()) + Assert.assertTrue("expected 2024-01-01 00:00:00.000000+08:00, got " + resRewriteOn[0][0], + resRewriteOn[0][0].toString().contains("2024-01-01 00:00:00.000000+08:00")) + Assert.assertEquals(10, resRewriteOn[0][1]) + + // The MV built in a UTC session must not be chosen for a +08:00 session query. + mv_rewrite_fail(""" + SELECT CAST(date_trunc(ts, 'day') AS STRING), SUM(v) + FROM ${tableName} + WHERE ts IS NOT NULL + GROUP BY date_trunc(ts, 'day') + """, mvName) + + // In the SAME (+08:00) session, an MV built in this session rewrites correctly and keeps results equal. + create_sync_mv(context.dbName, tableName, mvName, """ + SELECT date_trunc(ts, 'day') AS day_ts, SUM(v) AS day_sum + FROM ${tableName} + WHERE ts IS NOT NULL + GROUP BY date_trunc(ts, 'day') + """) + def resSameTz = sql """ + SELECT CAST(date_trunc(ts, 'day') AS STRING) AS day_ts, SUM(v) + FROM ${tableName} + WHERE ts IS NOT NULL + GROUP BY date_trunc(ts, 'day') + """ + Assert.assertEquals(resRewriteOn, resSameTz) +}