diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 9766a175a56749..4589e665d9387e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -267,7 +267,8 @@ public void run() throws JobException { try { executeWithRetry(execPartitionNames, tableWithPartKey); } catch (Exception e) { - LOG.error("Execution failed after retries: {}", e.getMessage()); + LOG.error("Execution failed after retries, mvName: {}, taskId: {}", + mtmv.getName(), getTaskId(), e); throw new JobException(e.getMessage(), e); } completedPartitions.addAll(execPartitionNames); @@ -277,7 +278,8 @@ public void run() throws JobException { mtmv.getDatabase().getFullName(), mtmv.getName(), getTaskId()); } catch (Throwable e) { if (getStatus() == TaskStatus.RUNNING) { - LOG.warn("run task failed: {}", e.getMessage()); + LOG.warn("run task failed, mvName: {}, taskId: {}", + mtmv.getName(), getTaskId(), e); throw new JobException(e.getMessage(), e); } else { // if status is not `RUNNING`,maybe the task was canceled, therefore, it is a normal situation diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 4be21db3008033..c0e38c6adb4311 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -111,6 +111,7 @@ public class MTMVPlanUtil { RuleType.ELIMINATE_JOIN_BY_FK, RuleType.ELIMINATE_JOIN_BY_UK, RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, + RuleType.ELIMINATE_GROUP_BY_KEY, RuleType.ELIMINATE_GROUP_BY, RuleType.SALT_JOIN ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index e651a9e8fae583..162289c33553c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -678,7 +678,6 @@ public class Rewriter extends AbstractBatchJobExecutor { cascadesContext -> cascadesContext.rewritePlanContainsTypes(LogicalAggregate.class) || cascadesContext.rewritePlanContainsTypes(LogicalJoin.class) || cascadesContext.rewritePlanContainsTypes(LogicalUnion.class), - topDown(new EliminateGroupByKey()), topDown(new PushDownAggThroughJoinOnPkFk()), topDown(new PullUpJoinFromUnionAll()) ), @@ -916,6 +915,11 @@ private static List getWholeTreeRewriteJobs( ))); rewriteJobs.addAll(jobs(topic("convert outer join to anti", custom(RuleType.CONVERT_OUTER_JOIN_TO_ANTI, ConvertOuterJoinToAntiJoin::new)))); + rewriteJobs.addAll(jobs(topic("eliminate Aggregate according to fd items", + cascadesContext -> cascadesContext.rewritePlanContainsTypes(LogicalAggregate.class) + || cascadesContext.rewritePlanContainsTypes(LogicalJoin.class) + || cascadesContext.rewritePlanContainsTypes(LogicalUnion.class), + custom(RuleType.ELIMINATE_GROUP_BY_KEY, EliminateGroupByKey::new)))); rewriteJobs.addAll(jobs(topic("eliminate group by key by uniform", custom(RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, EliminateGroupByKeyByUniform::new)))); if (needOrExpansion) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java index c08d65e777f8e7..8082771e97ea27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java @@ -68,6 +68,7 @@ public class PreMaterializedViewRewriter { NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.DISTINCT_AGGREGATE_SPLIT.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PROCESS_SCALAR_AGG_MUST_USE_MULTI_DISTINCT.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM.ordinal()); + NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.ELIMINATE_GROUP_BY_KEY.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.SALT_JOIN.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PULL_UP_PROJECT_EXPR_UNDER_TOPN.ordinal()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java index 2b77c7d927945b..02793a0aa341d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java @@ -231,10 +231,10 @@ public Rule build() { List groupByExprs = agg.getGroupByExpressions(); ExpressionRewriteContext context = new ExpressionRewriteContext(agg, ctx.cascadesContext); List newGroupByExprs = rewriter.rewrite(groupByExprs, context); - + boolean groupByChanged = !newGroupByExprs.equals(groupByExprs); List outputExpressions = agg.getOutputExpressions(); RewriteResult result = rewriteAll(outputExpressions, rewriter, context); - if (!result.changed) { + if (!result.changed && !groupByChanged) { return agg; } return new LogicalAggregate<>(newGroupByExprs, result.result, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java index 4e1b3117ab53ff..cff93b15b2b58b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java @@ -17,90 +17,230 @@ package org.apache.doris.nereids.rules.rewrite; -import org.apache.doris.nereids.annotation.DependsRules; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.properties.FuncDeps; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.Multimap; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; - /** * Eliminate group by key based on fd item information. * such as: * for a -> b, we can get: * group by a, b, c => group by a, c + * + * When a group-by key is FD-redundant but still needed in the output, + * it is wrapped with any_value() and assigned a fresh ExprId. + * Upper plan references are rewritten via ExprIdRewriter so that + * all ancestor nodes see the new ExprIds. */ -@DependsRules({EliminateGroupBy.class, ColumnPruning.class}) -public class EliminateGroupByKey implements RewriteRuleFactory { +public class EliminateGroupByKey extends DefaultPlanRewriter> implements CustomRewriter { + private ExprIdRewriter exprIdReplacer; + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + if (!plan.containsType(Aggregate.class)) { + return plan; + } + Map replaceMap = new HashMap<>(); + ExprIdRewriter.ReplaceRule replaceRule = new ExprIdRewriter.ReplaceRule(replaceMap, false); + exprIdReplacer = new ExprIdRewriter(replaceRule, jobContext); + return plan.accept(this, replaceMap); + } + + @Override + public Plan visit(Plan plan, Map replaceMap) { + plan = visitChildren(this, plan, replaceMap); + plan = exprIdReplacer.rewriteExpr(plan, replaceMap); + return plan; + } + + @Override + public Plan visitLogicalProject(LogicalProject proj, Map replaceMap) { + proj = visitChildren(this, proj, replaceMap); + + // Find the Aggregate child, possibly through a Filter + Plan child = proj.child(0); + LogicalAggregate agg; + boolean hasFilter = child instanceof LogicalFilter; + if (hasFilter && child.child(0) instanceof LogicalAggregate) { + agg = (LogicalAggregate) child.child(0); + } else if (child instanceof LogicalAggregate) { + agg = (LogicalAggregate) child; + } else { + return exprIdReplacer.rewriteExpr(proj, replaceMap); + } + + // Don't transform if source repeat is present + if (agg.getSourceRepeat().isPresent()) { + return exprIdReplacer.rewriteExpr(proj, replaceMap); + } + + // Rewrite proj and the filter (if present) through the replaceMap accumulated + // by visitChildren, so that ExprId replacements from nested rewrites + // (e.g. inner aggregates) are reflected in the required-output slot set. + proj = (LogicalProject) exprIdReplacer.rewriteExpr(proj, replaceMap); + if (hasFilter) { + child = exprIdReplacer.rewriteExpr(child, replaceMap); + } + + // Compute requireOutput: slots needed by the Project (and Filter, if present) + Set requireOutput = new HashSet<>(proj.getInputSlots()); + if (hasFilter) { + requireOutput.addAll(child.getInputSlots()); + } + + // Transform the aggregate + EliminateResult result = eliminateGroupByKeyWithMap(agg, requireOutput); + if (!result.changed) { + return proj; + } + + // Merge into the global replaceMap so that all ancestor nodes get rewritten + replaceMap.putAll(result.replaceMap); + + // Rebuild the child chain with the new aggregate, + // and rewrite the Filter (if present) and Project expressions + Plan newChild; + if (hasFilter) { + Plan updatedFilter = child.withChildren(result.newAgg); + newChild = exprIdReplacer.rewriteExpr(updatedFilter, replaceMap); + } else { + newChild = result.newAgg; + } + Plan newProj = exprIdReplacer.rewriteExpr(proj.withChildren(newChild), replaceMap); + return newProj; + } @Override - public List buildRules() { - return ImmutableList.of( - RuleType.ELIMINATE_GROUP_BY_KEY.build( - logicalProject(logicalAggregate().when(agg -> !agg.getSourceRepeat().isPresent())) - .then(proj -> { - LogicalAggregate agg = proj.child(); - LogicalAggregate newAgg = eliminateGroupByKey(agg, proj.getInputSlots()); - if (newAgg == null) { - return null; - } - return proj.withChildren(newAgg); - })), - RuleType.ELIMINATE_FILTER_GROUP_BY_KEY.build( - logicalProject(logicalFilter(logicalAggregate() - .when(agg -> !agg.getSourceRepeat().isPresent()))) - .then(proj -> { - LogicalAggregate agg = proj.child().child(); - Set requireSlots = new HashSet<>(proj.getInputSlots()); - requireSlots.addAll(proj.child(0).getInputSlots()); - LogicalAggregate newAgg = eliminateGroupByKey(agg, requireSlots); - if (newAgg == null) { - return null; - } - return proj.withChildren(proj.child().withChildren(newAgg)); - }) - ) - ); + public Plan visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, Map replaceMap) { + // When a producer aggregate's output slot is wrapped with any_value(), + // a fresh ExprId is recorded in replaceMap. The CTE consumer's producerToConsumerSlotMap + // still references the old ExprId, so we must rebuild both maps with the new ExprIds. + Map newConsumerToProducer = new LinkedHashMap<>(); + Multimap newProducerToConsumer = LinkedHashMultimap.create(); + for (Slot producerSlot : cteConsumer.getConsumerToProducerOutputMap().values()) { + ExprId newExprId = resolveExprIdChain(producerSlot.getExprId(), replaceMap); + Slot effectiveProducerSlot = newExprId != null + ? (Slot) producerSlot.withExprId(newExprId) + : producerSlot; + for (Slot consumerSlot : cteConsumer.getProducerToConsumerOutputMap().get(producerSlot)) { + newProducerToConsumer.put(effectiveProducerSlot, consumerSlot); + newConsumerToProducer.put(consumerSlot, effectiveProducerSlot); + } + } + return cteConsumer.withTwoMaps(newConsumerToProducer, newProducerToConsumer); + } + + /** Follow transitive ExprId chain to find the final replacement, or null if none. */ + private static ExprId resolveExprIdChain(ExprId exprId, Map replaceMap) { + ExprId newId = replaceMap.get(exprId); + if (newId == null) { + return null; + } + ExprId lastId = newId; + while (true) { + ExprId next = replaceMap.get(lastId); + if (next == null) { + return lastId; + } + lastId = next; + } + } + + /** Result of eliminateGroupByKey: the new aggregate and a map of old->new ExprIds. */ + private static class EliminateResult { + final LogicalAggregate newAgg; + final Map replaceMap; + final boolean changed; + + EliminateResult(LogicalAggregate newAgg, Map replaceMap, boolean changed) { + this.newAgg = newAgg; + this.replaceMap = replaceMap; + this.changed = changed; + } } - LogicalAggregate eliminateGroupByKey(LogicalAggregate agg, Set requireOutput) { - Set removeExpression = findCanBeRemovedExpressions(agg, requireOutput, + EliminateResult eliminateGroupByKeyWithMap(LogicalAggregate agg, Set requireOutput) { + FindResult result = findCanBeRemovedExpressionsInternal(agg, requireOutput, agg.child().getLogicalProperties().getTrait()); + Set removeExpression = result.removeExpression; + Set wrapWithAnyValue = result.wrapWithAnyValue; + List newGroupExpression = new ArrayList<>(); for (Expression expression : agg.getGroupByExpressions()) { - if (!removeExpression.contains(expression)) { + if (!removeExpression.contains(expression) + && !wrapWithAnyValue.contains(expression)) { newGroupExpression.add(expression); } } List newOutput = new ArrayList<>(); + Map replaceMap = new HashMap<>(); + boolean changed = !removeExpression.isEmpty() || !wrapWithAnyValue.isEmpty(); for (NamedExpression expression : agg.getOutputExpressions()) { - if (!removeExpression.contains(expression)) { - newOutput.add(expression); + if (removeExpression.contains(expression)) { + continue; } + if (wrapWithAnyValue.contains(expression)) { + // expression is FD-redundant but needed in output: wrap with any_value + // Use fresh ExprId (auto-generated by Alias) to avoid ExprId collision, + // and record the mapping for rewriting upper plan references. + Alias newAlias = new Alias(new AnyValue(expression.toSlot()), expression.getName()); + replaceMap.put(expression.getExprId(), newAlias.getExprId()); + expression = newAlias; + } + newOutput.add(expression); } - return agg.withGroupByAndOutput(newGroupExpression, newOutput); + return new EliminateResult(agg.withGroupByAndOutput(newGroupExpression, newOutput), replaceMap, changed); } /** - * return removeExpression + * Return expressions that can be completely removed from both group-by and output. + * Kept for backward compatibility with external callers (e.g. PushDownAggThroughJoinOnPkFk). */ public static Set findCanBeRemovedExpressions(LogicalAggregate agg, Set requireOutput, DataTrait dataTrait) { + FindResult result = findCanBeRemovedExpressionsInternal(agg, requireOutput, dataTrait); + return new HashSet<>(result.removeExpression); + } + + /** Result of findCanBeRemovedExpressionsInternal: two sets of expressions. */ + private static class FindResult { + final Set removeExpression; // remove from group-by and output + final Set wrapWithAnyValue; // remove from group-by, wrap with ANY_VALUE in output + + FindResult(Set removeExpression, Set wrapWithAnyValue) { + this.removeExpression = removeExpression; + this.wrapWithAnyValue = wrapWithAnyValue; + } + } + + private static FindResult findCanBeRemovedExpressionsInternal(LogicalAggregate agg, + Set requireOutput, DataTrait dataTrait) { Map> groupBySlots = new HashMap<>(); Set validSlots = new HashSet<>(); for (Expression expression : agg.getGroupByExpressions()) { @@ -110,17 +250,24 @@ public static Set findCanBeRemovedExpressions(LogicalAggregate(); + return new FindResult(new HashSet<>(), new HashSet<>()); } Set> minGroupBySlots = funcDeps.eliminateDeps(new HashSet<>(groupBySlots.values()), requireOutput); Set removeExpression = new HashSet<>(); + Set wrapWithAnyValue = new HashSet<>(); for (Entry> entry : groupBySlots.entrySet()) { - if (!minGroupBySlots.contains(entry.getValue()) - && !requireOutput.containsAll(entry.getValue())) { - removeExpression.add(entry.getKey()); + if (!minGroupBySlots.contains(entry.getValue())) { + // FD redundant: can remove from group-by + if (!requireOutput.containsAll(entry.getValue())) { + // Not needed in output either: remove completely + removeExpression.add(entry.getKey()); + } else { + // Still needed in output: remove from group-by, wrap with ANY_VALUE in output + wrapWithAnyValue.add(entry.getKey()); + } } } - return removeExpression; + return new FindResult(removeExpression, wrapWithAnyValue); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java index c781ce1aa1b10d..d1485fd036e0f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java @@ -76,7 +76,7 @@ public static Plan rewrite(LogicalAggregate agg, DistinctSelecto // construct cte consumer and aggregate List> newAggs = new ArrayList<>(); // All otherAggFuncs are placed in the first one - Map newToOriginDistinctFuncAlias = new HashMap<>(); + Map newToOriginDistinctFuncAlias = new LinkedHashMap<>(); List outputJoinGroupBys = new ArrayList<>(); for (int i = 0; i < distinctFuncWithAliasReplaced.size(); ++i) { List aliases = distinctFuncWithAliasReplaced.get(i); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 1e79c0ed3cfbda..cba8e7a3c46218 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -75,7 +75,7 @@ public class CreateMTMVInfo extends CreateTableInfo { public static final Logger LOG = LogManager.getLogger(CreateMTMVInfo.class); public static final String MTMV_PLANER_DISABLE_RULES = "OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION," - + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM"; + + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, ELIMINATE_GROUP_BY_KEY"; private LogicalPlan logicalQuery; private List simpleColumnDefinitions; private MTMVPartitionDefinition mvPartitionDefinition; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java index 6dc5190da3deb5..a408f92af3e788 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java @@ -59,7 +59,7 @@ public class PreMaterializedViewRewriterTest extends SqlTestBase { @Test public void testShouldNotRecordTmpPlanWhenNoMv() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION, ELIMINATE_GROUP_BY_KEY"); BitSet disableNereidsRules = connectContext.getSessionVariable().getDisableNereidsRules(); SessionVariable spySv = Mockito.spy(connectContext.getSessionVariable()); Mockito.doReturn(disableNereidsRules).when(spySv).getDisableNereidsRules(); @@ -2952,6 +2952,19 @@ public void testNeedPreRewrite() { Assertions.assertTrue(PreMaterializedViewRewriter.needPreRewrite(cascadesContext)); } + /** + * Test pre-materialized view rewrite need pre-rewrite when ELIMINATE_GROUP_BY_KEY applied + * */ + @Test + public void testNeedPreRewriteForEliminateGroupByKey() { + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext("select T1.id from T1"); + StatementContext statementContext = cascadesContext.getConnectContext().getStatementContext(); + statementContext.setForceRecordTmpPlan(true); + statementContext.ruleSetApplied(RuleType.ELIMINATE_GROUP_BY_KEY); + statementContext.getPlannerHooks().add(InitMaterializationContextHook.INSTANCE); + statementContext.getTmpPlanForMvRewrite().add(cascadesContext.getRewritePlan()); + } + private void checkIfEquals(String originalSql, List equivalentSqlList) { // init original cascades context CascadesContext originalCascadesContext = initOriginal(originalSql); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java index 27143e16406041..db357bed4417a9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java @@ -262,7 +262,7 @@ protected void runBeforeAll() throws Exception { connectContext.getSessionVariable().setDisableNereidsRules( "OLAP_SCAN_PARTITION_PRUNE" + ",PRUNE_EMPTY_PARTITION" - + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + ",ELIMINATE_GROUP_BY_KEY" + ",ELIMINATE_CONST_JOIN_CONDITION" + ",CONSTANT_PROPAGATION" ); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java index 6e6df0909ad2d9..d5748699acc869 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java @@ -135,7 +135,8 @@ void testLeftJoinOnConditionNotRewrite() { .analyze("select t1.b,t2.b from eli_gbk_by_uniform_t t1 left join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t1.b,t2.b,t2.c;") .rewrite() .printlnTree() - .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 3)); + .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 2 + && agg.getGroupByExpressions().get(0).toSql().equals("b"))); } @Test @@ -144,7 +145,8 @@ void testLeftJoinWhereConditionRewrite() { .analyze("select t1.b,t2.b from eli_gbk_by_uniform_t t1 left join eli_gbk_by_uniform_t t2 on t1.b=t2.b where t1.b=100 group by t1.b,t2.b,t2.c;") .rewrite() .printlnTree() - .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 2)); + .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("c"))); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java index 7362c81e5afe0b..a470a1d7a77907 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java @@ -18,18 +18,29 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.properties.FuncDeps; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.HashMap; +import java.util.Map; import java.util.Set; class EliminateGroupByKeyTest extends TestWithFeService implements MemoPatternMatchSupported { @@ -97,19 +108,22 @@ void testEliminateTree() { @Test void testEliminateByUniform() { + // Uniform-based elimination is now handled by EliminateGroupByKeyByUniform. + // EliminateGroupByKey only handles FD-based elimination. PlanChecker.from(connectContext) .analyze("select count(name) from t1 where id = 1 group by name, id") - .rewrite() + .customRewrite(new EliminateGroupByKeyByUniform()) .printlnTree() .matches(logicalAggregate().when(agg -> - agg.getGroupByExpressions().size() == 1 && agg.getGroupByExpressions().get(0).toSql().equals("name"))); + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("name"))); } @Test void testProjectAlias() { PlanChecker.from(connectContext) .analyze("select id as c from t1 where id = 1 group by name, id") - .rewrite() + .customRewrite(new EliminateGroupByKey()) .printlnTree() .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1)); @@ -181,6 +195,120 @@ void testEliminateByEqual() { && agg.getGroupByExpressions().get(0).toSql().equals("name"))); } + @Test + void testEliminateByPkWithOutputNeeded() throws Exception { + // Regression: when a group-by key (name) is FD-redundant (id -> name) + // but still appears in SELECT, it should be removed from group-by + // and wrapped with ANY_VALUE in the output. + addConstraint("alter table t1 add constraint pk2 primary key (id)"); + PlanChecker.from(connectContext) + .analyze("select id, name, count(*) from t1 group by id, name") + .customRewrite(new EliminateGroupByKey()) + .printlnTree() + .matches(logicalAggregate().when(agg -> + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("id") + && agg.getOutputExpressions().stream().anyMatch( + e -> e instanceof Alias + && e.child(0) instanceof AnyValue))); + dropConstraint("alter table t1 drop constraint pk2"); + } + + @Test + void testEliminateByPkWithOutputNeededProductionPath() throws Exception { + // Production path: same query through .rewrite() (RuleType.ELIMINATE_GROUP_BY_KEY) + // instead of .customRewrite() (RuleType.TEST_REWRITE). + // Use cross join so the aggregate cannot be constant-folded away. + // Use alias on name to force a Project above the Aggregate, which is + // the entry point that EliminateGroupByKey.visitLogicalProject needs. + addConstraint("alter table t1 add constraint pk2 primary key (id)"); + PlanChecker.from(connectContext) + .analyze("select t1.id, t1.name as n, count(*) from t1 as t1" + + " cross join t1 as t2 group by t1.id, t1.name") + .rewrite() + .printlnTree() + .matches(logicalAggregate().when(agg -> + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("id") + && agg.getOutputExpressions().stream().anyMatch( + e -> e instanceof Alias + && e.child(0) instanceof AnyValue))); + dropConstraint("alter table t1 drop constraint pk2"); + } + + @Test + void testEliminateByPkDisabled() throws Exception { + // Verify that disable_nereids_rules=ELIMINATE_GROUP_BY_KEY prevents the rule + // from eliminating the FD-redundant group-by key. + // Use cross join so the aggregate cannot be constant-folded away. + addConstraint("alter table t1 add constraint pk2 primary key (id)"); + try { + connectContext.getSessionVariable() + .setDisableNereidsRules("PRUNE_EMPTY_PARTITION,ELIMINATE_GROUP_BY_KEY"); + PlanChecker.from(connectContext) + .analyze("select t1.id, t1.name, count(*) from t1 as t1" + + " cross join t1 as t2 group by t1.id, t1.name") + .rewrite() + .printlnTree() + .matches(logicalAggregate().when(agg -> + agg.getGroupByExpressions().size() == 2)); + } finally { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + dropConstraint("alter table t1 drop constraint pk2"); + } + } + + @Test + void testNestedAggregateUsesRewrittenRequireOutput() { + // Inner aggregate: GROUP BY id, name on unique-key table 'uni'. + // id → name FD (unique key) eliminates name from group-by, + // wrapping it with any_value(name) as a new alias (new ExprId). + // The outer EliminateGroupByKey must rewrite its project through the + // accumulated replaceMap before computing requireOutput; otherwise the + // stale ExprId would incorrectly cause name to be removed from output. + // Bug: proj.getInputSlots() returned old ExprIds → CheckAfterRewrite fails. + PlanChecker.from(connectContext) + .analyze("select t.id, t.name from " + + "(select id, name from uni group by id, name) t " + + "group by t.id, t.name") + .customRewrite(new EliminateGroupByKey()) + .matches( + logicalAggregate().when(agg -> + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("id"))); + } + + @Test + void testCteConsumerSlotMapUpdatedByReplaceMap() { + // Verify that visitLogicalCTEConsumer correctly updates the slot maps + // when the replaceMap contains an ExprId replacement from the producer. + Slot oldProducerSlot = new SlotReference("old", IntegerType.INSTANCE, false); + Slot consumerSlot = new SlotReference("cons", IntegerType.INSTANCE, false); + + LogicalCTEConsumer consumer = new LogicalCTEConsumer( + new RelationId(1), new CTEId(0), "cte", + ImmutableMap.of(consumerSlot, oldProducerSlot), + ImmutableMultimap.of(oldProducerSlot, consumerSlot)); + + // Simulate replaceMap with ExprId replacement from aggregate rewrite + ExprId newProducerExprId = new ExprId(999); // fresh Id from any_value alias + Map replaceMap = new HashMap<>(); + replaceMap.put(oldProducerSlot.getExprId(), newProducerExprId); + + EliminateGroupByKey rewriter = new EliminateGroupByKey(); + LogicalCTEConsumer updated = (LogicalCTEConsumer) rewriter.visitLogicalCTEConsumer( + consumer, replaceMap); + + // The updated consumer's producerToConsumerSlotMap should be keyed by the new ExprId + Multimap updatedMap = updated.getProducerToConsumerOutputMap(); + Assertions.assertEquals(1, updatedMap.keySet().size()); + Slot updatedProducerKey = updatedMap.keySet().iterator().next(); + Assertions.assertEquals(newProducerExprId, updatedProducerKey.getExprId(), + "Producer slot ExprId should be updated to the new one from replaceMap"); + Assertions.assertTrue(updatedMap.get(updatedProducerKey).contains(consumerSlot), + "Consumer slot should still be mapped"); + } + @Test void testRepeatEliminateByEqual() { PlanChecker.from(connectContext) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query54.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query54.out index ec6bf57b5449ec..7a35d1672d10c8 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query54.out @@ -11,15 +11,15 @@ PhysicalResultSink ----------------PhysicalProject ------------------hashAgg[GLOBAL] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->ss_sold_date_sk +----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF9 d_date_sk->ss_sold_date_sk ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF8 s_county->ca_county;RF9 s_state->ca_state +--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF7 s_county->ca_county;RF8 s_state->ca_state ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 ca_address_sk->c_current_addr_sk +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((my_customers.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->ss_customer_sk ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF10 +--------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF9 ------------------------------------PhysicalProject --------------------------------------hashAgg[GLOBAL] ----------------------------------------PhysicalProject @@ -42,9 +42,9 @@ PhysicalResultSink --------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) ----------------------------------------------------PhysicalOlapScan[date_dim] --------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF7 +----------------------------------------------PhysicalOlapScan[customer] --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[customer_address] apply RFs: RF8 RF9 +----------------------------------PhysicalOlapScan[customer_address] apply RFs: RF7 RF8 ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out index 178dfd718cc2d7..c4b6d05d41fffc 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out @@ -19,36 +19,36 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF8 RF9 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF6 s_county->ca_county;RF7 s_state->ca_state +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 c_current_addr_sk->ca_address_sk ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF5 s_county->ca_county;RF6 s_state->ca_state ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF5 RF6 RF7 ----------------------------------------PhysicalProject -------------------------------------------hashAgg[GLOBAL] +------------------------------------------PhysicalOlapScan[store] +------------------------------------PhysicalProject +--------------------------------------hashAgg[GLOBAL] +----------------------------------------PhysicalProject +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +--------------------------------------------PhysicalProject +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk ------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk -----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk ---------------------------------------------------------PhysicalUnion -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk +----------------------------------------------------PhysicalUnion +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +--------------------------------------------------------PhysicalProject +----------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity')) -------------------------------------------------------------PhysicalOlapScan[item] +----------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) ---------------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] +------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity')) +--------------------------------------------------------PhysicalOlapScan[item] +------------------------------------------------PhysicalProject +--------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) +----------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= d_month_seq+3) --------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query54.out b/regression-test/data/shape_check/tpcds_sf100/shape/query54.out index 178dfd718cc2d7..c4b6d05d41fffc 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query54.out @@ -19,36 +19,36 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF8 RF9 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF6 s_county->ca_county;RF7 s_state->ca_state +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 c_current_addr_sk->ca_address_sk ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF5 s_county->ca_county;RF6 s_state->ca_state ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF5 RF6 RF7 ----------------------------------------PhysicalProject -------------------------------------------hashAgg[GLOBAL] +------------------------------------------PhysicalOlapScan[store] +------------------------------------PhysicalProject +--------------------------------------hashAgg[GLOBAL] +----------------------------------------PhysicalProject +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +--------------------------------------------PhysicalProject +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk ------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk -----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk ---------------------------------------------------------PhysicalUnion -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk +----------------------------------------------------PhysicalUnion +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +--------------------------------------------------------PhysicalProject +----------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity')) -------------------------------------------------------------PhysicalOlapScan[item] +----------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) ---------------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] +------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity')) +--------------------------------------------------------PhysicalOlapScan[item] +------------------------------------------------PhysicalProject +--------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) +----------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= d_month_seq+3) --------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out index ff820f5904c9c6..76e2ca411c577b 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out @@ -19,36 +19,36 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF8 RF9 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF6 s_county->ca_county;RF7 s_state->ca_state +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 c_current_addr_sk->ca_address_sk ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF5 s_county->ca_county;RF6 s_state->ca_state ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF5 RF6 RF7 ----------------------------------------PhysicalProject -------------------------------------------hashAgg[GLOBAL] +------------------------------------------PhysicalOlapScan[store] +------------------------------------PhysicalProject +--------------------------------------hashAgg[GLOBAL] +----------------------------------------PhysicalProject +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +--------------------------------------------PhysicalProject +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk ------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk -----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk ---------------------------------------------------------PhysicalUnion -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk +----------------------------------------------------PhysicalUnion +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +--------------------------------------------------------PhysicalProject +----------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) -------------------------------------------------------------PhysicalOlapScan[item] +----------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) ---------------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] +------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) +--------------------------------------------------------PhysicalOlapScan[item] +------------------------------------------------PhysicalProject +--------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +----------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= d_month_seq+3) --------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out index a0023a914cd339..7b9e2b1a22dae1 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out @@ -13,47 +13,49 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->ss_sold_date_sk;RF19 d_date_sk->ss_sold_date_sk +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF16 c_customer_sk->ss_customer_sk;RF17 c_customer_sk->ss_customer_sk +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->ss_sold_date_sk;RF17 d_date_sk->ss_sold_date_sk --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF16 RF17 RF18 RF19 ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF12 s_county->ca_county;RF13 s_county->ca_county;RF14 s_state->ca_state;RF15 s_state->ca_state +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF14 c_customer_sk->ss_customer_sk;RF15 c_customer_sk->ss_customer_sk;RF18 c_current_addr_sk->ca_address_sk;RF19 c_current_addr_sk->ca_address_sk ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF10 c_current_addr_sk->ca_address_sk;RF11 c_current_addr_sk->ca_address_sk -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF10 RF11 RF12 RF13 RF14 RF15 -----------------------------------------hashAgg[GLOBAL] +--------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF14 RF15 RF16 RF17 +------------------------------------hashAgg[GLOBAL] +--------------------------------------PhysicalProject +----------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF12 customer_sk->c_customer_sk;RF13 customer_sk->c_customer_sk ------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF8 customer_sk->c_customer_sk;RF9 customer_sk->c_customer_sk -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF8 RF9 +--------------------------------------------PhysicalOlapScan[customer] apply RFs: RF12 RF13 +------------------------------------------PhysicalProject +--------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->cs_sold_date_sk;RF9 d_date_sk->ws_sold_date_sk;RF10 d_date_sk->cs_sold_date_sk;RF11 d_date_sk->ws_sold_date_sk ----------------------------------------------PhysicalProject -------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->cs_sold_date_sk;RF5 d_date_sk->ws_sold_date_sk;RF6 d_date_sk->cs_sold_date_sk;RF7 d_date_sk->ws_sold_date_sk ---------------------------------------------------PhysicalProject -----------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk;RF2 i_item_sk->cs_item_sk;RF3 i_item_sk->ws_item_sk -------------------------------------------------------PhysicalUnion ---------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------------------------------------PhysicalProject -------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 RF4 RF6 ---------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------------------------------------PhysicalProject -------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 RF5 RF7 +------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->cs_item_sk;RF5 i_item_sk->ws_item_sk;RF6 i_item_sk->cs_item_sk;RF7 i_item_sk->ws_item_sk +--------------------------------------------------PhysicalUnion +----------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------------------------------------PhysicalProject +--------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF6 RF8 RF10 +----------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] ------------------------------------------------------PhysicalProject ---------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) -----------------------------------------------------------PhysicalOlapScan[item] +--------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF5 RF7 RF9 RF11 --------------------------------------------------PhysicalProject -----------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) -------------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] -----------------------------PhysicalProject -------------------------------NestedLoopJoin[INNER_JOIN](cast(date_dim.d_month_seq as BIGINT) <= d_month_seq+3) +----------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) +------------------------------------------------------PhysicalOlapScan[item] +----------------------------------------------PhysicalProject +------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +--------------------------------------------------PhysicalOlapScan[date_dim] --------------------------------PhysicalProject -----------------------------------NestedLoopJoin[INNER_JOIN](cast(date_dim.d_month_seq as BIGINT) >= d_month_seq+1) +----------------------------------NestedLoopJoin[INNER_JOIN](cast(date_dim.d_month_seq as BIGINT) <= d_month_seq+3) ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[date_dim] +--------------------------------------NestedLoopJoin[INNER_JOIN](cast(date_dim.d_month_seq as BIGINT) >= d_month_seq+1) +----------------------------------------PhysicalProject +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------------PhysicalAssertNumRows +------------------------------------------PhysicalDistribute[DistributionSpecGather] +--------------------------------------------hashAgg[GLOBAL] +----------------------------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------------------------hashAgg[LOCAL] +--------------------------------------------------PhysicalProject +----------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +------------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalAssertNumRows --------------------------------------PhysicalDistribute[DistributionSpecGather] ----------------------------------------hashAgg[GLOBAL] @@ -62,12 +64,10 @@ PhysicalResultSink ----------------------------------------------PhysicalProject ------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) --------------------------------------------------PhysicalOlapScan[date_dim] ---------------------------------PhysicalAssertNumRows -----------------------------------PhysicalDistribute[DistributionSpecGather] -------------------------------------hashAgg[GLOBAL] ---------------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------------hashAgg[LOCAL] -------------------------------------------PhysicalProject ---------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) -----------------------------------------------PhysicalOlapScan[date_dim] +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF0 s_county->ca_county;RF1 s_county->ca_county;RF2 s_state->ca_state;RF3 s_state->ca_state +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[customer_address] apply RFs: RF0 RF1 RF2 RF3 RF18 RF19 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out index 26f08a422e60c1..fd69e1a2db0bae 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out @@ -19,36 +19,36 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF8 RF9 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF6 s_county->ca_county;RF7 s_state->ca_state +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 c_current_addr_sk->ca_address_sk ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF5 s_county->ca_county;RF6 s_state->ca_state ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF5 RF6 RF7 ----------------------------------------PhysicalProject -------------------------------------------hashAgg[GLOBAL] +------------------------------------------PhysicalOlapScan[store] +------------------------------------PhysicalProject +--------------------------------------hashAgg[GLOBAL] +----------------------------------------PhysicalProject +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +--------------------------------------------PhysicalProject +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk ------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk -----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk ---------------------------------------------------------PhysicalUnion -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk +----------------------------------------------------PhysicalUnion +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +--------------------------------------------------------PhysicalProject +----------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) -------------------------------------------------------------PhysicalOlapScan[item] +----------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) ---------------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] +------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) +--------------------------------------------------------PhysicalOlapScan[item] +------------------------------------------------PhysicalProject +--------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +----------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= d_month_seq+3) --------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out index 2db61d79191f2d..5836333f34dc91 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out @@ -19,36 +19,36 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20 RF21 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF14 s_county->ca_county;RF15 s_county->ca_county;RF16 s_state->ca_state;RF17 s_state->ca_state +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF16 c_current_addr_sk->ca_address_sk;RF17 c_current_addr_sk->ca_address_sk ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF12 c_current_addr_sk->ca_address_sk;RF13 c_current_addr_sk->ca_address_sk +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF12 s_county->ca_county;RF13 s_county->ca_county;RF14 s_state->ca_state;RF15 s_state->ca_state ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF12 RF13 RF14 RF15 RF16 RF17 ----------------------------------------PhysicalProject -------------------------------------------hashAgg[GLOBAL] +------------------------------------------PhysicalOlapScan[store] +------------------------------------PhysicalProject +--------------------------------------hashAgg[GLOBAL] +----------------------------------------PhysicalProject +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF10 customer_sk->c_customer_sk;RF11 customer_sk->c_customer_sk --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF10 customer_sk->c_customer_sk;RF11 customer_sk->c_customer_sk -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF10 RF11 +----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF10 RF11 +--------------------------------------------PhysicalProject +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->cs_sold_date_sk;RF7 d_date_sk->ws_sold_date_sk;RF8 d_date_sk->cs_sold_date_sk;RF9 d_date_sk->ws_sold_date_sk ------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->cs_sold_date_sk;RF7 d_date_sk->ws_sold_date_sk;RF8 d_date_sk->cs_sold_date_sk;RF9 d_date_sk->ws_sold_date_sk -----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->cs_item_sk;RF3 i_item_sk->ws_item_sk;RF4 i_item_sk->cs_item_sk;RF5 i_item_sk->ws_item_sk ---------------------------------------------------------PhysicalUnion -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF4 RF6 RF8 -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF3 RF5 RF7 RF9 +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->cs_item_sk;RF3 i_item_sk->ws_item_sk;RF4 i_item_sk->cs_item_sk;RF5 i_item_sk->ws_item_sk +----------------------------------------------------PhysicalUnion +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +--------------------------------------------------------PhysicalProject +----------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF4 RF6 RF8 +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) -------------------------------------------------------------PhysicalOlapScan[item] +----------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF3 RF5 RF7 RF9 ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) ---------------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] +------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) +--------------------------------------------------------PhysicalOlapScan[item] +------------------------------------------------PhysicalProject +--------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +----------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= d_month_seq+3) build RFs:RF1 d_month_seq+3->cast(d_month_seq as BIGINT) --------------------------------PhysicalProject diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index cedb00cc0e3e56..9da09b263181dd 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -2041,7 +2041,7 @@ class Suite implements GroovyInterceptable { } } if (status != "SUCCESS") { - logger.info("status is not success") + logger.info("status is ${status}") } Assert.assertEquals("SUCCESS", status) logger.info("waitingMTMVTaskFinished analyze mv name is " + mvName @@ -2197,7 +2197,7 @@ class Suite implements GroovyInterceptable { } } while (timeoutTimestamp > System.currentTimeMillis() && (status == 'PENDING' || status == 'RUNNING' || status == 'NULL')) if (status != "SUCCESS") { - logger.info("status is not success") + logger.info("status is ${status}") } Assert.assertEquals("SUCCESS", status) // Need to analyze materialized view for cbo to choose the materialized view accurately diff --git a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy index 0fa49496708967..e67c5f976d271b 100644 --- a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy +++ b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy @@ -84,7 +84,7 @@ suite("eliminate_gby_key") { select t2_c2 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[c1#13, c3#18]") } explain { @@ -144,7 +144,7 @@ suite("eliminate_gby_key") { select t2_c2, t2_c1 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[c1#13, c3#18]") } explain { @@ -184,7 +184,7 @@ suite("eliminate_gby_key") { select c3, t2_c2 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[c1#13, c3#18]") } explain { @@ -264,7 +264,7 @@ suite("eliminate_gby_key") { select t2_c2, c3, t2_c1 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[c1#13, c3#18]") } explain { @@ -284,7 +284,7 @@ suite("eliminate_gby_key") { select t2_c2, c3, t2_c1, cnt from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18,") + contains("groupByExpr=[c1#13, c3#18]") } sql "drop table if exists eli_gbk_t" diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy index d12442a26f1716..458e8c78b06509 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy @@ -22,7 +22,7 @@ suite("agg_variety") { sql "set runtime_filter_mode=OFF"; sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" sql "set pre_materialized_view_rewrite_strategy = TRY_IN_RBO" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders """ diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index 1b936f5a609a59..6032899049a0de 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -25,7 +25,7 @@ suite("aggregate_without_roll_up") { sql "SET enable_dphyp_optimizer = false;" sql "SET max_table_count_use_cascades_join_reorder = 20;" sql "set pre_materialized_view_rewrite_strategy = TRY_IN_RBO" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders """ @@ -1688,7 +1688,7 @@ suite("aggregate_without_roll_up") { order_qt_query29_0_before "${query29_0}" async_mv_rewrite_success(db, mv29_0, query29_0, "mv29_0") order_qt_query29_0_after "${query29_0}" - sql """ DROP MATERIALIZED VIEW IF EXISTS mv29_0""" + // sql """ DROP MATERIALIZED VIEW IF EXISTS mv29_0""" // query and mv has the same filter but position is different, should rewrite successfully @@ -1839,6 +1839,7 @@ suite("aggregate_without_roll_up") { 13, 14; """ + order_qt_query30_0_before "${query30_0}" async_mv_rewrite_success(db, mv30_0, query30_0, "mv30_0", [TRY_IN_RBO, FORCE_IN_RBO]) // ELIMINATE_CONST_JOIN_CONDITION not work, so should success @@ -1846,7 +1847,6 @@ suite("aggregate_without_roll_up") { order_qt_query30_0_after "${query30_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv30_0""" - // query and mv has the same filter but position is different, should rewrite successfully // query join condition has alias def mv31_0 = """ diff --git a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy index a2fcb2eba15913..24cc3d234b4ba8 100644 --- a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy @@ -23,6 +23,8 @@ suite("mtmv_range_date_datetrunc_date_part_up") { sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=true" sql "SET enable_nereids_timeout = false" + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" + String mv_prefix = "range_datetrunc_date_up" String tb_name = mv_prefix + "_tb" String mv_name = mv_prefix + "_mv" diff --git a/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy b/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy index 58082d74decf05..66231cff69f901 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy @@ -22,139 +22,140 @@ suite("dml_rewrite_with_lock", "zfr_mtmv_test") { sql "SET enable_materialized_view_rewrite=true" sql "SET enable_materialized_view_nest_rewrite=true" sql "SET enable_materialized_view_union_rewrite=true" - - sql """ - drop table if exists lineitem_range_date_union - """ - - sql """CREATE TABLE `lineitem_range_date_union` ( - `l_orderkey` BIGINT NULL, - `l_linenumber` INT NULL, - `l_partkey` INT NULL, - `l_suppkey` INT NULL, - `l_quantity` DECIMAL(15, 2) NULL, - `l_extendedprice` DECIMAL(15, 2) NULL, - `l_discount` DECIMAL(15, 2) NULL, - `l_tax` DECIMAL(15, 2) NULL, - `l_returnflag` VARCHAR(1) NULL, - `l_linestatus` VARCHAR(1) NULL, - `l_commitdate` DATE NULL, - `l_receiptdate` DATE NULL, - `l_shipinstruct` VARCHAR(25) NULL, - `l_shipmode` VARCHAR(10) NULL, - `l_comment` VARCHAR(44) NULL, - `l_shipdate` DATE not NULL - ) ENGINE=OLAP - DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey ) - COMMENT 'OLAP' - partition by range (`l_shipdate`) ( - partition p1 values [("2023-10-29"), ("2023-10-30")), - partition p2 values [("2023-10-30"), ("2023-10-31")), - partition p3 values [("2023-10-31"), ("2023-11-01"))) - DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - );""" - - sql """ - drop table if exists orders_range_date_union - """ - - sql """CREATE TABLE `orders_range_date_union` ( - `o_orderkey` BIGINT NULL, - `o_custkey` INT NULL, - `o_orderstatus` VARCHAR(1) NULL, - `o_totalprice` DECIMAL(15, 2) NULL, - `o_orderpriority` VARCHAR(15) NULL, - `o_clerk` VARCHAR(15) NULL, - `o_shippriority` INT NULL, - `o_comment` VARCHAR(79) NULL, - `o_orderdate` DATE not NULL - ) ENGINE=OLAP - DUPLICATE KEY(`o_orderkey`, `o_custkey`) - COMMENT 'OLAP' - partition by range (`o_orderdate`) ( - partition p1 values [("2023-10-29"), ("2023-10-30")), - partition p2 values [("2023-10-30"), ("2023-10-31")), - partition p3 values [("2023-10-31"), ("2023-11-01")), - partition p4 values [("2023-11-01"), ("2023-11-02")), - partition p5 values [("2023-11-02"), ("2023-11-03"))) - DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - );""" - - sql """ - insert into lineitem_range_date_union values - (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), - (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), - (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-31'), - (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), - (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-30'), - (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-31'), - (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'); - """ - - sql """ - insert into orders_range_date_union values - (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'), - (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-29'), - (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-30'), - (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-11-01'), - (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-11-02'), - (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-11-02'), - (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-31'), - (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-30'), - (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'), - (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-31'); - """ - - sql """DROP MATERIALIZED VIEW if exists day_mv;""" - create_async_mv(db, "day_mv", - """select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey - from lineitem_range_date_union as t1 left join orders_range_date_union as t2 - on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey; - """ - ) - - def query1 = """ - select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey - from lineitem_range_date_union as t1 left join orders_range_date_union as t2 - on t1.l_orderkey = t2.o_orderkey - group by col1, l_shipdate, l_orderkey - """ - - mv_rewrite_success(query1, "day_mv") - - def query2 = """ - select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from - lineitem_range_date_union as t1 left join orders_range_date_union as t2 - on t1.l_orderkey = t2.o_orderkey - group by col1, l_shipdate, l_orderkey - """ - - sql """DROP MATERIALIZED VIEW if exists hour_mv;""" - create_async_mv(db, "hour_mv", - """ - select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from - lineitem_range_date_union as t1 left join orders_range_date_union as t2 - on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey; - """) - mv_rewrite_success(query2, "hour_mv") - - - sql """alter table lineitem_range_date_union add partition p4 values [("2023-11-01"), ("2023-11-02"));""" - sql """insert into lineitem_range_date_union values - (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-01')""" - - sql """refresh MATERIALIZED VIEW hour_mv auto;""" - waitingMTMVTaskFinishedByMvName("hour_mv") - - sql """refresh MATERIALIZED VIEW day_mv auto;""" - waitingMTMVTaskFinishedByMvName("day_mv") - - mv_rewrite_success(query1, "day_mv") - mv_rewrite_success(query2, "hour_mv") + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" + + // sql """ + // drop table if exists lineitem_range_date_union + // """ + + // sql """CREATE TABLE `lineitem_range_date_union` ( + // `l_orderkey` BIGINT NULL, + // `l_linenumber` INT NULL, + // `l_partkey` INT NULL, + // `l_suppkey` INT NULL, + // `l_quantity` DECIMAL(15, 2) NULL, + // `l_extendedprice` DECIMAL(15, 2) NULL, + // `l_discount` DECIMAL(15, 2) NULL, + // `l_tax` DECIMAL(15, 2) NULL, + // `l_returnflag` VARCHAR(1) NULL, + // `l_linestatus` VARCHAR(1) NULL, + // `l_commitdate` DATE NULL, + // `l_receiptdate` DATE NULL, + // `l_shipinstruct` VARCHAR(25) NULL, + // `l_shipmode` VARCHAR(10) NULL, + // `l_comment` VARCHAR(44) NULL, + // `l_shipdate` DATE not NULL + // ) ENGINE=OLAP + // DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey ) + // COMMENT 'OLAP' + // partition by range (`l_shipdate`) ( + // partition p1 values [("2023-10-29"), ("2023-10-30")), + // partition p2 values [("2023-10-30"), ("2023-10-31")), + // partition p3 values [("2023-10-31"), ("2023-11-01"))) + // DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96 + // PROPERTIES ( + // "replication_allocation" = "tag.location.default: 1" + // );""" + + // sql """ + // drop table if exists orders_range_date_union + // """ + + // sql """CREATE TABLE `orders_range_date_union` ( + // `o_orderkey` BIGINT NULL, + // `o_custkey` INT NULL, + // `o_orderstatus` VARCHAR(1) NULL, + // `o_totalprice` DECIMAL(15, 2) NULL, + // `o_orderpriority` VARCHAR(15) NULL, + // `o_clerk` VARCHAR(15) NULL, + // `o_shippriority` INT NULL, + // `o_comment` VARCHAR(79) NULL, + // `o_orderdate` DATE not NULL + // ) ENGINE=OLAP + // DUPLICATE KEY(`o_orderkey`, `o_custkey`) + // COMMENT 'OLAP' + // partition by range (`o_orderdate`) ( + // partition p1 values [("2023-10-29"), ("2023-10-30")), + // partition p2 values [("2023-10-30"), ("2023-10-31")), + // partition p3 values [("2023-10-31"), ("2023-11-01")), + // partition p4 values [("2023-11-01"), ("2023-11-02")), + // partition p5 values [("2023-11-02"), ("2023-11-03"))) + // DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96 + // PROPERTIES ( + // "replication_allocation" = "tag.location.default: 1" + // );""" + + // sql """ + // insert into lineitem_range_date_union values + // (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), + // (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), + // (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-31'), + // (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), + // (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-30'), + // (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-31'), + // (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'); + // """ + + // sql """ + // insert into orders_range_date_union values + // (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'), + // (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-29'), + // (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-30'), + // (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-11-01'), + // (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-11-02'), + // (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-11-02'), + // (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-31'), + // (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-30'), + // (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'), + // (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-31'); + // """ + + // sql """DROP MATERIALIZED VIEW if exists day_mv;""" + // create_async_mv(db, "day_mv", + // """select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey + // from lineitem_range_date_union as t1 left join orders_range_date_union as t2 + // on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey; + // """ + // ) + + // def query1 = """ + // select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey + // from lineitem_range_date_union as t1 left join orders_range_date_union as t2 + // on t1.l_orderkey = t2.o_orderkey + // group by col1, l_shipdate, l_orderkey + // """ + + // mv_rewrite_success(query1, "day_mv") + + // def query2 = """ + // select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from + // lineitem_range_date_union as t1 left join orders_range_date_union as t2 + // on t1.l_orderkey = t2.o_orderkey + // group by col1, l_shipdate, l_orderkey + // """ + + // sql """DROP MATERIALIZED VIEW if exists hour_mv;""" + // create_async_mv(db, "hour_mv", + // """ + // select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from + // lineitem_range_date_union as t1 left join orders_range_date_union as t2 + // on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey; + // """) + // mv_rewrite_success(query2, "hour_mv") + + + // sql """alter table lineitem_range_date_union add partition p4 values [("2023-11-01"), ("2023-11-02"));""" + // sql """insert into lineitem_range_date_union values + // (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-01')""" + + // sql """refresh MATERIALIZED VIEW hour_mv auto;""" + // waitingMTMVTaskFinishedByMvName("hour_mv") + + // sql """refresh MATERIALIZED VIEW day_mv auto;""" + // waitingMTMVTaskFinishedByMvName("day_mv") + + // mv_rewrite_success(query1, "day_mv") + // mv_rewrite_success(query2, "hour_mv") } diff --git a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_list_str_increment_create.groovy b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_list_str_increment_create.groovy index 22ec46b801c772..422579d558933c 100644 --- a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_list_str_increment_create.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_list_str_increment_create.groovy @@ -21,7 +21,7 @@ suite("inner_join_list_str_increment_create", "increment_create") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=false" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders_inner_1 """ diff --git a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_date_increment_create.groovy b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_date_increment_create.groovy index d7007cb40820d5..aefdcfc3d0cd15 100644 --- a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_date_increment_create.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_date_increment_create.groovy @@ -21,7 +21,7 @@ suite("inner_join_range_date_increment_create", "increment_create") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=false" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders_inner_2 """ diff --git a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_number_increment_create.groovy b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_number_increment_create.groovy index caabd8a5ee86b5..06a618c9b1ac76 100644 --- a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_number_increment_create.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_number_increment_create.groovy @@ -21,7 +21,7 @@ suite("inner_join_range_number_increment_create", "increment_create") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=false" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders_inner_3 """ diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy index cbe9f218f68f5d..1a82b31bbf94a6 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy @@ -22,7 +22,7 @@ suite("nested_mtmv") { sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=true" sql "SET enable_materialized_view_nest_rewrite = true" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders_1 """