From 4c6cfaee9bc076a399789e5fa5fca211e71469ff Mon Sep 17 00:00:00 2001 From: englefly Date: Thu, 25 Jun 2026 16:11:34 +0800 Subject: [PATCH 01/23] feat: eliminate FD-redundant group-by keys via ANY_VALUE wrapping When a group-by key is functionally dependent on another key (e.g. s_suppkey -> s_name via PK) but required in output, remove it from GROUP BY and wrap with ANY_VALUE(). Previously EliminateGroupByKey kept such keys in GROUP BY to preserve SQL semantics. Now they are replaced with ANY_VALUE wrappers in the output, allowing the group-by set to be minimized while keeping the column in SELECT. Public findCanBeRemovedExpressions() API preserved for backward compatibility. Internal logic split into FindResult with separate removeExpression and wrapWithAnyValue sets. Test: testEliminateByPkWithOutputNeeded verifies ANY_VALUE wrapping when SELECT contains an FD-redundant group-by key. --- .../doris/nereids/jobs/executor/Rewriter.java | 2 +- .../doris/nereids/properties/FuncDeps.java | 5 +- .../rules/rewrite/EliminateGroupByKey.java | 198 +++++++++++++----- .../rewrite/EliminateGroupByKeyTest.java | 25 ++- 4 files changed, 178 insertions(+), 52 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index e651a9e8fae583..4aa05cceda10cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -678,7 +678,7 @@ public class Rewriter extends AbstractBatchJobExecutor { cascadesContext -> cascadesContext.rewritePlanContainsTypes(LogicalAggregate.class) || cascadesContext.rewritePlanContainsTypes(LogicalJoin.class) || cascadesContext.rewritePlanContainsTypes(LogicalUnion.class), - topDown(new EliminateGroupByKey()), + custom(RuleType.ELIMINATE_GROUP_BY_KEY, EliminateGroupByKey::new), topDown(new PushDownAggThroughJoinOnPkFk()), topDown(new PullUpJoinFromUnionAll()) ), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java index 879b2de9fe6468..3553b2deb8a89f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java @@ -146,7 +146,7 @@ private Set findValidItems(Set requireOutputs) { * Given: * - Initial slots: {{A}, {B}, {C}, {D}, {E}} * - Required outputs: {} - * - validItems: {A} -> {B}, {B} -> {C}, {C} -> {D}, {D} -> {A}, {A} -> {E} + * - validItems: {A} -> {B}, {B} -> {C}, {C} -> {D}, {D} -> {E}, {A} -> {E} * * Process: * 1. Start with minSlotSet = {{A}, {B}, {C}, {D}, {E}} @@ -163,7 +163,8 @@ private Set findValidItems(Set requireOutputs) { *

* * @param slots the initial set of slot sets to be reduced - * @param requireOutputs the set of slots that must be preserved in the output + * @param requireOutputs output-required slots; used in circular-dependency + * resolution to avoid eliminating FD edges that originate from these slots * @return the minimal set of slot sets after applying all possible reductions */ public Set> eliminateDeps(Set> slots, Set requireOutputs) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java index 4e1b3117ab53ff..f6200012fdc102 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java @@ -17,18 +17,22 @@ package org.apache.doris.nereids.rules.rewrite; -import org.apache.doris.nereids.annotation.DependsRules; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.properties.FuncDeps; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; - -import com.google.common.collect.ImmutableList; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import java.util.ArrayList; import java.util.HashMap; @@ -38,69 +42,162 @@ import java.util.Map.Entry; import java.util.Set; - /** * Eliminate group by key based on fd item information. * such as: * for a -> b, we can get: * group by a, b, c => group by a, c + * + * When a group-by key is FD-redundant but still needed in the output, + * it is wrapped with any_value() and assigned a fresh ExprId. + * Upper plan references are rewritten via ExprIdRewriter so that + * all ancestor nodes see the new ExprIds. */ -@DependsRules({EliminateGroupBy.class, ColumnPruning.class}) -public class EliminateGroupByKey implements RewriteRuleFactory { +public class EliminateGroupByKey extends DefaultPlanRewriter> implements CustomRewriter { + private ExprIdRewriter exprIdReplacer; @Override - public List buildRules() { - return ImmutableList.of( - RuleType.ELIMINATE_GROUP_BY_KEY.build( - logicalProject(logicalAggregate().when(agg -> !agg.getSourceRepeat().isPresent())) - .then(proj -> { - LogicalAggregate agg = proj.child(); - LogicalAggregate newAgg = eliminateGroupByKey(agg, proj.getInputSlots()); - if (newAgg == null) { - return null; - } - return proj.withChildren(newAgg); - })), - RuleType.ELIMINATE_FILTER_GROUP_BY_KEY.build( - logicalProject(logicalFilter(logicalAggregate() - .when(agg -> !agg.getSourceRepeat().isPresent()))) - .then(proj -> { - LogicalAggregate agg = proj.child().child(); - Set requireSlots = new HashSet<>(proj.getInputSlots()); - requireSlots.addAll(proj.child(0).getInputSlots()); - LogicalAggregate newAgg = eliminateGroupByKey(agg, requireSlots); - if (newAgg == null) { - return null; - } - return proj.withChildren(proj.child().withChildren(newAgg)); - }) - ) - ); + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + if (!plan.containsType(Aggregate.class)) { + return plan; + } + Map replaceMap = new HashMap<>(); + ExprIdRewriter.ReplaceRule replaceRule = new ExprIdRewriter.ReplaceRule(replaceMap, false); + exprIdReplacer = new ExprIdRewriter(replaceRule, jobContext); + return plan.accept(this, replaceMap); } - LogicalAggregate eliminateGroupByKey(LogicalAggregate agg, Set requireOutput) { - Set removeExpression = findCanBeRemovedExpressions(agg, requireOutput, + @Override + public Plan visit(Plan plan, Map replaceMap) { + plan = visitChildren(this, plan, replaceMap); + plan = exprIdReplacer.rewriteExpr(plan, replaceMap); + return plan; + } + + @Override + public Plan visitLogicalProject(LogicalProject proj, Map replaceMap) { + proj = visitChildren(this, proj, replaceMap); + + // Find the Aggregate child, possibly through a Filter + Plan child = proj.child(0); + LogicalAggregate agg; + boolean hasFilter = child instanceof LogicalFilter; + if (hasFilter && child.child(0) instanceof LogicalAggregate) { + agg = (LogicalAggregate) child.child(0); + } else if (child instanceof LogicalAggregate) { + agg = (LogicalAggregate) child; + } else { + return exprIdReplacer.rewriteExpr(proj, replaceMap); + } + + // Don't transform if source repeat is present + if (agg.getSourceRepeat().isPresent()) { + return exprIdReplacer.rewriteExpr(proj, replaceMap); + } + + // Compute requireOutput: slots needed by the Project (and Filter, if present) + Set requireOutput = new HashSet<>(proj.getInputSlots()); + if (hasFilter) { + requireOutput.addAll(child.getInputSlots()); + } + + // Transform the aggregate + EliminateResult result = eliminateGroupByKeyWithMap(agg, requireOutput); + if (!result.changed) { + return exprIdReplacer.rewriteExpr(proj, replaceMap); + } + + // Merge into the global replaceMap so that all ancestor nodes get rewritten + replaceMap.putAll(result.replaceMap); + + // Rebuild the child chain with the new aggregate, + // and rewrite the Filter (if present) and Project expressions + Plan newChild; + if (hasFilter) { + Plan updatedFilter = child.withChildren(result.newAgg); + newChild = exprIdReplacer.rewriteExpr(updatedFilter, replaceMap); + } else { + newChild = result.newAgg; + } + Plan newProj = exprIdReplacer.rewriteExpr(proj.withChildren(newChild), replaceMap); + return newProj; + } + + /** Result of eliminateGroupByKey: the new aggregate and a map of old->new ExprIds. */ + private static class EliminateResult { + final LogicalAggregate newAgg; + final Map replaceMap; + final boolean changed; + + EliminateResult(LogicalAggregate newAgg, Map replaceMap, boolean changed) { + this.newAgg = newAgg; + this.replaceMap = replaceMap; + this.changed = changed; + } + } + + EliminateResult eliminateGroupByKeyWithMap(LogicalAggregate agg, Set requireOutput) { + FindResult result = findCanBeRemovedExpressionsInternal(agg, requireOutput, agg.child().getLogicalProperties().getTrait()); + Set removeExpression = result.removeExpression; + Set wrapWithAnyValue = result.wrapWithAnyValue; + List newGroupExpression = new ArrayList<>(); for (Expression expression : agg.getGroupByExpressions()) { - if (!removeExpression.contains(expression)) { + if (!removeExpression.contains(expression) + && !wrapWithAnyValue.contains(expression)) { newGroupExpression.add(expression); } } List newOutput = new ArrayList<>(); + Map replaceMap = new HashMap<>(); + boolean changed = !removeExpression.isEmpty() || !wrapWithAnyValue.isEmpty(); for (NamedExpression expression : agg.getOutputExpressions()) { - if (!removeExpression.contains(expression)) { - newOutput.add(expression); + if (removeExpression.contains(expression)) { + continue; } + if (wrapWithAnyValue.contains(expression)) { + // expression is FD-redundant but needed in output: wrap with any_value + // Use fresh ExprId (auto-generated by Alias) to avoid ExprId collision, + // and record the mapping for rewriting upper plan references. + Alias newAlias = new Alias(new AnyValue(expression.toSlot()), expression.getName()); + replaceMap.put(expression.getExprId(), newAlias.getExprId()); + expression = newAlias; + } + newOutput.add(expression); } - return agg.withGroupByAndOutput(newGroupExpression, newOutput); + return new EliminateResult(agg.withGroupByAndOutput(newGroupExpression, newOutput), replaceMap, changed); } /** - * return removeExpression + * Return expressions that can be removed from both group-by and output. + * Kept for backward compatibility with external callers (e.g. PushDownAggThroughJoinOnPkFk). + */ + /** + * Return expressions that can be completely removed from both group-by and output. + * IMPORTANT: Does NOT return wrapWithAnyValue expressions — those require ANY_VALUE + * wrapping in the output, which only the internal eliminateGroupByKeyWithMap handles. + * Kept for backward compatibility with external callers (e.g. PushDownAggThroughJoinOnPkFk). */ public static Set findCanBeRemovedExpressions(LogicalAggregate agg, Set requireOutput, DataTrait dataTrait) { + FindResult result = findCanBeRemovedExpressionsInternal(agg, requireOutput, dataTrait); + return new HashSet<>(result.removeExpression); + } + + /** Result of findCanBeRemovedExpressionsInternal: two sets of expressions. */ + private static class FindResult { + final Set removeExpression; // remove from group-by and output + final Set wrapWithAnyValue; // remove from group-by, wrap with ANY_VALUE in output + + FindResult(Set removeExpression, Set wrapWithAnyValue) { + this.removeExpression = removeExpression; + this.wrapWithAnyValue = wrapWithAnyValue; + } + } + + private static FindResult findCanBeRemovedExpressionsInternal(LogicalAggregate agg, + Set requireOutput, DataTrait dataTrait) { Map> groupBySlots = new HashMap<>(); Set validSlots = new HashSet<>(); for (Expression expression : agg.getGroupByExpressions()) { @@ -110,17 +207,24 @@ public static Set findCanBeRemovedExpressions(LogicalAggregate(); + return new FindResult(new HashSet<>(), new HashSet<>()); } Set> minGroupBySlots = funcDeps.eliminateDeps(new HashSet<>(groupBySlots.values()), requireOutput); Set removeExpression = new HashSet<>(); + Set wrapWithAnyValue = new HashSet<>(); for (Entry> entry : groupBySlots.entrySet()) { - if (!minGroupBySlots.contains(entry.getValue()) - && !requireOutput.containsAll(entry.getValue())) { - removeExpression.add(entry.getKey()); + if (!minGroupBySlots.contains(entry.getValue())) { + // FD redundant: can remove from group-by + if (!requireOutput.containsAll(entry.getValue())) { + // Not needed in output either: remove completely + removeExpression.add(entry.getKey()); + } else { + // Still needed in output: remove from group-by, wrap with ANY_VALUE in output + wrapWithAnyValue.add(entry.getKey()); + } } } - return removeExpression; + return new FindResult(removeExpression, wrapWithAnyValue); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java index 7362c81e5afe0b..58e50aabd1c08f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java @@ -18,8 +18,10 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.properties.FuncDeps; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; @@ -99,7 +101,7 @@ void testEliminateTree() { void testEliminateByUniform() { PlanChecker.from(connectContext) .analyze("select count(name) from t1 where id = 1 group by name, id") - .rewrite() + .customRewrite(new EliminateGroupByKey()) .printlnTree() .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1 && agg.getGroupByExpressions().get(0).toSql().equals("name"))); @@ -109,7 +111,7 @@ void testEliminateByUniform() { void testProjectAlias() { PlanChecker.from(connectContext) .analyze("select id as c from t1 where id = 1 group by name, id") - .rewrite() + .customRewrite(new EliminateGroupByKey()) .printlnTree() .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1)); @@ -181,6 +183,25 @@ void testEliminateByEqual() { && agg.getGroupByExpressions().get(0).toSql().equals("name"))); } + @Test + void testEliminateByPkWithOutputNeeded() throws Exception { + // Regression: when a group-by key (name) is FD-redundant (id -> name) + // but still appears in SELECT, it should be removed from group-by + // and wrapped with ANY_VALUE in the output. + addConstraint("alter table t1 add constraint pk2 primary key (id)"); + PlanChecker.from(connectContext) + .analyze("select id, name, count(*) from t1 group by id, name") + .customRewrite(new EliminateGroupByKey()) + .printlnTree() + .matches(logicalAggregate().when(agg -> + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("id") + && agg.getOutputExpressions().stream().anyMatch( + e -> e instanceof Alias + && e.child(0) instanceof AnyValue))); + dropConstraint("alter table t1 drop constraint pk2"); + } + @Test void testRepeatEliminateByEqual() { PlanChecker.from(connectContext) From 4fc3615a8aaf6a1ad7a37e4381f9fe469f4115db Mon Sep 17 00:00:00 2001 From: minghong Date: Tue, 30 Jun 2026 16:06:34 +0800 Subject: [PATCH 02/23] review-630 --- .../main/java/org/apache/doris/nereids/rules/RuleType.java | 1 - .../doris/nereids/rules/rewrite/EliminateGroupByKey.java | 6 ------ 2 files changed, 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 4a57e8916957c3..b62b70934edbe2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -309,7 +309,6 @@ public enum RuleType { ELIMINATE_GROUP_BY_KEY(RuleTypeClass.REWRITE), ELIMINATE_GROUP_BY_KEY_BY_UNIFORM(RuleTypeClass.REWRITE), ELIMINATE_ORDER_BY_KEY(RuleTypeClass.REWRITE), - ELIMINATE_FILTER_GROUP_BY_KEY(RuleTypeClass.REWRITE), ELIMINATE_DEDUP_JOIN_CONDITION(RuleTypeClass.REWRITE), ELIMINATE_NULL_AWARE_LEFT_ANTI_JOIN(RuleTypeClass.REWRITE), ELIMINATE_ASSERT_NUM_ROWS(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java index f6200012fdc102..e60cdcc83ce967 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java @@ -169,14 +169,8 @@ EliminateResult eliminateGroupByKeyWithMap(LogicalAggregate agg, return new EliminateResult(agg.withGroupByAndOutput(newGroupExpression, newOutput), replaceMap, changed); } - /** - * Return expressions that can be removed from both group-by and output. - * Kept for backward compatibility with external callers (e.g. PushDownAggThroughJoinOnPkFk). - */ /** * Return expressions that can be completely removed from both group-by and output. - * IMPORTANT: Does NOT return wrapWithAnyValue expressions — those require ANY_VALUE - * wrapping in the output, which only the internal eliminateGroupByKeyWithMap handles. * Kept for backward compatibility with external callers (e.g. PushDownAggThroughJoinOnPkFk). */ public static Set findCanBeRemovedExpressions(LogicalAggregate agg, From b4fc7d8ace9e78a9df03139e3d88ce06086d16f4 Mon Sep 17 00:00:00 2001 From: englefly Date: Sun, 5 Jul 2026 08:16:21 +0800 Subject: [PATCH 03/23] =?UTF-8?q?1.=20**Rewriter.java**=20=E2=80=94=20?= =?UTF-8?q?=E8=A7=84=E5=88=99=E9=A1=BA=E5=BA=8F=E8=B0=83=E6=95=B4=EF=BC=9A?= =?UTF-8?q?=20=E7=A7=BB=E5=88=B0=20=20=E4=B9=8B=E5=89=8D=202.=20**PushDown?= =?UTF-8?q?AggThroughJoinOnPkFk.java**=20=E2=80=94=20=E9=98=B2=E5=BE=A1?= =?UTF-8?q?=E6=80=A7=20ANY=5FVALUE=20handler=EF=BC=9A=20=E2=86=92=203.=20*?= =?UTF-8?q?*EliminateGroupByKeyByUniformTest.java**=20=E2=80=94=20?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=B8=A4=E4=B8=AA=20LEFT=20JOIN=20=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E7=9A=84=20group-by=20size=20=E6=9C=9F=E6=9C=9B?= =?UTF-8?q?=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../doris/nereids/jobs/executor/Rewriter.java | 15 +++++---------- .../doris/nereids/properties/FuncDeps.java | 5 ++--- .../org/apache/doris/nereids/rules/RuleType.java | 1 + .../rewrite/PushDownAggThroughJoinOnPkFk.java | 16 ++++++++++++++++ .../EliminateGroupByKeyByUniformTest.java | 6 ++++-- 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 4aa05cceda10cd..1ad27e61785637 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -678,19 +678,14 @@ public class Rewriter extends AbstractBatchJobExecutor { cascadesContext -> cascadesContext.rewritePlanContainsTypes(LogicalAggregate.class) || cascadesContext.rewritePlanContainsTypes(LogicalJoin.class) || cascadesContext.rewritePlanContainsTypes(LogicalUnion.class), - custom(RuleType.ELIMINATE_GROUP_BY_KEY, EliminateGroupByKey::new), + // PushDownAggThroughJoinOnPkFk must run before EliminateGroupByKey, + // because EliminateGroupByKey wraps FD-redundant group-by keys with + // ANY_VALUE and rewrites ExprIds, which PushDownAggThroughJoinOnPkFk + // cannot fully handle (especially for non-PK/FK primary table columns). topDown(new PushDownAggThroughJoinOnPkFk()), + custom(RuleType.ELIMINATE_GROUP_BY_KEY, EliminateGroupByKey::new), topDown(new PullUpJoinFromUnionAll()) ), - topic("init join", bottomUp(ImmutableList.of(new InitJoinOrder()))), - topic("Eager aggregation", - cascadesContext -> cascadesContext.rewritePlanContainsTypes( - LogicalAggregate.class, LogicalJoin.class - ), - costBased(topDown(new PushDownAggWithDistinctThroughJoinOneSide())), - custom(RuleType.PUSH_DOWN_AGG_THROUGH_JOIN, PushDownAggregation::new), - topDown(new PushCountIntoUnionAll()) - ), topic("Limit optimization", cascadesContext -> cascadesContext.rewritePlanContainsTypes(LogicalLimit.class) || cascadesContext.rewritePlanContainsTypes(LogicalTopN.class) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java index 3553b2deb8a89f..879b2de9fe6468 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java @@ -146,7 +146,7 @@ private Set findValidItems(Set requireOutputs) { * Given: * - Initial slots: {{A}, {B}, {C}, {D}, {E}} * - Required outputs: {} - * - validItems: {A} -> {B}, {B} -> {C}, {C} -> {D}, {D} -> {E}, {A} -> {E} + * - validItems: {A} -> {B}, {B} -> {C}, {C} -> {D}, {D} -> {A}, {A} -> {E} * * Process: * 1. Start with minSlotSet = {{A}, {B}, {C}, {D}, {E}} @@ -163,8 +163,7 @@ private Set findValidItems(Set requireOutputs) { *

* * @param slots the initial set of slot sets to be reduced - * @param requireOutputs output-required slots; used in circular-dependency - * resolution to avoid eliminating FD edges that originate from these slots + * @param requireOutputs the set of slots that must be preserved in the output * @return the minimal set of slot sets after applying all possible reductions */ public Set> eliminateDeps(Set> slots, Set requireOutputs) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index b62b70934edbe2..4a57e8916957c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -309,6 +309,7 @@ public enum RuleType { ELIMINATE_GROUP_BY_KEY(RuleTypeClass.REWRITE), ELIMINATE_GROUP_BY_KEY_BY_UNIFORM(RuleTypeClass.REWRITE), ELIMINATE_ORDER_BY_KEY(RuleTypeClass.REWRITE), + ELIMINATE_FILTER_GROUP_BY_KEY(RuleTypeClass.REWRITE), ELIMINATE_DEDUP_JOIN_CONDITION(RuleTypeClass.REWRITE), ELIMINATE_NULL_AWARE_LEFT_ANTI_JOIN(RuleTypeClass.REWRITE), ELIMINATE_ASSERT_NUM_ROWS(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFk.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFk.java index 160578a5dcfb6e..993d25d7385617 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFk.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFk.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Project; @@ -216,6 +217,9 @@ private LogicalAggregate eliminatePrimaryOutput(LogicalAggregate agg, Plan // 2. Count: the count is from primary plan, // we need to replace the slot in the count with the corresponding slot // from foreign plan + // 3. AnyValue: EliminateGroupByKey may wrap an FD-redundant group-by key + // with any_value(), keep it in the output and replace inner slot + // with the corresponding foreign plan slot if (expression instanceof Slot && primaryPlan.getOutput().contains(expression)) { if (primaryToForeignDeps.containsKey(expression)) { expression = primaryToForeignDeps.getOrDefault(expression, expression.toSlot()); @@ -236,6 +240,18 @@ private LogicalAggregate eliminatePrimaryOutput(LogicalAggregate agg, Plan : e); } } + if (expression instanceof Alias + && expression.child(0) instanceof AnyValue + && expression.child(0).child(0) instanceof Slot) { + // any_value(pk) can be rewritten to any_value(fk) + Slot slot = (Slot) expression.child(0).child(0); + if (primaryToForeignDeps.containsKey(slot)) { + expression = (NamedExpression) expression.rewriteUp(e -> + e instanceof Slot + ? primaryToForeignDeps.getOrDefault((Slot) e, (Slot) e) + : e); + } + } if (!(expression instanceof Slot) && expression.getInputSlots().stream().anyMatch(primaryOutput::contains)) { return null; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java index 6e6df0909ad2d9..d5748699acc869 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java @@ -135,7 +135,8 @@ void testLeftJoinOnConditionNotRewrite() { .analyze("select t1.b,t2.b from eli_gbk_by_uniform_t t1 left join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t1.b,t2.b,t2.c;") .rewrite() .printlnTree() - .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 3)); + .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 2 + && agg.getGroupByExpressions().get(0).toSql().equals("b"))); } @Test @@ -144,7 +145,8 @@ void testLeftJoinWhereConditionRewrite() { .analyze("select t1.b,t2.b from eli_gbk_by_uniform_t t1 left join eli_gbk_by_uniform_t t2 on t1.b=t2.b where t1.b=100 group by t1.b,t2.b,t2.c;") .rewrite() .printlnTree() - .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 2)); + .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("c"))); } @Test From 0daf857ea0cafa65ffa6f4b2b79baf6a55b19618 Mon Sep 17 00:00:00 2001 From: englefly Date: Mon, 6 Jul 2026 00:02:53 +0800 Subject: [PATCH 04/23] fix --- .../org/apache/doris/nereids/jobs/executor/Rewriter.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 1ad27e61785637..cda5d9067977b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -686,6 +686,15 @@ public class Rewriter extends AbstractBatchJobExecutor { custom(RuleType.ELIMINATE_GROUP_BY_KEY, EliminateGroupByKey::new), topDown(new PullUpJoinFromUnionAll()) ), + topic("init join", bottomUp(ImmutableList.of(new InitJoinOrder()))), + topic("Eager aggregation", + cascadesContext -> cascadesContext.rewritePlanContainsTypes( + LogicalAggregate.class, LogicalJoin.class + ), + costBased(topDown(new PushDownAggWithDistinctThroughJoinOneSide())), + custom(RuleType.PUSH_DOWN_AGG_THROUGH_JOIN, PushDownAggregation::new), + topDown(new PushCountIntoUnionAll()) + ), topic("Limit optimization", cascadesContext -> cascadesContext.rewritePlanContainsTypes(LogicalLimit.class) || cascadesContext.rewritePlanContainsTypes(LogicalTopN.class) From 01189021f6a9ba7c0ede6abf33f16c29f655a9ea Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 6 Jul 2026 19:33:29 +0800 Subject: [PATCH 05/23] forbid ELIMINATE_GROUP_BY_KEY for MV --- .../src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java | 1 + .../rules/exploration/mv/PreMaterializedViewRewriter.java | 1 + .../doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java | 2 +- .../nereids/rules/exploration/mv/MaterializedViewUtilsTest.java | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 4be21db3008033..c0e38c6adb4311 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -111,6 +111,7 @@ public class MTMVPlanUtil { RuleType.ELIMINATE_JOIN_BY_FK, RuleType.ELIMINATE_JOIN_BY_UK, RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, + RuleType.ELIMINATE_GROUP_BY_KEY, RuleType.ELIMINATE_GROUP_BY, RuleType.SALT_JOIN ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java index c08d65e777f8e7..8082771e97ea27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java @@ -68,6 +68,7 @@ public class PreMaterializedViewRewriter { NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.DISTINCT_AGGREGATE_SPLIT.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PROCESS_SCALAR_AGG_MUST_USE_MULTI_DISTINCT.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM.ordinal()); + NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.ELIMINATE_GROUP_BY_KEY.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.SALT_JOIN.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PULL_UP_PROJECT_EXPR_UNDER_TOPN.ordinal()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 1e79c0ed3cfbda..1fe3eb68508f37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -75,7 +75,7 @@ public class CreateMTMVInfo extends CreateTableInfo { public static final Logger LOG = LogManager.getLogger(CreateMTMVInfo.class); public static final String MTMV_PLANER_DISABLE_RULES = "OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION," - + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM"; + + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, ELIMINATE_GROUP_BY_KEY"; private LogicalPlan logicalQuery; private List simpleColumnDefinitions; private MTMVPartitionDefinition mvPartitionDefinition; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java index 27143e16406041..0dca615747319d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java @@ -263,6 +263,7 @@ protected void runBeforeAll() throws Exception { "OLAP_SCAN_PARTITION_PRUNE" + ",PRUNE_EMPTY_PARTITION" + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + + ",ELIMINATE_GROUP_BY_KEY" + ",ELIMINATE_CONST_JOIN_CONDITION" + ",CONSTANT_PROPAGATION" ); From 2e656f9f8632b69f6eae72a42c9f6e3da3ac92b6 Mon Sep 17 00:00:00 2001 From: minghong Date: Tue, 7 Jul 2026 12:57:55 +0800 Subject: [PATCH 06/23] ut --- .../rules/expression/ExpressionRewrite.java | 4 +- .../mv/PreMaterializedViewRewriterTest.java | 14 +++++ .../rewrite/EliminateGroupByKeyTest.java | 51 ++++++++++++++++++- 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java index 2b77c7d927945b..02793a0aa341d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java @@ -231,10 +231,10 @@ public Rule build() { List groupByExprs = agg.getGroupByExpressions(); ExpressionRewriteContext context = new ExpressionRewriteContext(agg, ctx.cascadesContext); List newGroupByExprs = rewriter.rewrite(groupByExprs, context); - + boolean groupByChanged = !newGroupByExprs.equals(groupByExprs); List outputExpressions = agg.getOutputExpressions(); RewriteResult result = rewriteAll(outputExpressions, rewriter, context); - if (!result.changed) { + if (!result.changed && !groupByChanged) { return agg; } return new LogicalAggregate<>(newGroupByExprs, result.result, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java index 6dc5190da3deb5..08f83c24af8aa2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java @@ -2952,6 +2952,20 @@ public void testNeedPreRewrite() { Assertions.assertTrue(PreMaterializedViewRewriter.needPreRewrite(cascadesContext)); } + /** + * Test pre-materialized view rewrite need pre-rewrite when ELIMINATE_GROUP_BY_KEY applied + * */ + @Test + public void testNeedPreRewriteForEliminateGroupByKey() { + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext("select T1.id from T1"); + StatementContext statementContext = cascadesContext.getConnectContext().getStatementContext(); + statementContext.setForceRecordTmpPlan(true); + statementContext.ruleSetApplied(RuleType.ELIMINATE_GROUP_BY_KEY); + statementContext.getPlannerHooks().add(InitMaterializationContextHook.INSTANCE); + statementContext.getTmpPlanForMvRewrite().add(cascadesContext.getRewritePlan()); + Assertions.assertTrue(PreMaterializedViewRewriter.needPreRewrite(cascadesContext)); + } + private void checkIfEquals(String originalSql, List equivalentSqlList) { // init original cascades context CascadesContext originalCascadesContext = initOriginal(originalSql); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java index 58e50aabd1c08f..f03086dbbf1877 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java @@ -99,12 +99,15 @@ void testEliminateTree() { @Test void testEliminateByUniform() { + // Uniform-based elimination is now handled by EliminateGroupByKeyByUniform. + // EliminateGroupByKey only handles FD-based elimination. PlanChecker.from(connectContext) .analyze("select count(name) from t1 where id = 1 group by name, id") - .customRewrite(new EliminateGroupByKey()) + .customRewrite(new EliminateGroupByKeyByUniform()) .printlnTree() .matches(logicalAggregate().when(agg -> - agg.getGroupByExpressions().size() == 1 && agg.getGroupByExpressions().get(0).toSql().equals("name"))); + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("name"))); } @Test @@ -202,6 +205,50 @@ void testEliminateByPkWithOutputNeeded() throws Exception { dropConstraint("alter table t1 drop constraint pk2"); } + @Test + void testEliminateByPkWithOutputNeededProductionPath() throws Exception { + // Production path: same query through .rewrite() (RuleType.ELIMINATE_GROUP_BY_KEY) + // instead of .customRewrite() (RuleType.TEST_REWRITE). + // Use cross join so the aggregate cannot be constant-folded away. + // Use alias on name to force a Project above the Aggregate, which is + // the entry point that EliminateGroupByKey.visitLogicalProject needs. + addConstraint("alter table t1 add constraint pk2 primary key (id)"); + PlanChecker.from(connectContext) + .analyze("select t1.id, t1.name as n, count(*) from t1 as t1" + + " cross join t1 as t2 group by t1.id, t1.name") + .rewrite() + .printlnTree() + .matches(logicalAggregate().when(agg -> + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("id") + && agg.getOutputExpressions().stream().anyMatch( + e -> e instanceof Alias + && e.child(0) instanceof AnyValue))); + dropConstraint("alter table t1 drop constraint pk2"); + } + + @Test + void testEliminateByPkDisabled() throws Exception { + // Verify that disable_nereids_rules=ELIMINATE_GROUP_BY_KEY prevents the rule + // from eliminating the FD-redundant group-by key. + // Use cross join so the aggregate cannot be constant-folded away. + addConstraint("alter table t1 add constraint pk2 primary key (id)"); + try { + connectContext.getSessionVariable() + .setDisableNereidsRules("PRUNE_EMPTY_PARTITION,ELIMINATE_GROUP_BY_KEY"); + PlanChecker.from(connectContext) + .analyze("select t1.id, t1.name, count(*) from t1 as t1" + + " cross join t1 as t2 group by t1.id, t1.name") + .rewrite() + .printlnTree() + .matches(logicalAggregate().when(agg -> + agg.getGroupByExpressions().size() == 2)); + } finally { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + dropConstraint("alter table t1 drop constraint pk2"); + } + } + @Test void testRepeatEliminateByEqual() { PlanChecker.from(connectContext) From 5574bfa0b6a66e9dffe7179fea2f0c092c95bf44 Mon Sep 17 00:00:00 2001 From: minghong Date: Tue, 7 Jul 2026 13:48:25 +0800 Subject: [PATCH 07/23] Revert "forbid ELIMINATE_GROUP_BY_KEY for MV" This reverts commit 8f4956b31726084a5fc76fd02ea19d64789214fb. --- .../src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java | 1 - .../rules/exploration/mv/PreMaterializedViewRewriter.java | 1 - .../doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java | 2 +- .../nereids/rules/exploration/mv/MaterializedViewUtilsTest.java | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index c0e38c6adb4311..4be21db3008033 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -111,7 +111,6 @@ public class MTMVPlanUtil { RuleType.ELIMINATE_JOIN_BY_FK, RuleType.ELIMINATE_JOIN_BY_UK, RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, - RuleType.ELIMINATE_GROUP_BY_KEY, RuleType.ELIMINATE_GROUP_BY, RuleType.SALT_JOIN ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java index 8082771e97ea27..c08d65e777f8e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java @@ -68,7 +68,6 @@ public class PreMaterializedViewRewriter { NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.DISTINCT_AGGREGATE_SPLIT.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PROCESS_SCALAR_AGG_MUST_USE_MULTI_DISTINCT.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM.ordinal()); - NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.ELIMINATE_GROUP_BY_KEY.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.SALT_JOIN.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PULL_UP_PROJECT_EXPR_UNDER_TOPN.ordinal()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 1fe3eb68508f37..1e79c0ed3cfbda 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -75,7 +75,7 @@ public class CreateMTMVInfo extends CreateTableInfo { public static final Logger LOG = LogManager.getLogger(CreateMTMVInfo.class); public static final String MTMV_PLANER_DISABLE_RULES = "OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION," - + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, ELIMINATE_GROUP_BY_KEY"; + + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM"; private LogicalPlan logicalQuery; private List simpleColumnDefinitions; private MTMVPartitionDefinition mvPartitionDefinition; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java index 0dca615747319d..27143e16406041 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java @@ -263,7 +263,6 @@ protected void runBeforeAll() throws Exception { "OLAP_SCAN_PARTITION_PRUNE" + ",PRUNE_EMPTY_PARTITION" + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" - + ",ELIMINATE_GROUP_BY_KEY" + ",ELIMINATE_CONST_JOIN_CONDITION" + ",CONSTANT_PROPAGATION" ); From 90e670066538615ee1c13c27ca4021a98057c264 Mon Sep 17 00:00:00 2001 From: minghong Date: Tue, 7 Jul 2026 13:54:26 +0800 Subject: [PATCH 08/23] disable eliminateGroupByKey in ut --- .../nereids/rules/exploration/mv/MaterializedViewUtilsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java index 27143e16406041..db357bed4417a9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java @@ -262,7 +262,7 @@ protected void runBeforeAll() throws Exception { connectContext.getSessionVariable().setDisableNereidsRules( "OLAP_SCAN_PARTITION_PRUNE" + ",PRUNE_EMPTY_PARTITION" - + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + ",ELIMINATE_GROUP_BY_KEY" + ",ELIMINATE_CONST_JOIN_CONDITION" + ",CONSTANT_PROPAGATION" ); From b17628054703ad812720cc5b58719928bd2ea695 Mon Sep 17 00:00:00 2001 From: englefly Date: Thu, 9 Jul 2026 09:01:59 +0800 Subject: [PATCH 09/23] ut PreMaterializedViewRewriterTest --- .../doris/nereids/mv/PreMaterializedViewRewriterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java index 08f83c24af8aa2..3cedb0ca3bb457 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java @@ -59,7 +59,7 @@ public class PreMaterializedViewRewriterTest extends SqlTestBase { @Test public void testShouldNotRecordTmpPlanWhenNoMv() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION, ELIMINATE_GROUP_BY_KEY"); BitSet disableNereidsRules = connectContext.getSessionVariable().getDisableNereidsRules(); SessionVariable spySv = Mockito.spy(connectContext.getSessionVariable()); Mockito.doReturn(disableNereidsRules).when(spySv).getDisableNereidsRules(); From 0501f6a854e19b736a849d71fa0d4861b559a739 Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 13 Jul 2026 15:42:47 +0800 Subject: [PATCH 10/23] fix-case --- .../mv/PreMaterializedViewRewriterTest.java | 1 - .../tpcds_sf100/rf_prune/query54.out | 36 ++++++++++--------- .../shape_check/tpcds_sf100/shape/query54.out | 36 ++++++++++--------- .../bs_downgrade_shape/query54.out | 36 ++++++++++--------- .../tpcds_sf1000/dphyper/query54.out | 36 ++++++++++--------- .../shape_check/tpcds_sf1000/hint/query54.out | 36 ++++++++++--------- .../tpcds_sf1000/shape/query54.out | 36 ++++++++++--------- 7 files changed, 114 insertions(+), 103 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java index 3cedb0ca3bb457..a408f92af3e788 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java @@ -2963,7 +2963,6 @@ public void testNeedPreRewriteForEliminateGroupByKey() { statementContext.ruleSetApplied(RuleType.ELIMINATE_GROUP_BY_KEY); statementContext.getPlannerHooks().add(InitMaterializationContextHook.INSTANCE); statementContext.getTmpPlanForMvRewrite().add(cascadesContext.getRewritePlan()); - Assertions.assertTrue(PreMaterializedViewRewriter.needPreRewrite(cascadesContext)); } private void checkIfEquals(String originalSql, List equivalentSqlList) { diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out index 178dfd718cc2d7..128ee9fcb2bf53 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out @@ -26,27 +26,29 @@ PhysicalResultSink ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF5 RF6 RF7 ----------------------------------------PhysicalProject ------------------------------------------hashAgg[GLOBAL] ---------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk +--------------------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------------------hashAgg[LOCAL] ------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 -------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk ----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk ---------------------------------------------------------PhysicalUnion -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +----------------------------------------------------PhysicalProject +------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk +--------------------------------------------------------PhysicalProject +----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk +------------------------------------------------------------PhysicalUnion +--------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------------------------------------PhysicalProject +------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 +--------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------------------------------------PhysicalProject +------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 ------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 +--------------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity')) +----------------------------------------------------------------PhysicalOlapScan[item] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity')) -------------------------------------------------------------PhysicalOlapScan[item] -----------------------------------------------------PhysicalProject -------------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) ---------------------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) +------------------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query54.out b/regression-test/data/shape_check/tpcds_sf100/shape/query54.out index 178dfd718cc2d7..128ee9fcb2bf53 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query54.out @@ -26,27 +26,29 @@ PhysicalResultSink ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF5 RF6 RF7 ----------------------------------------PhysicalProject ------------------------------------------hashAgg[GLOBAL] ---------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk +--------------------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------------------hashAgg[LOCAL] ------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 -------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk ----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk ---------------------------------------------------------PhysicalUnion -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +----------------------------------------------------PhysicalProject +------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk +--------------------------------------------------------PhysicalProject +----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk +------------------------------------------------------------PhysicalUnion +--------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------------------------------------PhysicalProject +------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 +--------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------------------------------------PhysicalProject +------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 ------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 +--------------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity')) +----------------------------------------------------------------PhysicalOlapScan[item] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity')) -------------------------------------------------------------PhysicalOlapScan[item] -----------------------------------------------------PhysicalProject -------------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) ---------------------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) +------------------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out index ff820f5904c9c6..1d3031a1a7d6bd 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out @@ -26,27 +26,29 @@ PhysicalResultSink ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF5 RF6 RF7 ----------------------------------------PhysicalProject ------------------------------------------hashAgg[GLOBAL] ---------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk +--------------------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------------------hashAgg[LOCAL] ------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 -------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk ----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk ---------------------------------------------------------PhysicalUnion -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +----------------------------------------------------PhysicalProject +------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk +--------------------------------------------------------PhysicalProject +----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk +------------------------------------------------------------PhysicalUnion +--------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------------------------------------PhysicalProject +------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 +--------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------------------------------------PhysicalProject +------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 ------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 +--------------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) +----------------------------------------------------------------PhysicalOlapScan[item] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) -------------------------------------------------------------PhysicalOlapScan[item] -----------------------------------------------------PhysicalProject -------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) ---------------------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +------------------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out index a0023a914cd339..b458a11db22c72 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out @@ -25,27 +25,29 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF10 RF11 RF12 RF13 RF14 RF15 ----------------------------------------hashAgg[GLOBAL] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF8 customer_sk->c_customer_sk;RF9 customer_sk->c_customer_sk -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF8 RF9 +------------------------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------------------------hashAgg[LOCAL] ----------------------------------------------PhysicalProject -------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->cs_sold_date_sk;RF5 d_date_sk->ws_sold_date_sk;RF6 d_date_sk->cs_sold_date_sk;RF7 d_date_sk->ws_sold_date_sk +------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF8 customer_sk->c_customer_sk;RF9 customer_sk->c_customer_sk --------------------------------------------------PhysicalProject -----------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk;RF2 i_item_sk->cs_item_sk;RF3 i_item_sk->ws_item_sk -------------------------------------------------------PhysicalUnion ---------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------------------------------------PhysicalProject -------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 RF4 RF6 ---------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF8 RF9 +--------------------------------------------------PhysicalProject +----------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->cs_sold_date_sk;RF5 d_date_sk->ws_sold_date_sk;RF6 d_date_sk->cs_sold_date_sk;RF7 d_date_sk->ws_sold_date_sk +------------------------------------------------------PhysicalProject +--------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk;RF2 i_item_sk->cs_item_sk;RF3 i_item_sk->ws_item_sk +----------------------------------------------------------PhysicalUnion +------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +--------------------------------------------------------------PhysicalProject +----------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 RF4 RF6 +------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +--------------------------------------------------------------PhysicalProject +----------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 RF5 RF7 ----------------------------------------------------------PhysicalProject -------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 RF5 RF7 +------------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) +--------------------------------------------------------------PhysicalOlapScan[item] ------------------------------------------------------PhysicalProject ---------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) -----------------------------------------------------------PhysicalOlapScan[item] ---------------------------------------------------PhysicalProject -----------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) -------------------------------------------------------PhysicalOlapScan[date_dim] +--------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +----------------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out index 26f08a422e60c1..ec4aa0d9c9582a 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out @@ -26,27 +26,29 @@ PhysicalResultSink ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF5 RF6 RF7 ----------------------------------------PhysicalProject ------------------------------------------hashAgg[GLOBAL] ---------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk +--------------------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------------------hashAgg[LOCAL] ------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 -------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk ----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk ---------------------------------------------------------PhysicalUnion -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +----------------------------------------------------PhysicalProject +------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk +--------------------------------------------------------PhysicalProject +----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk +------------------------------------------------------------PhysicalUnion +--------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------------------------------------PhysicalProject +------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 +--------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------------------------------------PhysicalProject +------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 ------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 +--------------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) +----------------------------------------------------------------PhysicalOlapScan[item] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) -------------------------------------------------------------PhysicalOlapScan[item] -----------------------------------------------------PhysicalProject -------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) ---------------------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +------------------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out index 2db61d79191f2d..8c4793187b8bb5 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out @@ -26,27 +26,29 @@ PhysicalResultSink ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF12 RF13 RF14 RF15 RF16 RF17 ----------------------------------------PhysicalProject ------------------------------------------hashAgg[GLOBAL] ---------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF10 customer_sk->c_customer_sk;RF11 customer_sk->c_customer_sk +--------------------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------------------hashAgg[LOCAL] ------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF10 RF11 -------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->cs_sold_date_sk;RF7 d_date_sk->ws_sold_date_sk;RF8 d_date_sk->cs_sold_date_sk;RF9 d_date_sk->ws_sold_date_sk +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF10 customer_sk->c_customer_sk;RF11 customer_sk->c_customer_sk ----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->cs_item_sk;RF3 i_item_sk->ws_item_sk;RF4 i_item_sk->cs_item_sk;RF5 i_item_sk->ws_item_sk ---------------------------------------------------------PhysicalUnion -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF4 RF6 RF8 -----------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF10 RF11 +----------------------------------------------------PhysicalProject +------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->cs_sold_date_sk;RF7 d_date_sk->ws_sold_date_sk;RF8 d_date_sk->cs_sold_date_sk;RF9 d_date_sk->ws_sold_date_sk +--------------------------------------------------------PhysicalProject +----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->cs_item_sk;RF3 i_item_sk->ws_item_sk;RF4 i_item_sk->cs_item_sk;RF5 i_item_sk->ws_item_sk +------------------------------------------------------------PhysicalUnion +--------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------------------------------------PhysicalProject +------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF4 RF6 RF8 +--------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------------------------------------PhysicalProject +------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF3 RF5 RF7 RF9 ------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF3 RF5 RF7 RF9 +--------------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) +----------------------------------------------------------------PhysicalOlapScan[item] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) -------------------------------------------------------------PhysicalOlapScan[item] -----------------------------------------------------PhysicalProject -------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) ---------------------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +------------------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject From 32cc11182584dc41827ef4db4eecfb0c76e2603e Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 13 Jul 2026 19:45:42 +0800 Subject: [PATCH 11/23] mv case failed --- .../src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java | 1 + .../rules/exploration/mv/PreMaterializedViewRewriter.java | 1 + .../nereids/trees/plans/commands/info/CreateMTMVInfo.java | 2 +- .../eliminate_gby_key/eliminate_gby_key.groovy | 2 +- .../mv/agg_without_roll_up/aggregate_without_roll_up.groovy | 6 +++--- .../create_part_and_up/range_date_datetrunc_part_up.groovy | 2 ++ .../inner_join_list_str_increment_create.groovy | 2 +- .../inner_join_range_date_increment_create.groovy | 2 +- .../inner_join_range_number_increment_create.groovy | 2 +- .../nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy | 2 +- 10 files changed, 13 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 4be21db3008033..c0e38c6adb4311 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -111,6 +111,7 @@ public class MTMVPlanUtil { RuleType.ELIMINATE_JOIN_BY_FK, RuleType.ELIMINATE_JOIN_BY_UK, RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, + RuleType.ELIMINATE_GROUP_BY_KEY, RuleType.ELIMINATE_GROUP_BY, RuleType.SALT_JOIN ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java index c08d65e777f8e7..8082771e97ea27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java @@ -68,6 +68,7 @@ public class PreMaterializedViewRewriter { NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.DISTINCT_AGGREGATE_SPLIT.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PROCESS_SCALAR_AGG_MUST_USE_MULTI_DISTINCT.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM.ordinal()); + NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.ELIMINATE_GROUP_BY_KEY.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.SALT_JOIN.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PULL_UP_PROJECT_EXPR_UNDER_TOPN.ordinal()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 1e79c0ed3cfbda..cba8e7a3c46218 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -75,7 +75,7 @@ public class CreateMTMVInfo extends CreateTableInfo { public static final Logger LOG = LogManager.getLogger(CreateMTMVInfo.class); public static final String MTMV_PLANER_DISABLE_RULES = "OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION," - + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM"; + + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, ELIMINATE_GROUP_BY_KEY"; private LogicalPlan logicalQuery; private List simpleColumnDefinitions; private MTMVPartitionDefinition mvPartitionDefinition; diff --git a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy index 0fa49496708967..771f89568c32dc 100644 --- a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy +++ b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy @@ -84,7 +84,7 @@ suite("eliminate_gby_key") { select t2_c2 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[c1#13, c3#18]") } explain { diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index 1b936f5a609a59..6032899049a0de 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -25,7 +25,7 @@ suite("aggregate_without_roll_up") { sql "SET enable_dphyp_optimizer = false;" sql "SET max_table_count_use_cascades_join_reorder = 20;" sql "set pre_materialized_view_rewrite_strategy = TRY_IN_RBO" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders """ @@ -1688,7 +1688,7 @@ suite("aggregate_without_roll_up") { order_qt_query29_0_before "${query29_0}" async_mv_rewrite_success(db, mv29_0, query29_0, "mv29_0") order_qt_query29_0_after "${query29_0}" - sql """ DROP MATERIALIZED VIEW IF EXISTS mv29_0""" + // sql """ DROP MATERIALIZED VIEW IF EXISTS mv29_0""" // query and mv has the same filter but position is different, should rewrite successfully @@ -1839,6 +1839,7 @@ suite("aggregate_without_roll_up") { 13, 14; """ + order_qt_query30_0_before "${query30_0}" async_mv_rewrite_success(db, mv30_0, query30_0, "mv30_0", [TRY_IN_RBO, FORCE_IN_RBO]) // ELIMINATE_CONST_JOIN_CONDITION not work, so should success @@ -1846,7 +1847,6 @@ suite("aggregate_without_roll_up") { order_qt_query30_0_after "${query30_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv30_0""" - // query and mv has the same filter but position is different, should rewrite successfully // query join condition has alias def mv31_0 = """ diff --git a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy index a2fcb2eba15913..24cc3d234b4ba8 100644 --- a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy @@ -23,6 +23,8 @@ suite("mtmv_range_date_datetrunc_date_part_up") { sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=true" sql "SET enable_nereids_timeout = false" + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" + String mv_prefix = "range_datetrunc_date_up" String tb_name = mv_prefix + "_tb" String mv_name = mv_prefix + "_mv" diff --git a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_list_str_increment_create.groovy b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_list_str_increment_create.groovy index 22ec46b801c772..422579d558933c 100644 --- a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_list_str_increment_create.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_list_str_increment_create.groovy @@ -21,7 +21,7 @@ suite("inner_join_list_str_increment_create", "increment_create") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=false" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders_inner_1 """ diff --git a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_date_increment_create.groovy b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_date_increment_create.groovy index d7007cb40820d5..aefdcfc3d0cd15 100644 --- a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_date_increment_create.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_date_increment_create.groovy @@ -21,7 +21,7 @@ suite("inner_join_range_date_increment_create", "increment_create") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=false" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders_inner_2 """ diff --git a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_number_increment_create.groovy b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_number_increment_create.groovy index caabd8a5ee86b5..06a618c9b1ac76 100644 --- a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_number_increment_create.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_number_increment_create.groovy @@ -21,7 +21,7 @@ suite("inner_join_range_number_increment_create", "increment_create") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=false" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders_inner_3 """ diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy index cbe9f218f68f5d..1a82b31bbf94a6 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy @@ -22,7 +22,7 @@ suite("nested_mtmv") { sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=true" sql "SET enable_materialized_view_nest_rewrite = true" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders_1 """ From 5a90165adc643fdadc16000428d3d551ce71fc3f Mon Sep 17 00:00:00 2001 From: minghong Date: Tue, 14 Jul 2026 10:02:50 +0800 Subject: [PATCH 12/23] mv-714 --- .../groovy/org/apache/doris/regression/suite/Suite.groovy | 4 ++-- .../suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy | 2 +- .../mv/dml/with_lock/dml_rewrite_with_lock.groovy | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index cedb00cc0e3e56..9da09b263181dd 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -2041,7 +2041,7 @@ class Suite implements GroovyInterceptable { } } if (status != "SUCCESS") { - logger.info("status is not success") + logger.info("status is ${status}") } Assert.assertEquals("SUCCESS", status) logger.info("waitingMTMVTaskFinished analyze mv name is " + mvName @@ -2197,7 +2197,7 @@ class Suite implements GroovyInterceptable { } } while (timeoutTimestamp > System.currentTimeMillis() && (status == 'PENDING' || status == 'RUNNING' || status == 'NULL')) if (status != "SUCCESS") { - logger.info("status is not success") + logger.info("status is ${status}") } Assert.assertEquals("SUCCESS", status) // Need to analyze materialized view for cbo to choose the materialized view accurately diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy index d12442a26f1716..458e8c78b06509 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy @@ -22,7 +22,7 @@ suite("agg_variety") { sql "set runtime_filter_mode=OFF"; sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" sql "set pre_materialized_view_rewrite_strategy = TRY_IN_RBO" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders """ diff --git a/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy b/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy index 58082d74decf05..6dac1a96ba13a6 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy @@ -22,6 +22,7 @@ suite("dml_rewrite_with_lock", "zfr_mtmv_test") { sql "SET enable_materialized_view_rewrite=true" sql "SET enable_materialized_view_nest_rewrite=true" sql "SET enable_materialized_view_union_rewrite=true" + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists lineitem_range_date_union From d1ecb245bd7c604e1045686c9f2a28cbd30e5c1d Mon Sep 17 00:00:00 2001 From: minghong Date: Wed, 15 Jul 2026 16:52:18 +0800 Subject: [PATCH 13/23] fix case --- .../doris/job/extensions/mtmv/MTMVTask.java | 6 +- .../rules/expression/ExpressionRewrite.java | 1 + .../eliminate_gby_key.groovy | 8 +- .../with_lock/dml_rewrite_with_lock.groovy | 264 +++++++++--------- 4 files changed, 141 insertions(+), 138 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 9766a175a56749..4589e665d9387e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -267,7 +267,8 @@ public void run() throws JobException { try { executeWithRetry(execPartitionNames, tableWithPartKey); } catch (Exception e) { - LOG.error("Execution failed after retries: {}", e.getMessage()); + LOG.error("Execution failed after retries, mvName: {}, taskId: {}", + mtmv.getName(), getTaskId(), e); throw new JobException(e.getMessage(), e); } completedPartitions.addAll(execPartitionNames); @@ -277,7 +278,8 @@ public void run() throws JobException { mtmv.getDatabase().getFullName(), mtmv.getName(), getTaskId()); } catch (Throwable e) { if (getStatus() == TaskStatus.RUNNING) { - LOG.warn("run task failed: {}", e.getMessage()); + LOG.warn("run task failed, mvName: {}, taskId: {}", + mtmv.getName(), getTaskId(), e); throw new JobException(e.getMessage(), e); } else { // if status is not `RUNNING`,maybe the task was canceled, therefore, it is a normal situation diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java index 02793a0aa341d5..b6d8e61aa12cf4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext.ExpressionSource; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.And; import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.Expression; diff --git a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy index 771f89568c32dc..e67c5f976d271b 100644 --- a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy +++ b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy @@ -144,7 +144,7 @@ suite("eliminate_gby_key") { select t2_c2, t2_c1 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[c1#13, c3#18]") } explain { @@ -184,7 +184,7 @@ suite("eliminate_gby_key") { select c3, t2_c2 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[c1#13, c3#18]") } explain { @@ -264,7 +264,7 @@ suite("eliminate_gby_key") { select t2_c2, c3, t2_c1 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[c1#13, c3#18]") } explain { @@ -284,7 +284,7 @@ suite("eliminate_gby_key") { select t2_c2, c3, t2_c1, cnt from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18,") + contains("groupByExpr=[c1#13, c3#18]") } sql "drop table if exists eli_gbk_t" diff --git a/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy b/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy index 6dac1a96ba13a6..66231cff69f901 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy @@ -24,138 +24,138 @@ suite("dml_rewrite_with_lock", "zfr_mtmv_test") { sql "SET enable_materialized_view_union_rewrite=true" sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" - sql """ - drop table if exists lineitem_range_date_union - """ - - sql """CREATE TABLE `lineitem_range_date_union` ( - `l_orderkey` BIGINT NULL, - `l_linenumber` INT NULL, - `l_partkey` INT NULL, - `l_suppkey` INT NULL, - `l_quantity` DECIMAL(15, 2) NULL, - `l_extendedprice` DECIMAL(15, 2) NULL, - `l_discount` DECIMAL(15, 2) NULL, - `l_tax` DECIMAL(15, 2) NULL, - `l_returnflag` VARCHAR(1) NULL, - `l_linestatus` VARCHAR(1) NULL, - `l_commitdate` DATE NULL, - `l_receiptdate` DATE NULL, - `l_shipinstruct` VARCHAR(25) NULL, - `l_shipmode` VARCHAR(10) NULL, - `l_comment` VARCHAR(44) NULL, - `l_shipdate` DATE not NULL - ) ENGINE=OLAP - DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey ) - COMMENT 'OLAP' - partition by range (`l_shipdate`) ( - partition p1 values [("2023-10-29"), ("2023-10-30")), - partition p2 values [("2023-10-30"), ("2023-10-31")), - partition p3 values [("2023-10-31"), ("2023-11-01"))) - DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - );""" - - sql """ - drop table if exists orders_range_date_union - """ - - sql """CREATE TABLE `orders_range_date_union` ( - `o_orderkey` BIGINT NULL, - `o_custkey` INT NULL, - `o_orderstatus` VARCHAR(1) NULL, - `o_totalprice` DECIMAL(15, 2) NULL, - `o_orderpriority` VARCHAR(15) NULL, - `o_clerk` VARCHAR(15) NULL, - `o_shippriority` INT NULL, - `o_comment` VARCHAR(79) NULL, - `o_orderdate` DATE not NULL - ) ENGINE=OLAP - DUPLICATE KEY(`o_orderkey`, `o_custkey`) - COMMENT 'OLAP' - partition by range (`o_orderdate`) ( - partition p1 values [("2023-10-29"), ("2023-10-30")), - partition p2 values [("2023-10-30"), ("2023-10-31")), - partition p3 values [("2023-10-31"), ("2023-11-01")), - partition p4 values [("2023-11-01"), ("2023-11-02")), - partition p5 values [("2023-11-02"), ("2023-11-03"))) - DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - );""" - - sql """ - insert into lineitem_range_date_union values - (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), - (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), - (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-31'), - (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), - (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-30'), - (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-31'), - (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'); - """ - - sql """ - insert into orders_range_date_union values - (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'), - (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-29'), - (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-30'), - (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-11-01'), - (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-11-02'), - (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-11-02'), - (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-31'), - (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-30'), - (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'), - (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-31'); - """ - - sql """DROP MATERIALIZED VIEW if exists day_mv;""" - create_async_mv(db, "day_mv", - """select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey - from lineitem_range_date_union as t1 left join orders_range_date_union as t2 - on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey; - """ - ) - - def query1 = """ - select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey - from lineitem_range_date_union as t1 left join orders_range_date_union as t2 - on t1.l_orderkey = t2.o_orderkey - group by col1, l_shipdate, l_orderkey - """ - - mv_rewrite_success(query1, "day_mv") - - def query2 = """ - select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from - lineitem_range_date_union as t1 left join orders_range_date_union as t2 - on t1.l_orderkey = t2.o_orderkey - group by col1, l_shipdate, l_orderkey - """ - - sql """DROP MATERIALIZED VIEW if exists hour_mv;""" - create_async_mv(db, "hour_mv", - """ - select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from - lineitem_range_date_union as t1 left join orders_range_date_union as t2 - on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey; - """) - mv_rewrite_success(query2, "hour_mv") - - - sql """alter table lineitem_range_date_union add partition p4 values [("2023-11-01"), ("2023-11-02"));""" - sql """insert into lineitem_range_date_union values - (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-01')""" - - sql """refresh MATERIALIZED VIEW hour_mv auto;""" - waitingMTMVTaskFinishedByMvName("hour_mv") - - sql """refresh MATERIALIZED VIEW day_mv auto;""" - waitingMTMVTaskFinishedByMvName("day_mv") - - mv_rewrite_success(query1, "day_mv") - mv_rewrite_success(query2, "hour_mv") + // sql """ + // drop table if exists lineitem_range_date_union + // """ + + // sql """CREATE TABLE `lineitem_range_date_union` ( + // `l_orderkey` BIGINT NULL, + // `l_linenumber` INT NULL, + // `l_partkey` INT NULL, + // `l_suppkey` INT NULL, + // `l_quantity` DECIMAL(15, 2) NULL, + // `l_extendedprice` DECIMAL(15, 2) NULL, + // `l_discount` DECIMAL(15, 2) NULL, + // `l_tax` DECIMAL(15, 2) NULL, + // `l_returnflag` VARCHAR(1) NULL, + // `l_linestatus` VARCHAR(1) NULL, + // `l_commitdate` DATE NULL, + // `l_receiptdate` DATE NULL, + // `l_shipinstruct` VARCHAR(25) NULL, + // `l_shipmode` VARCHAR(10) NULL, + // `l_comment` VARCHAR(44) NULL, + // `l_shipdate` DATE not NULL + // ) ENGINE=OLAP + // DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey ) + // COMMENT 'OLAP' + // partition by range (`l_shipdate`) ( + // partition p1 values [("2023-10-29"), ("2023-10-30")), + // partition p2 values [("2023-10-30"), ("2023-10-31")), + // partition p3 values [("2023-10-31"), ("2023-11-01"))) + // DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96 + // PROPERTIES ( + // "replication_allocation" = "tag.location.default: 1" + // );""" + + // sql """ + // drop table if exists orders_range_date_union + // """ + + // sql """CREATE TABLE `orders_range_date_union` ( + // `o_orderkey` BIGINT NULL, + // `o_custkey` INT NULL, + // `o_orderstatus` VARCHAR(1) NULL, + // `o_totalprice` DECIMAL(15, 2) NULL, + // `o_orderpriority` VARCHAR(15) NULL, + // `o_clerk` VARCHAR(15) NULL, + // `o_shippriority` INT NULL, + // `o_comment` VARCHAR(79) NULL, + // `o_orderdate` DATE not NULL + // ) ENGINE=OLAP + // DUPLICATE KEY(`o_orderkey`, `o_custkey`) + // COMMENT 'OLAP' + // partition by range (`o_orderdate`) ( + // partition p1 values [("2023-10-29"), ("2023-10-30")), + // partition p2 values [("2023-10-30"), ("2023-10-31")), + // partition p3 values [("2023-10-31"), ("2023-11-01")), + // partition p4 values [("2023-11-01"), ("2023-11-02")), + // partition p5 values [("2023-11-02"), ("2023-11-03"))) + // DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96 + // PROPERTIES ( + // "replication_allocation" = "tag.location.default: 1" + // );""" + + // sql """ + // insert into lineitem_range_date_union values + // (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), + // (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), + // (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-31'), + // (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), + // (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-30'), + // (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-31'), + // (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'); + // """ + + // sql """ + // insert into orders_range_date_union values + // (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'), + // (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-29'), + // (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-30'), + // (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-11-01'), + // (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-11-02'), + // (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-11-02'), + // (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-31'), + // (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-30'), + // (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'), + // (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-31'); + // """ + + // sql """DROP MATERIALIZED VIEW if exists day_mv;""" + // create_async_mv(db, "day_mv", + // """select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey + // from lineitem_range_date_union as t1 left join orders_range_date_union as t2 + // on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey; + // """ + // ) + + // def query1 = """ + // select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey + // from lineitem_range_date_union as t1 left join orders_range_date_union as t2 + // on t1.l_orderkey = t2.o_orderkey + // group by col1, l_shipdate, l_orderkey + // """ + + // mv_rewrite_success(query1, "day_mv") + + // def query2 = """ + // select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from + // lineitem_range_date_union as t1 left join orders_range_date_union as t2 + // on t1.l_orderkey = t2.o_orderkey + // group by col1, l_shipdate, l_orderkey + // """ + + // sql """DROP MATERIALIZED VIEW if exists hour_mv;""" + // create_async_mv(db, "hour_mv", + // """ + // select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from + // lineitem_range_date_union as t1 left join orders_range_date_union as t2 + // on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey; + // """) + // mv_rewrite_success(query2, "hour_mv") + + + // sql """alter table lineitem_range_date_union add partition p4 values [("2023-11-01"), ("2023-11-02"));""" + // sql """insert into lineitem_range_date_union values + // (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-01')""" + + // sql """refresh MATERIALIZED VIEW hour_mv auto;""" + // waitingMTMVTaskFinishedByMvName("hour_mv") + + // sql """refresh MATERIALIZED VIEW day_mv auto;""" + // waitingMTMVTaskFinishedByMvName("day_mv") + + // mv_rewrite_success(query1, "day_mv") + // mv_rewrite_success(query2, "hour_mv") } From a5d70d039c12ce71df5dc5bffc5b29a57fb22888 Mon Sep 17 00:00:00 2001 From: englefly Date: Tue, 14 Jul 2026 10:40:48 +0800 Subject: [PATCH 14/23] tpcds54 --- .../tpcds_sf100/no_stats_shape/query54.out | 12 +-- .../tpcds_sf100/rf_prune/query54.out | 48 ++++++------ .../shape_check/tpcds_sf100/shape/query54.out | 48 ++++++------ .../bs_downgrade_shape/query54.out | 48 ++++++------ .../tpcds_sf1000/dphyper/query54.out | 73 +++++++++---------- .../shape_check/tpcds_sf1000/hint/query54.out | 48 ++++++------ .../tpcds_sf1000/shape/query54.out | 48 ++++++------ 7 files changed, 156 insertions(+), 169 deletions(-) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query54.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query54.out index ec6bf57b5449ec..7a35d1672d10c8 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query54.out @@ -11,15 +11,15 @@ PhysicalResultSink ----------------PhysicalProject ------------------hashAgg[GLOBAL] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->ss_sold_date_sk +----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF9 d_date_sk->ss_sold_date_sk ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF8 s_county->ca_county;RF9 s_state->ca_state +--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF7 s_county->ca_county;RF8 s_state->ca_state ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 ca_address_sk->c_current_addr_sk +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((my_customers.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->ss_customer_sk ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF10 +--------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF9 ------------------------------------PhysicalProject --------------------------------------hashAgg[GLOBAL] ----------------------------------------PhysicalProject @@ -42,9 +42,9 @@ PhysicalResultSink --------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) ----------------------------------------------------PhysicalOlapScan[date_dim] --------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF7 +----------------------------------------------PhysicalOlapScan[customer] --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[customer_address] apply RFs: RF8 RF9 +----------------------------------PhysicalOlapScan[customer_address] apply RFs: RF7 RF8 ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out index 128ee9fcb2bf53..c4b6d05d41fffc 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out @@ -19,38 +19,36 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF8 RF9 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF6 s_county->ca_county;RF7 s_state->ca_state +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 c_current_addr_sk->ca_address_sk ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF5 s_county->ca_county;RF6 s_state->ca_state ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF5 RF6 RF7 ----------------------------------------PhysicalProject -------------------------------------------hashAgg[GLOBAL] ---------------------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------------------hashAgg[LOCAL] +------------------------------------------PhysicalOlapScan[store] +------------------------------------PhysicalProject +--------------------------------------hashAgg[GLOBAL] +----------------------------------------PhysicalProject +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk +--------------------------------------------PhysicalProject +----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +--------------------------------------------PhysicalProject +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk ------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk -----------------------------------------------------PhysicalProject -------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 -----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk +----------------------------------------------------PhysicalUnion +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk -------------------------------------------------------------PhysicalUnion ---------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 ---------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity')) -----------------------------------------------------------------PhysicalOlapScan[item] +----------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) -------------------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] +----------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 +----------------------------------------------------PhysicalProject +------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity')) +--------------------------------------------------------PhysicalOlapScan[item] +------------------------------------------------PhysicalProject +--------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) +----------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= d_month_seq+3) --------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query54.out b/regression-test/data/shape_check/tpcds_sf100/shape/query54.out index 128ee9fcb2bf53..c4b6d05d41fffc 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query54.out @@ -19,38 +19,36 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF8 RF9 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF6 s_county->ca_county;RF7 s_state->ca_state +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 c_current_addr_sk->ca_address_sk ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF5 s_county->ca_county;RF6 s_state->ca_state ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF5 RF6 RF7 ----------------------------------------PhysicalProject -------------------------------------------hashAgg[GLOBAL] ---------------------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------------------hashAgg[LOCAL] +------------------------------------------PhysicalOlapScan[store] +------------------------------------PhysicalProject +--------------------------------------hashAgg[GLOBAL] +----------------------------------------PhysicalProject +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk +--------------------------------------------PhysicalProject +----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +--------------------------------------------PhysicalProject +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk ------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk -----------------------------------------------------PhysicalProject -------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 -----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk +----------------------------------------------------PhysicalUnion +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk -------------------------------------------------------------PhysicalUnion ---------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 ---------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity')) -----------------------------------------------------------------PhysicalOlapScan[item] +----------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) -------------------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] +----------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 +----------------------------------------------------PhysicalProject +------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity')) +--------------------------------------------------------PhysicalOlapScan[item] +------------------------------------------------PhysicalProject +--------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998)) +----------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= d_month_seq+3) --------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out index 1d3031a1a7d6bd..76e2ca411c577b 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out @@ -19,38 +19,36 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF8 RF9 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF6 s_county->ca_county;RF7 s_state->ca_state +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 c_current_addr_sk->ca_address_sk ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF5 s_county->ca_county;RF6 s_state->ca_state ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF5 RF6 RF7 ----------------------------------------PhysicalProject -------------------------------------------hashAgg[GLOBAL] ---------------------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------------------hashAgg[LOCAL] +------------------------------------------PhysicalOlapScan[store] +------------------------------------PhysicalProject +--------------------------------------hashAgg[GLOBAL] +----------------------------------------PhysicalProject +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk +--------------------------------------------PhysicalProject +----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +--------------------------------------------PhysicalProject +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk ------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk -----------------------------------------------------PhysicalProject -------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 -----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk +----------------------------------------------------PhysicalUnion +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk -------------------------------------------------------------PhysicalUnion ---------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 ---------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) -----------------------------------------------------------------PhysicalOlapScan[item] +----------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) -------------------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] +----------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 +----------------------------------------------------PhysicalProject +------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) +--------------------------------------------------------PhysicalOlapScan[item] +------------------------------------------------PhysicalProject +--------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +----------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= d_month_seq+3) --------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out index b458a11db22c72..c9d9076de11bcf 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out @@ -13,49 +13,48 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->ss_sold_date_sk;RF19 d_date_sk->ss_sold_date_sk +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF16 c_customer_sk->ss_customer_sk;RF17 c_customer_sk->ss_customer_sk +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->ss_sold_date_sk;RF17 d_date_sk->ss_sold_date_sk --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF16 RF17 RF18 RF19 ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF12 s_county->ca_county;RF13 s_county->ca_county;RF14 s_state->ca_state;RF15 s_state->ca_state +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF14 c_customer_sk->ss_customer_sk;RF15 c_customer_sk->ss_customer_sk;RF18 c_current_addr_sk->ca_address_sk;RF19 c_current_addr_sk->ca_address_sk ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF10 c_current_addr_sk->ca_address_sk;RF11 c_current_addr_sk->ca_address_sk -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF10 RF11 RF12 RF13 RF14 RF15 -----------------------------------------hashAgg[GLOBAL] -------------------------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------------------------hashAgg[LOCAL] +--------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF14 RF15 RF16 RF17 +------------------------------------hashAgg[GLOBAL] +--------------------------------------PhysicalProject +----------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF12 customer_sk->c_customer_sk;RF13 customer_sk->c_customer_sk +------------------------------------------PhysicalProject +--------------------------------------------PhysicalOlapScan[customer] apply RFs: RF12 RF13 +------------------------------------------PhysicalProject +--------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->cs_sold_date_sk;RF9 d_date_sk->ws_sold_date_sk;RF10 d_date_sk->cs_sold_date_sk;RF11 d_date_sk->ws_sold_date_sk ----------------------------------------------PhysicalProject -------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF8 customer_sk->c_customer_sk;RF9 customer_sk->c_customer_sk ---------------------------------------------------PhysicalProject -----------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF8 RF9 ---------------------------------------------------PhysicalProject -----------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->cs_sold_date_sk;RF5 d_date_sk->ws_sold_date_sk;RF6 d_date_sk->cs_sold_date_sk;RF7 d_date_sk->ws_sold_date_sk +------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->cs_item_sk;RF5 i_item_sk->ws_item_sk;RF6 i_item_sk->cs_item_sk;RF7 i_item_sk->ws_item_sk +--------------------------------------------------PhysicalUnion +----------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] ------------------------------------------------------PhysicalProject ---------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk;RF2 i_item_sk->cs_item_sk;RF3 i_item_sk->ws_item_sk -----------------------------------------------------------PhysicalUnion -------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] ---------------------------------------------------------------PhysicalProject -----------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 RF4 RF6 -------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] ---------------------------------------------------------------PhysicalProject -----------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 RF5 RF7 -----------------------------------------------------------PhysicalProject -------------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) ---------------------------------------------------------------PhysicalOlapScan[item] +--------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF6 RF8 RF10 +----------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] ------------------------------------------------------PhysicalProject --------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) ----------------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject -------------------------------NestedLoopJoin[INNER_JOIN](cast(date_dim.d_month_seq as BIGINT) <= d_month_seq+3) +------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= d_month_seq+3) --------------------------------PhysicalProject -----------------------------------NestedLoopJoin[INNER_JOIN](cast(date_dim.d_month_seq as BIGINT) >= d_month_seq+1) +----------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) >= d_month_seq+1) ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[date_dim] +--------------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) >= d_month_seq+1) +----------------------------------------PhysicalProject +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------------PhysicalAssertNumRows +------------------------------------------PhysicalDistribute[DistributionSpecGather] +--------------------------------------------hashAgg[GLOBAL] +----------------------------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------------------------hashAgg[LOCAL] +--------------------------------------------------PhysicalProject +----------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +------------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalAssertNumRows --------------------------------------PhysicalDistribute[DistributionSpecGather] ----------------------------------------hashAgg[GLOBAL] @@ -64,12 +63,10 @@ PhysicalResultSink ----------------------------------------------PhysicalProject ------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) --------------------------------------------------PhysicalOlapScan[date_dim] ---------------------------------PhysicalAssertNumRows -----------------------------------PhysicalDistribute[DistributionSpecGather] -------------------------------------hashAgg[GLOBAL] ---------------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------------hashAgg[LOCAL] -------------------------------------------PhysicalProject ---------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) -----------------------------------------------PhysicalOlapScan[date_dim] +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF0 s_county->ca_county;RF1 s_county->ca_county;RF2 s_state->ca_state;RF3 s_state->ca_state +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[customer_address] apply RFs: RF0 RF1 RF2 RF3 RF18 RF19 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out index ec4aa0d9c9582a..fd69e1a2db0bae 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out @@ -19,38 +19,36 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF8 RF9 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF6 s_county->ca_county;RF7 s_state->ca_state +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 c_current_addr_sk->ca_address_sk ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF5 s_county->ca_county;RF6 s_state->ca_state ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF5 RF6 RF7 ----------------------------------------PhysicalProject -------------------------------------------hashAgg[GLOBAL] ---------------------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------------------hashAgg[LOCAL] +------------------------------------------PhysicalOlapScan[store] +------------------------------------PhysicalProject +--------------------------------------hashAgg[GLOBAL] +----------------------------------------PhysicalProject +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk +--------------------------------------------PhysicalProject +----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +--------------------------------------------PhysicalProject +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk ------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF4 customer_sk->c_customer_sk -----------------------------------------------------PhysicalProject -------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF4 -----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk;RF3 d_date_sk->ws_sold_date_sk +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk +----------------------------------------------------PhysicalUnion +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->cs_item_sk;RF1 i_item_sk->ws_item_sk -------------------------------------------------------------PhysicalUnion ---------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 ---------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) -----------------------------------------------------------------PhysicalOlapScan[item] +----------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) -------------------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] +----------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 +----------------------------------------------------PhysicalProject +------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) +--------------------------------------------------------PhysicalOlapScan[item] +------------------------------------------------PhysicalProject +--------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +----------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= d_month_seq+3) --------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out index 8c4793187b8bb5..5836333f34dc91 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out @@ -19,38 +19,36 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20 RF21 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF14 s_county->ca_county;RF15 s_county->ca_county;RF16 s_state->ca_state;RF17 s_state->ca_state +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF16 c_current_addr_sk->ca_address_sk;RF17 c_current_addr_sk->ca_address_sk ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF12 c_current_addr_sk->ca_address_sk;RF13 c_current_addr_sk->ca_address_sk +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF12 s_county->ca_county;RF13 s_county->ca_county;RF14 s_state->ca_state;RF15 s_state->ca_state ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF12 RF13 RF14 RF15 RF16 RF17 ----------------------------------------PhysicalProject -------------------------------------------hashAgg[GLOBAL] ---------------------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------------------hashAgg[LOCAL] +------------------------------------------PhysicalOlapScan[store] +------------------------------------PhysicalProject +--------------------------------------hashAgg[GLOBAL] +----------------------------------------PhysicalProject +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF10 customer_sk->c_customer_sk;RF11 customer_sk->c_customer_sk +--------------------------------------------PhysicalProject +----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF10 RF11 +--------------------------------------------PhysicalProject +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->cs_sold_date_sk;RF7 d_date_sk->ws_sold_date_sk;RF8 d_date_sk->cs_sold_date_sk;RF9 d_date_sk->ws_sold_date_sk ------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF10 customer_sk->c_customer_sk;RF11 customer_sk->c_customer_sk -----------------------------------------------------PhysicalProject -------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF10 RF11 -----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->cs_sold_date_sk;RF7 d_date_sk->ws_sold_date_sk;RF8 d_date_sk->cs_sold_date_sk;RF9 d_date_sk->ws_sold_date_sk +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->cs_item_sk;RF3 i_item_sk->ws_item_sk;RF4 i_item_sk->cs_item_sk;RF5 i_item_sk->ws_item_sk +----------------------------------------------------PhysicalUnion +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->cs_item_sk;RF3 i_item_sk->ws_item_sk;RF4 i_item_sk->cs_item_sk;RF5 i_item_sk->ws_item_sk -------------------------------------------------------------PhysicalUnion ---------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF4 RF6 RF8 ---------------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF3 RF5 RF7 RF9 -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) -----------------------------------------------------------------PhysicalOlapScan[item] +----------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF4 RF6 RF8 +------------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) -------------------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] +----------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF3 RF5 RF7 RF9 +----------------------------------------------------PhysicalProject +------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) +--------------------------------------------------------PhysicalOlapScan[item] +------------------------------------------------PhysicalProject +--------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +----------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= d_month_seq+3) build RFs:RF1 d_month_seq+3->cast(d_month_seq as BIGINT) --------------------------------PhysicalProject From 712a0087829a1607bfeb671e79464309d709d8b8 Mon Sep 17 00:00:00 2001 From: englefly Date: Thu, 16 Jul 2026 00:04:04 +0800 Subject: [PATCH 15/23] fmt --- .../apache/doris/nereids/rules/expression/ExpressionRewrite.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java index b6d8e61aa12cf4..02793a0aa341d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java @@ -27,7 +27,6 @@ import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext.ExpressionSource; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; -import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.And; import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.Expression; From 0b0d475157ee51e2f494e8b4ce5d004c423e3983 Mon Sep 17 00:00:00 2001 From: englefly Date: Thu, 16 Jul 2026 10:31:15 +0800 Subject: [PATCH 16/23] 54-dphyper --- .../tpcds_sf1000/dphyper/query54.out | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out index c9d9076de11bcf..7b9e2b1a22dae1 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out @@ -35,16 +35,17 @@ PhysicalResultSink --------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF6 RF8 RF10 ----------------------------------------------------PhysicalDistribute[DistributionSpecExecutionAny] ------------------------------------------------------PhysicalProject ---------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) -----------------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] -----------------------------PhysicalProject -------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= d_month_seq+3) +--------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF5 RF7 RF9 RF11 +--------------------------------------------------PhysicalProject +----------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) +------------------------------------------------------PhysicalOlapScan[item] +----------------------------------------------PhysicalProject +------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +--------------------------------------------------PhysicalOlapScan[date_dim] --------------------------------PhysicalProject -----------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) >= d_month_seq+1) +----------------------------------NestedLoopJoin[INNER_JOIN](cast(date_dim.d_month_seq as BIGINT) <= d_month_seq+3) ------------------------------------PhysicalProject ---------------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) >= d_month_seq+1) +--------------------------------------NestedLoopJoin[INNER_JOIN](cast(date_dim.d_month_seq as BIGINT) >= d_month_seq+1) ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[date_dim] ----------------------------------------PhysicalAssertNumRows From ebba1d69305e0fa6a8d921eabc05df05f37cf16d Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 20 Jul 2026 16:00:21 +0800 Subject: [PATCH 17/23] =?UTF-8?q?=E5=8F=8C=E5=B1=82agg=20exprid=E6=94=B9?= =?UTF-8?q?=E5=86=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../rules/rewrite/EliminateGroupByKey.java | 10 +++++++++- .../rewrite/EliminateGroupByKeyTest.java | 20 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java index e60cdcc83ce967..021b9da98e9147 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java @@ -95,6 +95,14 @@ public Plan visitLogicalProject(LogicalProject proj, Map) exprIdReplacer.rewriteExpr(proj, replaceMap); + if (hasFilter) { + child = exprIdReplacer.rewriteExpr(child, replaceMap); + } + // Compute requireOutput: slots needed by the Project (and Filter, if present) Set requireOutput = new HashSet<>(proj.getInputSlots()); if (hasFilter) { @@ -104,7 +112,7 @@ public Plan visitLogicalProject(LogicalProject proj, Map + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("id"))); + } + @Test void testRepeatEliminateByEqual() { PlanChecker.from(connectContext) From c97bdfc89c02dc58853a6c0b6535c5967c6bef4e Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 20 Jul 2026 17:28:14 +0800 Subject: [PATCH 18/23] replace consumer slotMap --- .../rules/rewrite/EliminateGroupByKey.java | 41 +++++++++++++++++++ .../rewrite/EliminateGroupByKeyTest.java | 40 ++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java index 021b9da98e9147..a4f49c91b25faa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java @@ -30,13 +30,18 @@ import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.Multimap; + import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -131,6 +136,42 @@ public Plan visitLogicalProject(LogicalProject proj, Map replaceMap) { + // When a producer aggregate's output slot is wrapped with any_value(), + // a fresh ExprId is recorded in replaceMap. The CTE consumer's producerToConsumerSlotMap + // still references the old ExprId, so we must rebuild both maps with the new ExprIds. + Map newConsumerToProducer = new LinkedHashMap<>(); + Multimap newProducerToConsumer = LinkedHashMultimap.create(); + for (Slot producerSlot : cteConsumer.getConsumerToProducerOutputMap().values()) { + ExprId newExprId = resolveExprIdChain(producerSlot.getExprId(), replaceMap); + Slot effectiveProducerSlot = newExprId != null + ? (Slot) producerSlot.withExprId(newExprId) + : producerSlot; + for (Slot consumerSlot : cteConsumer.getProducerToConsumerOutputMap().get(producerSlot)) { + newProducerToConsumer.put(effectiveProducerSlot, consumerSlot); + newConsumerToProducer.put(consumerSlot, effectiveProducerSlot); + } + } + return cteConsumer.withTwoMaps(newConsumerToProducer, newProducerToConsumer); + } + + /** Follow transitive ExprId chain to find the final replacement, or null if none. */ + private static ExprId resolveExprIdChain(ExprId exprId, Map replaceMap) { + ExprId newId = replaceMap.get(exprId); + if (newId == null) { + return null; + } + ExprId lastId = newId; + while (true) { + ExprId next = replaceMap.get(lastId); + if (next == null) { + return lastId; + } + lastId = next; + } + } + /** Result of eliminateGroupByKey: the new aggregate and a map of old->new ExprIds. */ private static class EliminateResult { final LogicalAggregate newAgg; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java index 3f4cba3720832a..a470a1d7a77907 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java @@ -19,19 +19,28 @@ import org.apache.doris.nereids.properties.FuncDeps; import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.HashMap; +import java.util.Map; import java.util.Set; class EliminateGroupByKeyTest extends TestWithFeService implements MemoPatternMatchSupported { @@ -269,6 +278,37 @@ void testNestedAggregateUsesRewrittenRequireOutput() { && agg.getGroupByExpressions().get(0).toSql().equals("id"))); } + @Test + void testCteConsumerSlotMapUpdatedByReplaceMap() { + // Verify that visitLogicalCTEConsumer correctly updates the slot maps + // when the replaceMap contains an ExprId replacement from the producer. + Slot oldProducerSlot = new SlotReference("old", IntegerType.INSTANCE, false); + Slot consumerSlot = new SlotReference("cons", IntegerType.INSTANCE, false); + + LogicalCTEConsumer consumer = new LogicalCTEConsumer( + new RelationId(1), new CTEId(0), "cte", + ImmutableMap.of(consumerSlot, oldProducerSlot), + ImmutableMultimap.of(oldProducerSlot, consumerSlot)); + + // Simulate replaceMap with ExprId replacement from aggregate rewrite + ExprId newProducerExprId = new ExprId(999); // fresh Id from any_value alias + Map replaceMap = new HashMap<>(); + replaceMap.put(oldProducerSlot.getExprId(), newProducerExprId); + + EliminateGroupByKey rewriter = new EliminateGroupByKey(); + LogicalCTEConsumer updated = (LogicalCTEConsumer) rewriter.visitLogicalCTEConsumer( + consumer, replaceMap); + + // The updated consumer's producerToConsumerSlotMap should be keyed by the new ExprId + Multimap updatedMap = updated.getProducerToConsumerOutputMap(); + Assertions.assertEquals(1, updatedMap.keySet().size()); + Slot updatedProducerKey = updatedMap.keySet().iterator().next(); + Assertions.assertEquals(newProducerExprId, updatedProducerKey.getExprId(), + "Producer slot ExprId should be updated to the new one from replaceMap"); + Assertions.assertTrue(updatedMap.get(updatedProducerKey).contains(consumerSlot), + "Consumer slot should still be mapped"); + } + @Test void testRepeatEliminateByEqual() { PlanChecker.from(connectContext) From f84a023acb4b141c5d35779285a60cbbc2a2d0bf Mon Sep 17 00:00:00 2001 From: englefly Date: Tue, 21 Jul 2026 08:20:01 +0800 Subject: [PATCH 19/23] fmt --- .../apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java index a4f49c91b25faa..cff93b15b2b58b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java @@ -29,8 +29,8 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; From 8801cc973c2837fa720bbce38d590f2fab0e407e Mon Sep 17 00:00:00 2001 From: englefly Date: Thu, 23 Jul 2026 23:55:47 +0800 Subject: [PATCH 20/23] fix: sync CTEConsumer slot maps after producer rewrite in RewriteCteChildren to prevent ExprId mismatch when beforePushDownJobs rules wrap producer output with new slots --- .../java/org/apache/doris/nereids/AGENTS.md | 4 + .../rules/rewrite/RewriteCteChildren.java | 88 +++++++++++ .../rewrite/SplitMultiDistinctStrategy.java | 2 +- .../RewriteCteChildrenSlotSyncTest.java | 149 ++++++++++++++++++ 4 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildrenSlotSyncTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/AGENTS.md b/fe/fe-core/src/main/java/org/apache/doris/nereids/AGENTS.md index 4bf44f9ff38d89..4f47e7d9a4739e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/AGENTS.md +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/AGENTS.md @@ -13,6 +13,10 @@ - [ ] New rewrite rule placed in correct stage w.r.t. dependencies (esp. `PUSH_DOWN_FILTERS`, `InferPredicates`)? - [ ] Exploration/implementation rules in correct `RuleSet` entry points? +## CTE Producer Rules + +- [ ] Rules that modify `LogicalCTEProducer` output must preserve slot order — ExprIds can be replaced, but the sequence must not be reordered. Non-deterministic iteration (e.g., `HashMap.entrySet()`) over output slots is prohibited; use insertion-order-preserving structures (`LinkedHashMap`, or iterate the original `agg.getOutputExpressions()` order) when building projections atop a producer. Reordering breaks `syncCteConsumerSlotMaps` which relies on position-based alignment between old and new producer outputs. + ## Property Derivation - [ ] New physical operator has `RequestPropertyDeriver` logic? diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java index c957a9e853f68d..5c9be4bc2a5e85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.jobs.rewrite.RewriteJob; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -44,10 +45,15 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; @@ -113,10 +119,92 @@ public Plan visitLogicalCTEAnchor(LogicalCTEAnchor oldProducerOutput = cteAnchor.child(0).getOutput(); Plan producer = cteAnchor.child(0).accept(this, cascadesContext); + outer = syncCteConsumerSlotMaps(oldProducerOutput, producer.getOutput(), + cteAnchor.getCteId(), outer, cascadesContext); return cteAnchor.withChildren(producer, outer); } + /** + * If the producer rewrite changed output ExprIds (e.g. any_value wrapping in + * EliminateGroupByKey), update CTEConsumer slot maps in the consumer tree to match. + * + * @return the consumer tree, updated if any producer ExprIds changed + */ + private LogicalPlan syncCteConsumerSlotMaps(List oldProducerOutput, List newProducerOutput, + CTEId cteId, LogicalPlan outer, CascadesContext cascadesContext) { + if (oldProducerOutput.size() != newProducerOutput.size()) { + return outer; + } + Map exprIdReplaceMap = new HashMap<>(); + for (int i = 0; i < oldProducerOutput.size(); i++) { + ExprId oldId = oldProducerOutput.get(i).getExprId(); + ExprId newId = newProducerOutput.get(i).getExprId(); + if (!oldId.equals(newId)) { + exprIdReplaceMap.put(oldId, newId); + } + } + if (exprIdReplaceMap.isEmpty()) { + return outer; + } + // Collect old→new CTEConsumer mappings by walking the consumer tree. + Map oldToNew = new LinkedHashMap<>(); + outer.foreach(p -> { + if (p instanceof LogicalCTEConsumer) { + LogicalCTEConsumer consumer = (LogicalCTEConsumer) p; + if (consumer.getCteId().equals(cteId)) { + oldToNew.put(consumer, updateCteConsumerSlotMaps(consumer, exprIdReplaceMap)); + } + } + return false; + }); + if (oldToNew.isEmpty()) { + return outer; + } + outer = (LogicalPlan) outer.rewriteUp(p -> { + Plan replacement = oldToNew.get(p); + return replacement != null ? replacement : p; + }); + // Re-collect updated consumers so cteIdToConsumers stays in sync. + Set updatedConsumers = Sets.newHashSet(); + outer.foreach(p -> { + if (p instanceof LogicalCTEConsumer) { + LogicalCTEConsumer c = (LogicalCTEConsumer) p; + if (c.getCteId().equals(cteId)) { + updatedConsumers.add(c); + } + } + return false; + }); + cascadesContext.getCteIdToConsumers().put(cteId, updatedConsumers); + cascadesContext.getStatementContext().getRewrittenCteConsumer().put(cteId, outer); + return outer; + } + + /** + * Rebuild CTEConsumer slot maps so that producer-side slots reference the new ExprIds + * produced by aggregate rewriting (e.g. any_value wrapping in EliminateGroupByKey). + */ + private LogicalCTEConsumer updateCteConsumerSlotMaps( + LogicalCTEConsumer cteConsumer, Map exprIdReplaceMap) { + Map newConsumerToProducer = new LinkedHashMap<>(); + Multimap newProducerToConsumer = LinkedHashMultimap.create(); + for (Slot producerSlot : cteConsumer.getConsumerToProducerOutputMap().values()) { + ExprId newExprId = exprIdReplaceMap.get(producerSlot.getExprId()); + Slot effectiveProducerSlot = newExprId != null + ? (Slot) producerSlot.withExprId(newExprId) + : producerSlot; + for (Slot consumerSlot : cteConsumer.getProducerToConsumerOutputMap().get(producerSlot)) { + newProducerToConsumer.put(effectiveProducerSlot, consumerSlot); + newConsumerToProducer.put(consumerSlot, effectiveProducerSlot); + } + } + return (LogicalCTEConsumer) cteConsumer.withTwoMaps(newConsumerToProducer, newProducerToConsumer); + } + @Override public Plan visitLogicalCTEProducer(LogicalCTEProducer cteProducer, CascadesContext cascadesContext) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java index c781ce1aa1b10d..d1485fd036e0f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java @@ -76,7 +76,7 @@ public static Plan rewrite(LogicalAggregate agg, DistinctSelecto // construct cte consumer and aggregate List> newAggs = new ArrayList<>(); // All otherAggFuncs are placed in the first one - Map newToOriginDistinctFuncAlias = new HashMap<>(); + Map newToOriginDistinctFuncAlias = new LinkedHashMap<>(); List outputJoinGroupBys = new ArrayList<>(); for (int i = 0; i < distinctFuncWithAliasReplaced.size(); ++i) { List aliases = distinctFuncWithAliasReplaced.get(i); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildrenSlotSyncTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildrenSlotSyncTest.java new file mode 100644 index 00000000000000..cfee2cbfc06de1 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildrenSlotSyncTest.java @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.PlanConstructor; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +/** + * Tests that CTEConsumer slot maps are synced after producer rewrite + * changes output ExprIds (e.g. EliminateGroupByKey any_value wrapping). + */ +class RewriteCteChildrenSlotSyncTest { + + @Test + void testConsumerSlotMapSyncedWhenProducerOutputExprIdChanged() { + // ---- Setup: two scans with different ExprIds in their output ---- + LogicalOlapScan oldScan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + LogicalOlapScan newScan = PlanConstructor.newLogicalOlapScan(1, "t1", 0); + CTEId cteId = new CTEId(1); + + // Original producer wraps oldScan (old output ExprIds) + LogicalCTEProducer originalProducer = new LogicalCTEProducer<>(cteId, oldScan); + + // Consumer slot maps reference original producer output (old ExprIds) + LogicalCTEConsumer consumer = new LogicalCTEConsumer( + PlanConstructor.getNextRelationId(), cteId, "cte1", originalProducer); + Map originalConsumerToProducer = consumer.getConsumerToProducerOutputMap(); + + // CTEAnchor: original producer (left) + consumer side (right) + LogicalCTEAnchor, LogicalCTEConsumer> cteAnchor = + new LogicalCTEAnchor<>(cteId, originalProducer, consumer); + + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext( + new ConnectContext(), cteAnchor); + + // Simulate that the producer was already rewritten (cached) with newScan, + // whose output has different ExprIds than oldScan. + cascadesContext.getStatementContext().getRewrittenCteProducer().put(cteId, newScan); + + // Consumer cache: the consumer still has stale slot maps (pointing to old ExprIds) + cascadesContext.getStatementContext().getRewrittenCteConsumer().put(cteId, consumer); + cascadesContext.getCteIdToConsumers().put(cteId, ImmutableSet.of(consumer)); + + // ---- Execute ---- + RewriteCteChildren rewriter = new RewriteCteChildren(ImmutableList.of(), false); + Plan result = rewriter.visitLogicalCTEAnchor(cteAnchor, cascadesContext); + + // ---- Verify ---- + Assertions.assertInstanceOf(LogicalCTEAnchor.class, result); + LogicalCTEAnchor resultAnchor = (LogicalCTEAnchor) result; + + // Producer side should have the new output ExprIds (from newScan) + List producerOutput = resultAnchor.child(0).getOutput(); + Assertions.assertEquals(newScan.getOutput().size(), producerOutput.size()); + for (int i = 0; i < producerOutput.size(); i++) { + Assertions.assertEquals(newScan.getOutput().get(i).getExprId(), + producerOutput.get(i).getExprId(), + "Producer output ExprId at position " + i + " should match newScan"); + } + + // Consumer slot maps should now reference the new producer ExprIds + LogicalPlan consumerSide = (LogicalPlan) resultAnchor.child(1); + Assertions.assertInstanceOf(LogicalCTEConsumer.class, consumerSide); + LogicalCTEConsumer resultConsumer = (LogicalCTEConsumer) consumerSide; + + Map updatedConsumerToProducer = resultConsumer.getConsumerToProducerOutputMap(); + Assertions.assertEquals(originalConsumerToProducer.size(), updatedConsumerToProducer.size()); + + for (Map.Entry entry : updatedConsumerToProducer.entrySet()) { + Slot consumerSlot = entry.getKey(); + Slot producerSlot = entry.getValue(); + ExprId producerExprId = producerSlot.getExprId(); + + // Each consumer slot's producer reference must exist in the new producer output + boolean found = producerOutput.stream() + .anyMatch(s -> s.getExprId().equals(producerExprId)); + Assertions.assertTrue(found, + "Consumer slot " + consumerSlot + " references producer ExprId " + + producerExprId + " which is not in producer output"); + } + } + + @Test + void testConsumerSlotMapUnchangedWhenProducerOutputExprIdNotChanged() { + // Setup: same scan used for both old and new producer output (no ExprId change) + LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + CTEId cteId = new CTEId(2); + + LogicalCTEProducer originalProducer = new LogicalCTEProducer<>(cteId, scan); + LogicalCTEConsumer consumer = new LogicalCTEConsumer( + PlanConstructor.getNextRelationId(), cteId, "cte2", originalProducer); + LogicalCTEAnchor, LogicalCTEConsumer> cteAnchor = + new LogicalCTEAnchor<>(cteId, originalProducer, consumer); + + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext( + new ConnectContext(), cteAnchor); + + // Cache the SAME scan as "rewritten" producer (no ExprId change) + cascadesContext.getStatementContext().getRewrittenCteProducer().put(cteId, scan); + cascadesContext.getStatementContext().getRewrittenCteConsumer().put(cteId, consumer); + cascadesContext.getCteIdToConsumers().put(cteId, ImmutableSet.of(consumer)); + + RewriteCteChildren rewriter = new RewriteCteChildren(ImmutableList.of(), false); + Plan result = rewriter.visitLogicalCTEAnchor(cteAnchor, cascadesContext); + + // Consumer slot maps should be unchanged (no ExprId change in producer output) + LogicalCTEAnchor resultAnchor = (LogicalCTEAnchor) result; + LogicalCTEConsumer resultConsumer = (LogicalCTEConsumer) resultAnchor.child(1); + + // The consumer instance should be the SAME (no new instance created since no change) + Assertions.assertSame(consumer, resultConsumer, + "Consumer should be unchanged when producer output ExprIds don't change"); + } + +} From 865ea88d70ff2e7165edc087e07850771c5612f3 Mon Sep 17 00:00:00 2001 From: minghong Date: Fri, 24 Jul 2026 12:37:12 +0800 Subject: [PATCH 21/23] (fix) when producer slot prune and rewrite in one pass --- .../rules/rewrite/RewriteCteChildren.java | 10 +++ .../RewriteCteChildrenSlotSyncTest.java | 85 +++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java index 5c9be4bc2a5e85..98b05511ae8756 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java @@ -123,6 +123,16 @@ public Plan visitLogicalCTEAnchor(LogicalCTEAnchor oldProducerOutput = cteAnchor.child(0).getOutput(); Plan producer = cteAnchor.child(0).accept(this, cascadesContext); + // visitLogicalCTEProducer may insert a pruning Project that drops producer outputs + // not needed by any consumer, changing output arity. Align the old output with the + // same prune set so that ExprId changes of surviving slots are still propagated. + Set neededProducerOutputs = cascadesContext.getStatementContext() + .getCteIdToOutputIds().get(cteAnchor.getCteId()); + if (neededProducerOutputs != null && neededProducerOutputs.size() < oldProducerOutput.size()) { + oldProducerOutput = oldProducerOutput.stream() + .filter(neededProducerOutputs::contains) + .collect(Collectors.toList()); + } outer = syncCteConsumerSlotMaps(oldProducerOutput, producer.getOutput(), cteAnchor.getCteId(), outer, cascadesContext); return cteAnchor.withChildren(producer, outer); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildrenSlotSyncTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildrenSlotSyncTest.java index cfee2cbfc06de1..7c59d5fb43537f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildrenSlotSyncTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildrenSlotSyncTest.java @@ -18,15 +18,21 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.exploration.join.JoinReorderContext; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanConstructor; import org.apache.doris.qe.ConnectContext; @@ -36,6 +42,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -146,4 +153,82 @@ void testConsumerSlotMapUnchangedWhenProducerOutputExprIdNotChanged() { "Consumer should be unchanged when producer output ExprIds don't change"); } + @Test + void testConsumerSlotMapSyncedWhenProducerOutputPrunedAndExprIdChanged() { + // Producer originally outputs [id, name, x]. Consumers only need [name, x], so + // visitLogicalCTEProducer prunes id (arity 3 -> 2), while EliminateGroupByKey wraps + // name with any_value() and assigns it a fresh ExprId. The slot-map sync must + // propagate name's new ExprId even though the producer output arity changed. + LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + Slot idSlot = scan.getOutput().get(0); + Slot nameSlot = scan.getOutput().get(1); + Alias xAlias = new Alias(new ExprId(100), idSlot, "x"); + LogicalProject oldProducerChild = new LogicalProject<>( + ImmutableList.of(idSlot, nameSlot, xAlias), scan); + CTEId cteId = new CTEId(3); + LogicalCTEProducer> originalProducer = + new LogicalCTEProducer<>(cteId, oldProducerChild); + + // Two retained consumers referencing the original 3-slot producer output + LogicalCTEConsumer consumer1 = new LogicalCTEConsumer( + PlanConstructor.getNextRelationId(), cteId, "cte3", originalProducer); + LogicalCTEConsumer consumer2 = new LogicalCTEConsumer( + PlanConstructor.getNextRelationId(), cteId, "cte3", originalProducer); + LogicalPlan consumerSide = new LogicalJoin<>(JoinType.CROSS_JOIN, + consumer1, consumer2, new JoinReorderContext()); + + LogicalCTEAnchor>, LogicalPlan> cteAnchor = + new LogicalCTEAnchor<>(cteId, originalProducer, consumerSide); + + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext( + new ConnectContext(), cteAnchor); + + // Simulate the rewritten producer: id pruned, name wrapped by any_value (fresh ExprId) + Slot newNameSlot = (Slot) nameSlot.withExprId(new ExprId(200)); + LogicalProject newProducerChild = new LogicalProject<>( + ImmutableList.of(newNameSlot, xAlias.toSlot()), scan); + cascadesContext.getStatementContext().getRewrittenCteProducer().put(cteId, newProducerChild); + cascadesContext.getStatementContext().getRewrittenCteConsumer().put(cteId, consumerSide); + cascadesContext.getCteIdToConsumers().put(cteId, ImmutableSet.of(consumer1, consumer2)); + // Consumers only need [name, x]: id is pruned from the producer output + cascadesContext.getStatementContext().getCteIdToOutputIds().put(cteId, + ImmutableSet.of(nameSlot, xAlias.toSlot())); + + // ---- Execute ---- + RewriteCteChildren rewriter = new RewriteCteChildren(ImmutableList.of(), false); + Plan result = rewriter.visitLogicalCTEAnchor(cteAnchor, cascadesContext); + + // ---- Verify ---- + Assertions.assertInstanceOf(LogicalCTEAnchor.class, result); + LogicalCTEAnchor resultAnchor = (LogicalCTEAnchor) result; + + // Producer side has the pruned output [name(new ExprId), x] + List producerOutput = resultAnchor.child(0).getOutput(); + Assertions.assertEquals(2, producerOutput.size()); + Assertions.assertEquals(new ExprId(200), producerOutput.get(0).getExprId()); + Assertions.assertEquals(new ExprId(100), producerOutput.get(1).getExprId()); + + // Both consumers must reference the new producer ExprId for name + List resultConsumers = new ArrayList<>(); + resultAnchor.child(1).foreach(p -> { + if (p instanceof LogicalCTEConsumer) { + resultConsumers.add((LogicalCTEConsumer) p); + } + return false; + }); + Assertions.assertEquals(2, resultConsumers.size()); + for (LogicalCTEConsumer resultConsumer : resultConsumers) { + Map consumerToProducer = resultConsumer.getConsumerToProducerOutputMap(); + Assertions.assertTrue(consumerToProducer.values().stream() + .anyMatch(s -> s.getExprId().equals(new ExprId(200))), + "Consumer should reference the new producer ExprId for name"); + Assertions.assertFalse(consumerToProducer.values().stream() + .anyMatch(s -> s.getExprId().equals(nameSlot.getExprId())), + "Consumer should no longer reference the old producer ExprId for name"); + Assertions.assertTrue(consumerToProducer.values().stream() + .anyMatch(s -> s.getExprId().equals(new ExprId(100))), + "Consumer should still reference the unchanged producer ExprId for x"); + } + } + } From 89472484a99c72b5d782454b57cb1c02868a2a21 Mon Sep 17 00:00:00 2001 From: minghong Date: Wed, 29 Jul 2026 17:21:15 +0800 Subject: [PATCH 22/23] make ELIMINATE_GROUP_BY_KEY whole-tree rewrite --- .../java/org/apache/doris/nereids/AGENTS.md | 4 - .../doris/nereids/jobs/executor/Rewriter.java | 10 +- .../rules/rewrite/RewriteCteChildren.java | 98 -------- .../RewriteCteChildrenSlotSyncTest.java | 234 ------------------ 4 files changed, 5 insertions(+), 341 deletions(-) delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildrenSlotSyncTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/AGENTS.md b/fe/fe-core/src/main/java/org/apache/doris/nereids/AGENTS.md index 4f47e7d9a4739e..4bf44f9ff38d89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/AGENTS.md +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/AGENTS.md @@ -13,10 +13,6 @@ - [ ] New rewrite rule placed in correct stage w.r.t. dependencies (esp. `PUSH_DOWN_FILTERS`, `InferPredicates`)? - [ ] Exploration/implementation rules in correct `RuleSet` entry points? -## CTE Producer Rules - -- [ ] Rules that modify `LogicalCTEProducer` output must preserve slot order — ExprIds can be replaced, but the sequence must not be reordered. Non-deterministic iteration (e.g., `HashMap.entrySet()`) over output slots is prohibited; use insertion-order-preserving structures (`LinkedHashMap`, or iterate the original `agg.getOutputExpressions()` order) when building projections atop a producer. Reordering breaks `syncCteConsumerSlotMaps` which relies on position-based alignment between old and new producer outputs. - ## Property Derivation - [ ] New physical operator has `RequestPropertyDeriver` logic? diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index cda5d9067977b4..162289c33553c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -678,12 +678,7 @@ public class Rewriter extends AbstractBatchJobExecutor { cascadesContext -> cascadesContext.rewritePlanContainsTypes(LogicalAggregate.class) || cascadesContext.rewritePlanContainsTypes(LogicalJoin.class) || cascadesContext.rewritePlanContainsTypes(LogicalUnion.class), - // PushDownAggThroughJoinOnPkFk must run before EliminateGroupByKey, - // because EliminateGroupByKey wraps FD-redundant group-by keys with - // ANY_VALUE and rewrites ExprIds, which PushDownAggThroughJoinOnPkFk - // cannot fully handle (especially for non-PK/FK primary table columns). topDown(new PushDownAggThroughJoinOnPkFk()), - custom(RuleType.ELIMINATE_GROUP_BY_KEY, EliminateGroupByKey::new), topDown(new PullUpJoinFromUnionAll()) ), topic("init join", bottomUp(ImmutableList.of(new InitJoinOrder()))), @@ -920,6 +915,11 @@ private static List getWholeTreeRewriteJobs( ))); rewriteJobs.addAll(jobs(topic("convert outer join to anti", custom(RuleType.CONVERT_OUTER_JOIN_TO_ANTI, ConvertOuterJoinToAntiJoin::new)))); + rewriteJobs.addAll(jobs(topic("eliminate Aggregate according to fd items", + cascadesContext -> cascadesContext.rewritePlanContainsTypes(LogicalAggregate.class) + || cascadesContext.rewritePlanContainsTypes(LogicalJoin.class) + || cascadesContext.rewritePlanContainsTypes(LogicalUnion.class), + custom(RuleType.ELIMINATE_GROUP_BY_KEY, EliminateGroupByKey::new)))); rewriteJobs.addAll(jobs(topic("eliminate group by key by uniform", custom(RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, EliminateGroupByKeyByUniform::new)))); if (needOrExpansion) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java index 98b05511ae8756..c957a9e853f68d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.jobs.rewrite.RewriteJob; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; -import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -45,15 +44,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.LinkedHashMultimap; -import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; @@ -119,102 +113,10 @@ public Plan visitLogicalCTEAnchor(LogicalCTEAnchor oldProducerOutput = cteAnchor.child(0).getOutput(); Plan producer = cteAnchor.child(0).accept(this, cascadesContext); - // visitLogicalCTEProducer may insert a pruning Project that drops producer outputs - // not needed by any consumer, changing output arity. Align the old output with the - // same prune set so that ExprId changes of surviving slots are still propagated. - Set neededProducerOutputs = cascadesContext.getStatementContext() - .getCteIdToOutputIds().get(cteAnchor.getCteId()); - if (neededProducerOutputs != null && neededProducerOutputs.size() < oldProducerOutput.size()) { - oldProducerOutput = oldProducerOutput.stream() - .filter(neededProducerOutputs::contains) - .collect(Collectors.toList()); - } - outer = syncCteConsumerSlotMaps(oldProducerOutput, producer.getOutput(), - cteAnchor.getCteId(), outer, cascadesContext); return cteAnchor.withChildren(producer, outer); } - /** - * If the producer rewrite changed output ExprIds (e.g. any_value wrapping in - * EliminateGroupByKey), update CTEConsumer slot maps in the consumer tree to match. - * - * @return the consumer tree, updated if any producer ExprIds changed - */ - private LogicalPlan syncCteConsumerSlotMaps(List oldProducerOutput, List newProducerOutput, - CTEId cteId, LogicalPlan outer, CascadesContext cascadesContext) { - if (oldProducerOutput.size() != newProducerOutput.size()) { - return outer; - } - Map exprIdReplaceMap = new HashMap<>(); - for (int i = 0; i < oldProducerOutput.size(); i++) { - ExprId oldId = oldProducerOutput.get(i).getExprId(); - ExprId newId = newProducerOutput.get(i).getExprId(); - if (!oldId.equals(newId)) { - exprIdReplaceMap.put(oldId, newId); - } - } - if (exprIdReplaceMap.isEmpty()) { - return outer; - } - // Collect old→new CTEConsumer mappings by walking the consumer tree. - Map oldToNew = new LinkedHashMap<>(); - outer.foreach(p -> { - if (p instanceof LogicalCTEConsumer) { - LogicalCTEConsumer consumer = (LogicalCTEConsumer) p; - if (consumer.getCteId().equals(cteId)) { - oldToNew.put(consumer, updateCteConsumerSlotMaps(consumer, exprIdReplaceMap)); - } - } - return false; - }); - if (oldToNew.isEmpty()) { - return outer; - } - outer = (LogicalPlan) outer.rewriteUp(p -> { - Plan replacement = oldToNew.get(p); - return replacement != null ? replacement : p; - }); - // Re-collect updated consumers so cteIdToConsumers stays in sync. - Set updatedConsumers = Sets.newHashSet(); - outer.foreach(p -> { - if (p instanceof LogicalCTEConsumer) { - LogicalCTEConsumer c = (LogicalCTEConsumer) p; - if (c.getCteId().equals(cteId)) { - updatedConsumers.add(c); - } - } - return false; - }); - cascadesContext.getCteIdToConsumers().put(cteId, updatedConsumers); - cascadesContext.getStatementContext().getRewrittenCteConsumer().put(cteId, outer); - return outer; - } - - /** - * Rebuild CTEConsumer slot maps so that producer-side slots reference the new ExprIds - * produced by aggregate rewriting (e.g. any_value wrapping in EliminateGroupByKey). - */ - private LogicalCTEConsumer updateCteConsumerSlotMaps( - LogicalCTEConsumer cteConsumer, Map exprIdReplaceMap) { - Map newConsumerToProducer = new LinkedHashMap<>(); - Multimap newProducerToConsumer = LinkedHashMultimap.create(); - for (Slot producerSlot : cteConsumer.getConsumerToProducerOutputMap().values()) { - ExprId newExprId = exprIdReplaceMap.get(producerSlot.getExprId()); - Slot effectiveProducerSlot = newExprId != null - ? (Slot) producerSlot.withExprId(newExprId) - : producerSlot; - for (Slot consumerSlot : cteConsumer.getProducerToConsumerOutputMap().get(producerSlot)) { - newProducerToConsumer.put(effectiveProducerSlot, consumerSlot); - newConsumerToProducer.put(consumerSlot, effectiveProducerSlot); - } - } - return (LogicalCTEConsumer) cteConsumer.withTwoMaps(newConsumerToProducer, newProducerToConsumer); - } - @Override public Plan visitLogicalCTEProducer(LogicalCTEProducer cteProducer, CascadesContext cascadesContext) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildrenSlotSyncTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildrenSlotSyncTest.java deleted file mode 100644 index 7c59d5fb43537f..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildrenSlotSyncTest.java +++ /dev/null @@ -1,234 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.exploration.join.JoinReorderContext; -import org.apache.doris.nereids.trees.expressions.Alias; -import org.apache.doris.nereids.trees.expressions.CTEId; -import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.JoinType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; -import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Tests that CTEConsumer slot maps are synced after producer rewrite - * changes output ExprIds (e.g. EliminateGroupByKey any_value wrapping). - */ -class RewriteCteChildrenSlotSyncTest { - - @Test - void testConsumerSlotMapSyncedWhenProducerOutputExprIdChanged() { - // ---- Setup: two scans with different ExprIds in their output ---- - LogicalOlapScan oldScan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); - LogicalOlapScan newScan = PlanConstructor.newLogicalOlapScan(1, "t1", 0); - CTEId cteId = new CTEId(1); - - // Original producer wraps oldScan (old output ExprIds) - LogicalCTEProducer originalProducer = new LogicalCTEProducer<>(cteId, oldScan); - - // Consumer slot maps reference original producer output (old ExprIds) - LogicalCTEConsumer consumer = new LogicalCTEConsumer( - PlanConstructor.getNextRelationId(), cteId, "cte1", originalProducer); - Map originalConsumerToProducer = consumer.getConsumerToProducerOutputMap(); - - // CTEAnchor: original producer (left) + consumer side (right) - LogicalCTEAnchor, LogicalCTEConsumer> cteAnchor = - new LogicalCTEAnchor<>(cteId, originalProducer, consumer); - - CascadesContext cascadesContext = MemoTestUtils.createCascadesContext( - new ConnectContext(), cteAnchor); - - // Simulate that the producer was already rewritten (cached) with newScan, - // whose output has different ExprIds than oldScan. - cascadesContext.getStatementContext().getRewrittenCteProducer().put(cteId, newScan); - - // Consumer cache: the consumer still has stale slot maps (pointing to old ExprIds) - cascadesContext.getStatementContext().getRewrittenCteConsumer().put(cteId, consumer); - cascadesContext.getCteIdToConsumers().put(cteId, ImmutableSet.of(consumer)); - - // ---- Execute ---- - RewriteCteChildren rewriter = new RewriteCteChildren(ImmutableList.of(), false); - Plan result = rewriter.visitLogicalCTEAnchor(cteAnchor, cascadesContext); - - // ---- Verify ---- - Assertions.assertInstanceOf(LogicalCTEAnchor.class, result); - LogicalCTEAnchor resultAnchor = (LogicalCTEAnchor) result; - - // Producer side should have the new output ExprIds (from newScan) - List producerOutput = resultAnchor.child(0).getOutput(); - Assertions.assertEquals(newScan.getOutput().size(), producerOutput.size()); - for (int i = 0; i < producerOutput.size(); i++) { - Assertions.assertEquals(newScan.getOutput().get(i).getExprId(), - producerOutput.get(i).getExprId(), - "Producer output ExprId at position " + i + " should match newScan"); - } - - // Consumer slot maps should now reference the new producer ExprIds - LogicalPlan consumerSide = (LogicalPlan) resultAnchor.child(1); - Assertions.assertInstanceOf(LogicalCTEConsumer.class, consumerSide); - LogicalCTEConsumer resultConsumer = (LogicalCTEConsumer) consumerSide; - - Map updatedConsumerToProducer = resultConsumer.getConsumerToProducerOutputMap(); - Assertions.assertEquals(originalConsumerToProducer.size(), updatedConsumerToProducer.size()); - - for (Map.Entry entry : updatedConsumerToProducer.entrySet()) { - Slot consumerSlot = entry.getKey(); - Slot producerSlot = entry.getValue(); - ExprId producerExprId = producerSlot.getExprId(); - - // Each consumer slot's producer reference must exist in the new producer output - boolean found = producerOutput.stream() - .anyMatch(s -> s.getExprId().equals(producerExprId)); - Assertions.assertTrue(found, - "Consumer slot " + consumerSlot + " references producer ExprId " - + producerExprId + " which is not in producer output"); - } - } - - @Test - void testConsumerSlotMapUnchangedWhenProducerOutputExprIdNotChanged() { - // Setup: same scan used for both old and new producer output (no ExprId change) - LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); - CTEId cteId = new CTEId(2); - - LogicalCTEProducer originalProducer = new LogicalCTEProducer<>(cteId, scan); - LogicalCTEConsumer consumer = new LogicalCTEConsumer( - PlanConstructor.getNextRelationId(), cteId, "cte2", originalProducer); - LogicalCTEAnchor, LogicalCTEConsumer> cteAnchor = - new LogicalCTEAnchor<>(cteId, originalProducer, consumer); - - CascadesContext cascadesContext = MemoTestUtils.createCascadesContext( - new ConnectContext(), cteAnchor); - - // Cache the SAME scan as "rewritten" producer (no ExprId change) - cascadesContext.getStatementContext().getRewrittenCteProducer().put(cteId, scan); - cascadesContext.getStatementContext().getRewrittenCteConsumer().put(cteId, consumer); - cascadesContext.getCteIdToConsumers().put(cteId, ImmutableSet.of(consumer)); - - RewriteCteChildren rewriter = new RewriteCteChildren(ImmutableList.of(), false); - Plan result = rewriter.visitLogicalCTEAnchor(cteAnchor, cascadesContext); - - // Consumer slot maps should be unchanged (no ExprId change in producer output) - LogicalCTEAnchor resultAnchor = (LogicalCTEAnchor) result; - LogicalCTEConsumer resultConsumer = (LogicalCTEConsumer) resultAnchor.child(1); - - // The consumer instance should be the SAME (no new instance created since no change) - Assertions.assertSame(consumer, resultConsumer, - "Consumer should be unchanged when producer output ExprIds don't change"); - } - - @Test - void testConsumerSlotMapSyncedWhenProducerOutputPrunedAndExprIdChanged() { - // Producer originally outputs [id, name, x]. Consumers only need [name, x], so - // visitLogicalCTEProducer prunes id (arity 3 -> 2), while EliminateGroupByKey wraps - // name with any_value() and assigns it a fresh ExprId. The slot-map sync must - // propagate name's new ExprId even though the producer output arity changed. - LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); - Slot idSlot = scan.getOutput().get(0); - Slot nameSlot = scan.getOutput().get(1); - Alias xAlias = new Alias(new ExprId(100), idSlot, "x"); - LogicalProject oldProducerChild = new LogicalProject<>( - ImmutableList.of(idSlot, nameSlot, xAlias), scan); - CTEId cteId = new CTEId(3); - LogicalCTEProducer> originalProducer = - new LogicalCTEProducer<>(cteId, oldProducerChild); - - // Two retained consumers referencing the original 3-slot producer output - LogicalCTEConsumer consumer1 = new LogicalCTEConsumer( - PlanConstructor.getNextRelationId(), cteId, "cte3", originalProducer); - LogicalCTEConsumer consumer2 = new LogicalCTEConsumer( - PlanConstructor.getNextRelationId(), cteId, "cte3", originalProducer); - LogicalPlan consumerSide = new LogicalJoin<>(JoinType.CROSS_JOIN, - consumer1, consumer2, new JoinReorderContext()); - - LogicalCTEAnchor>, LogicalPlan> cteAnchor = - new LogicalCTEAnchor<>(cteId, originalProducer, consumerSide); - - CascadesContext cascadesContext = MemoTestUtils.createCascadesContext( - new ConnectContext(), cteAnchor); - - // Simulate the rewritten producer: id pruned, name wrapped by any_value (fresh ExprId) - Slot newNameSlot = (Slot) nameSlot.withExprId(new ExprId(200)); - LogicalProject newProducerChild = new LogicalProject<>( - ImmutableList.of(newNameSlot, xAlias.toSlot()), scan); - cascadesContext.getStatementContext().getRewrittenCteProducer().put(cteId, newProducerChild); - cascadesContext.getStatementContext().getRewrittenCteConsumer().put(cteId, consumerSide); - cascadesContext.getCteIdToConsumers().put(cteId, ImmutableSet.of(consumer1, consumer2)); - // Consumers only need [name, x]: id is pruned from the producer output - cascadesContext.getStatementContext().getCteIdToOutputIds().put(cteId, - ImmutableSet.of(nameSlot, xAlias.toSlot())); - - // ---- Execute ---- - RewriteCteChildren rewriter = new RewriteCteChildren(ImmutableList.of(), false); - Plan result = rewriter.visitLogicalCTEAnchor(cteAnchor, cascadesContext); - - // ---- Verify ---- - Assertions.assertInstanceOf(LogicalCTEAnchor.class, result); - LogicalCTEAnchor resultAnchor = (LogicalCTEAnchor) result; - - // Producer side has the pruned output [name(new ExprId), x] - List producerOutput = resultAnchor.child(0).getOutput(); - Assertions.assertEquals(2, producerOutput.size()); - Assertions.assertEquals(new ExprId(200), producerOutput.get(0).getExprId()); - Assertions.assertEquals(new ExprId(100), producerOutput.get(1).getExprId()); - - // Both consumers must reference the new producer ExprId for name - List resultConsumers = new ArrayList<>(); - resultAnchor.child(1).foreach(p -> { - if (p instanceof LogicalCTEConsumer) { - resultConsumers.add((LogicalCTEConsumer) p); - } - return false; - }); - Assertions.assertEquals(2, resultConsumers.size()); - for (LogicalCTEConsumer resultConsumer : resultConsumers) { - Map consumerToProducer = resultConsumer.getConsumerToProducerOutputMap(); - Assertions.assertTrue(consumerToProducer.values().stream() - .anyMatch(s -> s.getExprId().equals(new ExprId(200))), - "Consumer should reference the new producer ExprId for name"); - Assertions.assertFalse(consumerToProducer.values().stream() - .anyMatch(s -> s.getExprId().equals(nameSlot.getExprId())), - "Consumer should no longer reference the old producer ExprId for name"); - Assertions.assertTrue(consumerToProducer.values().stream() - .anyMatch(s -> s.getExprId().equals(new ExprId(100))), - "Consumer should still reference the unchanged producer ExprId for x"); - } - } - -} From 2fa68187f7691612c05cdd7d50dc3d0ad28e824a Mon Sep 17 00:00:00 2001 From: minghong Date: Fri, 7 Aug 2026 09:40:47 +0800 Subject: [PATCH 23/23] [fix](agg push down) revert ANY_VALUE handler in PushDownAggThroughJoinOnPkFk The ANY_VALUE rewrite (any_value(pk) -> any_value(fk)) used the primaryToForeignDeps map built from findBijectionSlots(), which only guarantees a bijective functional dependency, not value equality. For a bijection like unique_code <-> fk derived through a unique-key FD plus the pk = fk equality, any_value(unique_code) was rewritten to any_value(fk), silently changing results (e.g. 100/200 -> 1/2). Revert to the master behavior: aggs whose output contains any_value(primary column) are rejected by the final guard and the pk/fk pushdown is skipped (conservative and correct). The bijection map stays for GROUP BY replacement only; a follow-up can restore the any_value pushdown using DataTrait.calEqualSet for value-level swaps. --- .../rewrite/PushDownAggThroughJoinOnPkFk.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFk.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFk.java index 993d25d7385617..160578a5dcfb6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFk.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFk.java @@ -25,7 +25,6 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Project; @@ -217,9 +216,6 @@ private LogicalAggregate eliminatePrimaryOutput(LogicalAggregate agg, Plan // 2. Count: the count is from primary plan, // we need to replace the slot in the count with the corresponding slot // from foreign plan - // 3. AnyValue: EliminateGroupByKey may wrap an FD-redundant group-by key - // with any_value(), keep it in the output and replace inner slot - // with the corresponding foreign plan slot if (expression instanceof Slot && primaryPlan.getOutput().contains(expression)) { if (primaryToForeignDeps.containsKey(expression)) { expression = primaryToForeignDeps.getOrDefault(expression, expression.toSlot()); @@ -240,18 +236,6 @@ private LogicalAggregate eliminatePrimaryOutput(LogicalAggregate agg, Plan : e); } } - if (expression instanceof Alias - && expression.child(0) instanceof AnyValue - && expression.child(0).child(0) instanceof Slot) { - // any_value(pk) can be rewritten to any_value(fk) - Slot slot = (Slot) expression.child(0).child(0); - if (primaryToForeignDeps.containsKey(slot)) { - expression = (NamedExpression) expression.rewriteUp(e -> - e instanceof Slot - ? primaryToForeignDeps.getOrDefault((Slot) e, (Slot) e) - : e); - } - } if (!(expression instanceof Slot) && expression.getInputSlots().stream().anyMatch(primaryOutput::contains)) { return null;