diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 520ef17fb0bb18..8ed66818fd42bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -274,8 +274,9 @@ public enum TableFrom { // Record used table and it's used partitions private final Multimap, Pair>> tableUsedPartitionNameMap = HashMultimap.create(); - // Record query common table id to relation id mapping, this is used for mv rewrite - private final Multimap commonTableIdToRelationIdToMap = HashMultimap.create(); + // Record statement-scope table ids to relation ids for MV rewrite. + // One table id may map to multiple relation ids because of aliases and nested MV scan alternatives. + private final Multimap tableIdToRelationIds = HashMultimap.create(); // Record mtmv and valid partitions map because this is time-consuming behavior private final Map> mvCanRewritePartitionsMap = new HashMap<>(); @@ -304,8 +305,6 @@ public enum TableFrom { // mark is rewritten in RBO phase, if rewritten in RBO phase should set true private boolean preMvRewritten = false; - private final Set> materializationRewrittenSuccessSet = new HashSet<>(); - private boolean isInsert = false; private Optional>> mvRefreshPredicates = Optional.empty(); @@ -1109,20 +1108,12 @@ public void setPreMvRewritten(boolean preMvRewritten) { this.preMvRewritten = preMvRewritten; } - public Set> getMaterializationRewrittenSuccessSet() { - return materializationRewrittenSuccessSet; - } - - public void addMaterializationRewrittenSuccess(List materializationQualifier) { - this.materializationRewrittenSuccessSet.add(materializationQualifier); - } - public Multimap, Pair>> getTableUsedPartitionNameMap() { return tableUsedPartitionNameMap; } - public Multimap getCommonTableIdToRelationIdMap() { - return commonTableIdToRelationIdToMap; + public Multimap getTableIdToRelationIds() { + return tableIdToRelationIds; } public Map> getMvCanRewritePartitionsMap() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java index 49db21ad5a9822..ea7c314652535a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java @@ -83,7 +83,7 @@ public class Group { private List chosenEnforcerPropertiesList = new ArrayList<>(); private List chosenEnforcerIdList = new ArrayList<>(); - private StructInfoMap structInfoMap = new StructInfoMap(); + private final StructInfoMap structInfoMap; /** * Constructor for Group. @@ -92,6 +92,7 @@ public class Group { */ public Group(GroupId groupId, GroupExpression groupExpression, LogicalProperties logicalProperties) { this.groupId = groupId; + this.structInfoMap = new StructInfoMap(this); addGroupExpression(groupExpression); this.logicalProperties = logicalProperties; this.groupPlan = new GroupPlan(this); @@ -104,6 +105,7 @@ public Group(GroupId groupId, GroupExpression groupExpression, LogicalProperties */ public Group(GroupId groupId, LogicalProperties logicalProperties) { this.groupId = groupId; + this.structInfoMap = new StructInfoMap(this); this.logicalProperties = logicalProperties; this.groupPlan = new GroupPlan(this); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index 58e81073c4b719..64ea24134b4280 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -17,9 +17,9 @@ package org.apache.doris.nereids.memo; -import org.apache.doris.catalog.MTMV; import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.cost.Cost; import org.apache.doris.nereids.cost.CostCalculator; import org.apache.doris.nereids.metrics.EventChannel; @@ -52,7 +52,6 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; -import java.util.BitSet; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -61,7 +60,6 @@ import java.util.Optional; import java.util.PriorityQueue; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; @@ -76,9 +74,7 @@ public class Memo { EventChannel.getDefaultChannel().addConsumers(new LogConsumer(GroupMergeEvent.class, EventChannel.LOG))); private static long stateId = 0; private final ConnectContext connectContext; - // The key is the query tableId, the value is the refresh version when last refresh, this is needed - // because struct info refresh base on target tableId. - private final Map refreshVersion = new HashMap<>(); + private final StatementContext statementContext; private final Map, Set> materializationCheckSuccessMap = new LinkedHashMap<>(); private final Map, Set> materializationCheckFailMap = @@ -93,11 +89,17 @@ public class Memo { public Memo() { this.root = null; this.connectContext = null; + this.statementContext = null; } public Memo(ConnectContext connectContext, Plan plan) { + this(connectContext == null ? null : connectContext.getStatementContext(), plan); + } + + private Memo(StatementContext statementContext, Plan plan) { + this.statementContext = statementContext; + this.connectContext = statementContext == null ? null : statementContext.getConnectContext(); this.root = init(plan); - this.connectContext = connectContext; } public static long getStateId() { @@ -132,30 +134,6 @@ public int getGroupExpressionsSize() { return groupExpressions.size(); } - /** get the refresh version map*/ - public Map getRefreshVersion() { - return refreshVersion; - } - - /** return the incremented refresh version for the given commonTableId*/ - public long incrementAndGetRefreshVersion(int commonTableId) { - return refreshVersion.compute(commonTableId, (k, v) -> { - if (v == null) { - return new AtomicInteger(1); - } - v.incrementAndGet(); - return v; - }).get(); - } - - /** return the incremented refresh version for the given relationId set*/ - public void incrementAndGetRefreshVersion(BitSet commonTableIdSet) { - for (int i = commonTableIdSet.nextSetBit(0); i >= 0; - i = commonTableIdSet.nextSetBit(i + 1)) { - incrementAndGetRefreshVersion(i); - } - } - /** * Record materialization check result for performance */ @@ -379,6 +357,7 @@ public Plan copyOut(GroupExpression logicalExpression, boolean includeGroupExpre */ private Group init(Plan plan) { Preconditions.checkArgument(!(plan instanceof GroupPlan), "Cannot init memo by a GroupPlan"); + registerRelationIdentity(plan); // initialize children recursively List childrenGroups = new ArrayList<>(plan.arity()); @@ -481,15 +460,7 @@ private CopyInResult doCopyIn(Plan plan, @Nullable Group targetGroup, @Nullable plan.getLogicalProperties(), targetGroup.getLogicalProperties()); throw new IllegalStateException("Insert a plan into targetGroup but differ in logicalproperties"); } - if (connectContext != null - && connectContext.getSessionVariable().isEnableMaterializedViewNestRewrite() - && plan instanceof LogicalCatalogRelation - && ((CatalogRelation) plan).getTable() instanceof MTMV - && !plan.getGroupExpression().isPresent()) { - TableId mvCommonTableId - = this.connectContext.getStatementContext().getTableId(((CatalogRelation) plan).getTable()); - incrementAndGetRefreshVersion(mvCommonTableId.asInt()); - } + registerRelationIdentity(plan); Optional groupExpr = plan.getGroupExpression(); if (groupExpr.isPresent()) { Preconditions.checkState(groupExpressions.containsKey(groupExpr.get())); @@ -513,6 +484,24 @@ private CopyInResult doCopyIn(Plan plan, @Nullable Group targetGroup, @Nullable // TODO: need to derive logical property if generate new group. currently we not copy logical plan into } + private void registerRelationIdentity(Plan plan) { + if (statementContext == null) { + return; + } + if (plan instanceof LogicalCatalogRelation) { + // StructInfoMap searches query alternatives by relation id, but each MV context starts from the + // table ids in the MV definition. Register both original scans and nested MV scans copied into memo + // so those table ids can be expanded back to the currently available relation ids. + CatalogRelation catalogRelation = (CatalogRelation) plan; + TableId tableId = statementContext.getTableId(catalogRelation.getTable()); + boolean relationIdentityChanged = statementContext.getTableIdToRelationIds() + .put(tableId.asInt(), catalogRelation.getRelationId().asInt()); + if (relationIdentityChanged) { + groups.values().forEach(group -> group.getStructInfoMap().clearCandidateCache()); + } + } + } + private List rewriteChildrenPlansToGroups(Plan plan, Group targetGroup) { List childrenGroups = Lists.newArrayList(); for (int i = 0; i < plan.children().size(); i++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index ac567c4bd344a1..ef994efd190a25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -17,377 +17,275 @@ package org.apache.doris.nereids.memo; -import org.apache.doris.catalog.TableIf; -import org.apache.doris.common.Pair; +import org.apache.doris.catalog.MTMV; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; import org.apache.doris.nereids.rules.exploration.mv.StructInfo; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; -import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.util.MoreFieldsThread; -import com.google.common.collect.Sets; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import com.google.common.collect.Multimap; import java.util.ArrayList; import java.util.BitSet; -import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedList; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nullable; /** - * Representation for group in cascades optimizer. + * Search StructInfo candidates for a memo group. + * + *

MV definitions use statement-scope table ids, while memo alternatives use relation ids. This class expands + * MV table ids to currently visible relation ids before searching candidates. */ public class StructInfoMap { - - public static final Logger LOG = LogManager.getLogger(StructInfoMap.class); - // 2166136261 - private static final int FNV32_OFFSET_BASIS = 0x811C9DC5; - // 16777619 - private static final int FNV32_PRIME = 0x01000193; - /** - * Strategy for table ID mode - */ - private static final IdModeStrategy TABLE_ID_STRATEGY = new IdModeStrategy() { - @Override - public Map>> getGroupExpressionMap(StructInfoMap structInfoMap) { - return structInfoMap.groupExpressionMapByTableId; - } - - @Override - public Map getInfoMap(StructInfoMap structInfoMap) { - return structInfoMap.infoMapByTableId; - } - - @Override - public BitSet constructLeaf(GroupExpression groupExpression, CascadesContext cascadesContext, - boolean forceRefresh) { - Plan plan = groupExpression.getPlan(); - BitSet tableMap = new BitSet(); - if (plan instanceof LogicalCatalogRelation) { - LogicalCatalogRelation relation = (LogicalCatalogRelation) plan; - TableIf table = relation.getTable(); - if (!forceRefresh && cascadesContext.getStatementContext() - .getMaterializationRewrittenSuccessSet().contains(table.getFullQualifiers())) { - return tableMap; - } - tableMap.set(cascadesContext.getStatementContext().getTableId(table).asInt()); - } - return tableMap; - } - - @Override - public int computeMemoVersion(BitSet targetIdMap, CascadesContext cascadesContext) { - return getMemoVersion(targetIdMap, cascadesContext.getMemo().getRefreshVersion()); - } - }; - - /** - * Strategy for relation ID mode - */ - private static final IdModeStrategy RELATION_ID_STRATEGY = new IdModeStrategy() { - @Override - public Map>> getGroupExpressionMap(StructInfoMap structInfoMap) { - return structInfoMap.groupExpressionMapByRelationId; - } - - @Override - public Map getInfoMap(StructInfoMap structInfoMap) { - return structInfoMap.infoMapByRelationId; - } - - @Override - public BitSet constructLeaf(GroupExpression groupExpression, CascadesContext cascadesContext, - boolean forceRefresh) { - Plan plan = groupExpression.getPlan(); - BitSet tableMap = new BitSet(); - if (plan instanceof LogicalCatalogRelation) { - LogicalCatalogRelation relation = (LogicalCatalogRelation) plan; - TableIf table = relation.getTable(); - if (!forceRefresh && cascadesContext.getStatementContext() - .getMaterializationRewrittenSuccessSet().contains(table.getFullQualifiers())) { - return tableMap; - } - tableMap.set(relation.getRelationId().asInt()); - } - if (plan instanceof LogicalCTEConsumer || plan instanceof LogicalEmptyRelation - || plan instanceof LogicalOneRowRelation) { - tableMap.set(((LogicalRelation) plan).getRelationId().asInt()); - } - return tableMap; - } - - @Override - public int computeMemoVersion(BitSet targetIdMap, CascadesContext cascadesContext) { - return getMemoVersion(targetIdMap, cascadesContext.getMemo().getRefreshVersion()); - } - }; - /** - * The map key is the relation id bit set to get corresponding plan accurately - */ - private final Map>> groupExpressionMapByRelationId = new HashMap<>(); - /** - * The map key is the relation id bit set to get corresponding plan accurately - */ - private final Map infoMapByRelationId = new HashMap<>(); - - /** - * The map key is the common table id bit set to get corresponding plan accurately - */ - private final Map>> groupExpressionMapByTableId = new HashMap<>(); - /** - * The map key is the common table id bit set to get corresponding plan accurately - */ - private final Map infoMapByTableId = new HashMap<>(); - - // The key is the tableIds query used, the value is the refresh version when last refresh - private final Map refreshVersion = new HashMap<>(); + private final Group ownerGroup; + // Outer key: relation-id search space expanded from one MV definition. + // Inner key: exact relation-id set contained by one candidate plan tree. + private final Map> candidatesByTargetRelationIdSet = + new LinkedHashMap<>(); + + StructInfoMap(Group ownerGroup) { + this.ownerGroup = ownerGroup; + } /** - * get struct info according to table map - * - * @param targetIdMap the original table map - * @param group the group that the mv matched - * @return struct info or null if not found + * Get the StructInfo whose exact relation-id set matches the given key in the current owner group. */ - public @Nullable StructInfo getStructInfo(CascadesContext cascadesContext, BitSet targetIdMap, Group group, - Plan originPlan, boolean forceRefresh, boolean tableIdMode) { - IdModeStrategy strategy = getStrategy(tableIdMode); - Map infoMap = strategy.getInfoMap(this); - Map>> groupExprMap = strategy.getGroupExpressionMap(this); - - StructInfo structInfo = infoMap.get(targetIdMap); - if (structInfo != null) { - return structInfo; - } - if (groupExprMap.isEmpty() || !groupExprMap.containsKey(targetIdMap)) { - int memoVersion = strategy.computeMemoVersion(targetIdMap, cascadesContext); - refresh(group, cascadesContext, targetIdMap, new HashSet<>(), forceRefresh, memoVersion, tableIdMode); - group.getStructInfoMap().setRefreshVersion(targetIdMap, cascadesContext.getMemo().getRefreshVersion()); + public @Nullable StructInfo getStructInfoByRelationIdSet(CascadesContext cascadesContext, BitSet relationIdSet, + @Nullable Plan originPlan) { + boolean skipMaterializedViewLeaf = !cascadesContext.getConnectContext().getSessionVariable() + .isEnableMaterializedViewNestRewrite(); + StructInfoCandidate candidate = getCandidatesByRelationIdSet(relationIdSet, skipMaterializedViewLeaf) + .get(relationIdSet); + if (candidate == null) { + return null; } - if (groupExprMap.containsKey(targetIdMap)) { - Pair> groupExpressionBitSetPair = - getGroupExpressionWithChildren(targetIdMap, tableIdMode); - // NOTICE: During the transition from physicalAggregate to logical aggregation, - // the original function signature needs to remain unchanged because the constructor - // of LogicalAggregation will recalculate the signature of the aggregation function. - // When the calculated signature is inconsistent with the original signature - // (e.g. due to the influence of the session variable enable_decimal256), - // a problem will arise where the output type of the rewritten plan is inconsistent with - // the output type of the upper-level operator. - structInfo = MoreFieldsThread.keepFunctionSignature(() -> - constructStructInfo(groupExpressionBitSetPair.first, groupExpressionBitSetPair.second, - originPlan, cascadesContext, tableIdMode)); - infoMap.put(targetIdMap, structInfo); - } - return structInfo; - } - - public Set getTableMaps(boolean tableIdMode) { - return getStrategy(tableIdMode).getGroupExpressionMap(this).keySet(); - } - - public Pair> getGroupExpressionWithChildren(BitSet tableMap, boolean tableIdMode) { - return getStrategy(tableIdMode).getGroupExpressionMap(this).get(tableMap); - } - - // Set the refresh version for the given targetIdSet - public void setRefreshVersion(BitSet targetIdSet, Map memoRefreshVersionMap) { - this.refreshVersion.put(targetIdSet, getMemoVersion(targetIdSet, memoRefreshVersionMap)); - } - - // Set the refresh version for the given targetIdSet - public void setRefreshVersion(BitSet targetIdSet, int memoRefreshVersion) { - this.refreshVersion.put(targetIdSet, memoRefreshVersion); - } - - // Get the refresh version for the given targetIdSet, if not exist, return 0 - public long getRefreshVersion(BitSet targetIdSet) { - return refreshVersion.computeIfAbsent(targetIdSet, k -> 0); + return candidate.toStructInfo(originPlan, cascadesContext); } /** - * Compute a compact "version fingerprint" for the given relation id set. - * Algorithm: - * - Uses a 32-bit FNV-1a-style hash. Start from FNV32_OFFSET_BASIS and multiply by FNV32_PRIME. - * - Iterate each set bit (target id) in the BitSet: - * - Fetch its current refresh version from memoRefreshVersionMap (default 0 if absent). - * - Mix the version into the hash by XOR, then diffuse by multiplying the FNV prime. - * - Returns the final hash as the memo version for this set of relations. - * Benefits: - * - Stable fingerprint: any change in any relation's version produces a different hash, enabling - * fast cache invalidation checks without scanning all versions every time. - * - Order-independent: relies on set iteration; the same set yields the same hash regardless of order. - * - Low memory and CPU overhead: compresses multiple integers into a single 32-bit value efficiently. - * - Incremental-friendly: new relations/versions can be incorporated by re-running on the changed set. - * - Good diffusion: XOR + prime multiplication reduces collisions compared to simple sums. - * Notes: - * - The Integer.MAX_VALUE guard prevents potential overflow edge cases in BitSet iteration. + * Collect query StructInfo candidates that may match the MV base table ids. */ - public static int getMemoVersion(BitSet targetIdSet, Map memoRefreshVersionMap) { - int hash = FNV32_OFFSET_BASIS; - for (int id = targetIdSet.nextSetBit(0); - id >= 0; id = targetIdSet.nextSetBit(id + 1)) { - AtomicInteger ver = memoRefreshVersionMap.get(id); - int tmpVer = ver == null ? 0 : ver.get(); - hash ^= tmpVer; - hash *= FNV32_PRIME; - if (id == Integer.MAX_VALUE) { - break; - } + public List collectStructInfosByMvBaseTableId(CascadesContext cascadesContext, + BitSet mvBaseTableIdSet, @Nullable Plan originPlan) { + BitSet targetRelationIdSet = createTargetRelationIdSet(mvBaseTableIdSet, cascadesContext); + boolean skipMaterializedViewLeaf = !cascadesContext.getConnectContext().getSessionVariable() + .isEnableMaterializedViewNestRewrite(); + Map candidates = getCandidatesByRelationIdSet( + targetRelationIdSet, skipMaterializedViewLeaf); + List structInfos = new ArrayList<>(candidates.size()); + for (StructInfoCandidate candidate : candidates.values()) { + structInfos.add(candidate.toStructInfo(originPlan, cascadesContext)); } - return hash; + return structInfos; } - private StructInfo constructStructInfo(GroupExpression groupExpression, List children, - Plan originPlan, CascadesContext cascadesContext, boolean tableIdMode) { - // this plan is not origin plan, should record origin plan in struct info - Plan plan = constructPlan(groupExpression, children, tableIdMode); - return originPlan == null ? StructInfo.of(plan, cascadesContext) - : StructInfo.of(plan, originPlan, cascadesContext); + private Map getCandidatesByRelationIdSet(BitSet targetRelationIdSet, + boolean skipMaterializedViewLeaf) { + Map candidates = + candidatesByTargetRelationIdSet.get(targetRelationIdSet); + if (candidates != null) { + return candidates; + } + candidates = collectStructInfoCandidates(targetRelationIdSet, skipMaterializedViewLeaf); + candidatesByTargetRelationIdSet.put((BitSet) targetRelationIdSet.clone(), candidates); + return candidates; } - private Plan constructPlan(GroupExpression groupExpression, List children, boolean tableIdMode) { - List childrenPlan = new ArrayList<>(); - for (int i = 0; i < children.size(); i++) { - StructInfoMap structInfoMap = groupExpression.child(i).getStructInfoMap(); - BitSet childMap = children.get(i); - Pair> groupExpressionBitSetPair - = structInfoMap.getGroupExpressionWithChildren(childMap, tableIdMode); - childrenPlan.add( - constructPlan(groupExpressionBitSetPair.first, groupExpressionBitSetPair.second, tableIdMode)); - } - // need to clear current group expression info by using withGroupExpression - // this plan would copy into memo, if with group expression, would cause err - return groupExpression.getPlan().withChildren(childrenPlan).withGroupExpression(Optional.empty()); + void clearCandidateCache() { + candidatesByTargetRelationIdSet.clear(); } - /** - * refresh group expression map - * - * @param group the root group - * @param targetBitSet refreshed group expression table bitset must intersect with the targetBitSet - */ - public void refresh(Group group, CascadesContext cascadesContext, - BitSet targetBitSet, Set refreshedGroup, - boolean forceRefresh, int memoVersion, boolean tableIdMode) { - IdModeStrategy strategy = getStrategy(tableIdMode); - Map>> groupExprMap = strategy.getGroupExpressionMap(this); - StructInfoMap structInfoMap = group.getStructInfoMap(); - refreshedGroup.add(group.getGroupId().asInt()); - if (!structInfoMap.getTableMaps(tableIdMode).isEmpty() - && memoVersion == structInfoMap.getRefreshVersion(targetBitSet)) { - return; - } - for (GroupExpression groupExpression : group.getLogicalExpressions()) { - List> childrenTableMap = new LinkedList<>(); + private Map collectStructInfoCandidates(BitSet targetRelationIdSet, + boolean skipMaterializedViewLeaf) { + Map collectedCandidates = new LinkedHashMap<>(); + for (GroupExpression groupExpression : ownerGroup.getLogicalExpressions()) { if (groupExpression.children().isEmpty()) { - BitSet leaf = strategy.constructLeaf(groupExpression, cascadesContext, forceRefresh); - if (leaf.isEmpty()) { - break; - } - groupExprMap.put(leaf, Pair.of(groupExpression, new LinkedList<>())); - continue; - } - // this is used for filter group expression whose children's table map all not in targetBitSet - BitSet filteredTableMaps = new BitSet(); - // groupExpression self could be pruned - for (Group child : groupExpression.children()) { - // group in expression should all be reserved - StructInfoMap childStructInfoMap = child.getStructInfoMap(); - if (!refreshedGroup.contains(child.getGroupId().asInt())) { - childStructInfoMap.refresh(child, cascadesContext, targetBitSet, - refreshedGroup, forceRefresh, memoVersion, tableIdMode); - childStructInfoMap.setRefreshVersion(targetBitSet, memoVersion); + Plan leafPlan = groupExpression.getPlan(); + if (!(leafPlan instanceof LogicalRelation)) { + continue; } - Set groupTableSet = new HashSet<>(); - for (BitSet tableMaps : child.getStructInfoMap().getTableMaps(tableIdMode)) { - groupTableSet.add(tableMaps); - filteredTableMaps.or(tableMaps); + BitSet leafRelationIdSet = new BitSet(); + leafRelationIdSet.set(((LogicalRelation) leafPlan).getRelationId().asInt()); + if (!MaterializedViewUtils.containsAll(targetRelationIdSet, leafRelationIdSet)) { + continue; } - if (!filteredTableMaps.isEmpty()) { - childrenTableMap.add(groupTableSet); + // Without nested MV rewrite, rewritten MV scan leaves must not become query candidates. + if (skipMaterializedViewLeaf + && leafPlan instanceof CatalogRelation + && ((CatalogRelation) leafPlan).getTable() instanceof MTMV) { + continue; } - } - // filter the tableSet that used intersects with targetBitSet, make sure the at least constructed - if (!structInfoMap.getTableMaps(tableIdMode).isEmpty() && !targetBitSet.isEmpty() - && !filteredTableMaps.isEmpty() && !filteredTableMaps.intersects(targetBitSet)) { - continue; - } - if (childrenTableMap.isEmpty()) { + collectedCandidates.putIfAbsent((BitSet) leafRelationIdSet.clone(), + StructInfoCandidate.ofLeaf(leafPlan.withGroupExpression(Optional.empty()))); continue; } - // if groupExpression which has the same table set have refreshed, continue - BitSet eachGroupExpressionTableSet = new BitSet(); - for (Set groupExpressionBitSet : childrenTableMap) { - for (BitSet bitSet : groupExpressionBitSet) { - eachGroupExpressionTableSet.or(bitSet); + List> childCandidatesByChild = new ArrayList<>(); + boolean hasEmptyChild = false; + for (Group child : groupExpression.children()) { + Map childCandidates = + child.getStructInfoMap().getCandidatesByRelationIdSet( + targetRelationIdSet, skipMaterializedViewLeaf); + if (childCandidates.isEmpty()) { + hasEmptyChild = true; + break; } + childCandidatesByChild.add(childCandidates); } - if (groupExprMap.containsKey(eachGroupExpressionTableSet)) { - // for the group expressions of group, only need to refresh any of the group expression - // when they have the same group expression table set + if (hasEmptyChild) { continue; } - // if cumulative child table map is different from current - // or current group expression map is empty, should update the groupExpressionMap currently - Collection>> bitSetWithChildren = cartesianProduct(childrenTableMap); - for (Pair> bitSetWithChild : bitSetWithChildren) { - groupExprMap.putIfAbsent(bitSetWithChild.first, - Pair.of(groupExpression, bitSetWithChild.second)); + enumerateChildCandidateCombinations(groupExpression, childCandidatesByChild, 0, new BitSet(), + new ArrayList<>(childCandidatesByChild.size()), new HashMap<>(), collectedCandidates, + targetRelationIdSet); + } + return collectedCandidates; + } + + /* + * Enumerate child candidate combinations for one group expression. Each combination produces a union of + * relation ids; only unions contained by targetRelationIdSet can become composite candidates. + */ + private void enumerateChildCandidateCombinations(GroupExpression groupExpression, + List> childCandidatesByChild, int childOffset, + BitSet currentRelationIdUnion, List currentChildren, + Map> visitedRelationIdUnionsByOffset, + Map candidates, BitSet targetRelationIdSet) { + // Same relation-id union at the same child offset is equivalent for later matching. + Set visitedRelationIdUnions = visitedRelationIdUnionsByOffset.computeIfAbsent( + childOffset, ignored -> new HashSet<>()); + if (!visitedRelationIdUnions.add(currentRelationIdUnion)) { + return; + } + if (childOffset >= childCandidatesByChild.size()) { + candidates.putIfAbsent(currentRelationIdUnion, + StructInfoCandidate.ofComposite(groupExpression, currentChildren)); + return; + } + for (Map.Entry childCandidate + : childCandidatesByChild.get(childOffset).entrySet()) { + BitSet nextRelationIdUnion = (BitSet) currentRelationIdUnion.clone(); + nextRelationIdUnion.or(childCandidate.getKey()); + if (!MaterializedViewUtils.containsAll(targetRelationIdSet, nextRelationIdUnion)) { + continue; } + currentChildren.add(childCandidate.getValue()); + enumerateChildCandidateCombinations(groupExpression, childCandidatesByChild, childOffset + 1, + nextRelationIdUnion, currentChildren, visitedRelationIdUnionsByOffset, candidates, + targetRelationIdSet); + currentChildren.remove(currentChildren.size() - 1); } } - private Collection>> cartesianProduct(List> childrenTableMap) { - Set> cartesianLists = Sets.cartesianProduct(childrenTableMap); - List>> resultPairSet = new LinkedList<>(); - for (List bitSetList : cartesianLists) { - BitSet bitSet = new BitSet(); - for (BitSet b : bitSetList) { - bitSet.or(b); + private BitSet createTargetRelationIdSet(BitSet mvBaseTableIdSet, CascadesContext cascadesContext) { + StatementContext statementContext = cascadesContext.getStatementContext(); + Multimap tableIdToRelationIds = statementContext.getTableIdToRelationIds(); + BitSet targetRelationIdSet = new BitSet(); + // Include original query relations and nested MV scan relations that have been copied into the memo. + for (int tableId = mvBaseTableIdSet.nextSetBit(0); + tableId >= 0; tableId = mvBaseTableIdSet.nextSetBit(tableId + 1)) { + for (Integer relationId : tableIdToRelationIds.get(tableId)) { + targetRelationIdSet.set(relationId); + } + if (tableId == Integer.MAX_VALUE) { + break; } - resultPairSet.add(Pair.of(bitSet, bitSetList)); } - return resultPairSet; + return targetRelationIdSet; } - /** - * Strategy interface to handle different ID modes (tableId vs relationId) + /* + * Lightweight representative of one candidate plan tree. The relation-id set is kept as the map key outside + * this object; StructInfo and rebuilt Plan are created lazily after relation-level filtering. */ - private interface IdModeStrategy { - Map>> getGroupExpressionMap(StructInfoMap structInfoMap); + private static final class StructInfoCandidate { + private final GroupExpression groupExpression; + private final List children; + private final Map structInfosByOriginPlan = new IdentityHashMap<>(); + private List relations; + private Plan materializedPlan; + + private StructInfoCandidate(Plan materializedPlan, GroupExpression groupExpression, + List children, + List relations) { + this.materializedPlan = materializedPlan; + this.groupExpression = groupExpression; + this.children = children; + this.relations = relations; + } - Map getInfoMap(StructInfoMap structInfoMap); + private static StructInfoCandidate ofLeaf(Plan plan) { + List relations = new ArrayList<>(); + if (plan instanceof CatalogRelation) { + relations.add((CatalogRelation) plan); + } + return new StructInfoCandidate(plan, null, Collections.emptyList(), relations); + } - BitSet constructLeaf(GroupExpression groupExpression, CascadesContext cascadesContext, boolean forceRefresh); + private static StructInfoCandidate ofComposite(GroupExpression groupExpression, + List children) { + List copiedChildren = new ArrayList<>(children); + return new StructInfoCandidate(null, groupExpression, copiedChildren, null); + } - int computeMemoVersion(BitSet targetIdMap, CascadesContext cascadesContext); - } + private List getRelations() { + if (relations != null) { + return relations; + } + relations = new ArrayList<>(); + for (StructInfoCandidate child : children) { + relations.addAll(child.getRelations()); + } + return relations; + } - private static IdModeStrategy getStrategy(boolean tableIdMode) { - return tableIdMode ? TABLE_ID_STRATEGY : RELATION_ID_STRATEGY; + private StructInfo toStructInfo(@Nullable Plan originPlan, CascadesContext cascadesContext) { + StructInfo structInfo = structInfosByOriginPlan.get(originPlan); + if (structInfo != null) { + return structInfo; + } + // Rebuilt candidates drop memo group expressions, but aggregate function signatures must keep the + // original resolved signature. Recomputing them here can change output types under session variables. + structInfo = MoreFieldsThread.keepFunctionSignature(() -> { + Plan plan = toPlan(); + return originPlan == null ? StructInfo.of(plan, cascadesContext) + : StructInfo.of(plan, originPlan, cascadesContext); + }); + structInfosByOriginPlan.put(originPlan, structInfo); + return structInfo; + } + + private Plan toPlan() { + if (materializedPlan != null) { + return materializedPlan; + } + List childrenPlans = new ArrayList<>(children.size()); + for (StructInfoCandidate child : children) { + childrenPlans.add(child.toPlan()); + } + // Rebuild an equivalent plain plan tree without keeping the original memo binding. + materializedPlan = groupExpression.getPlan() + .withChildren(childrenPlans) + .withGroupExpression(Optional.empty()); + return materializedPlan; + } } @Override public String toString() { return "StructInfoMap{" - + " groupExpressionMapByRelationId=" + groupExpressionMapByRelationId.keySet() - + ", infoMapByRelationId=" + infoMapByRelationId.keySet() - + ", groupExpressionMapByTableId=" + groupExpressionMapByTableId.keySet() - + ", infoMapByTableId=" + infoMapByTableId.keySet() - + ", refreshVersion=" + refreshVersion + + "ownerGroup=" + ownerGroup.getGroupId() + '}'; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index b9c6e2579ab14e..71e6660040226e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.rules.analysis.NormalizeRepeat; import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewAggregateRule.AggregateExpressionRewriteContext.ExpressionRewriteMode; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PatternCheckResult; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanSplitContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; @@ -611,9 +612,10 @@ protected Pair> splitToTopPlanAndAggregate(StructIn */ @Override protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); + PatternCheckResult checkResult = structInfo.getPlanPatternCheckResult(SUPPORTED_JOIN_TYPE_SET); + PlanCheckContext checkContext = checkResult.getCheckContext(); // if query or mv contains more than one top aggregate, should fail - return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) + return checkResult.isAccepted() && checkContext.isContainsTopAggregate() && checkContext.getTopAggregateNum() <= 1 && !checkContext.isContainsTopLimit() && !checkContext.isContainsTopTopN() && !checkContext.isContainsTopWindow(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index f56cb00c5d61bd..58c2f68cb3cffb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PatternCheckResult; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.expressions.Alias; @@ -45,10 +46,11 @@ protected Plan rewriteQueryByView(MatchMode matchMode, Plan tempRewritedPlan, MaterializationContext materializationContext, CascadesContext cascadesContext) { - // Rewrite top projects, represent the query projects by view - List expressionsRewritten = rewriteExpression( + // Keep the original output order and rewrite from StructInfo output lineage, instead of stopping at + // top-level aliases such as out_bu/out_mode. + List expressionsRewritten = rewriteShuttledExpressions( queryStructInfo.getExpressions(), - queryStructInfo.getTopPlan(), + queryStructInfo.getPlanOutputShuttledExpressions(), materializationContext.getShuttledExprToScanExprMapping(), targetToSourceMapping, ImmutableMap.of(), cascadesContext @@ -78,8 +80,9 @@ protected Plan rewriteQueryByView(MatchMode matchMode, */ @Override protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); - return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) + PatternCheckResult checkResult = structInfo.getPlanPatternCheckResult(SUPPORTED_JOIN_TYPE_SET); + PlanCheckContext checkContext = checkResult.getCheckContext(); + return checkResult.isAccepted() && !checkContext.isContainsTopAggregate() && !checkContext.isContainsTopLimit() && !checkContext.isContainsTopTopN() && !checkContext.isContainsTopWindow(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 64361ba9a979ad..b515ef178cc034 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -79,6 +79,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.BitSet; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -133,7 +134,7 @@ public List rewrite(Plan queryPlan, CascadesContext cascadesContext) { } for (MaterializationContext materializationContext : cascadesContext.getMaterializationContexts()) { statementContext.getMaterializedViewStopwatch().reset().start(); - if (checkIfRewritten(queryPlan, materializationContext)) { + if (isQueryGroupAlreadyRewritten(queryPlan, materializationContext)) { continue; } // check mv plan is valid or not @@ -184,17 +185,25 @@ public List rewrite(Plan queryPlan, CascadesContext cascadesContext) { return rewrittenPlans; } + private boolean isQueryGroupAlreadyRewritten(Plan queryPlan, MaterializationContext materializationContext) { + return queryPlan.getGroupExpression().isPresent() + && materializationContext.alreadyRewrite( + queryPlan.getGroupExpression().get().getOwnerGroup().getGroupId()); + } + /** * Get valid query struct infos, if invalid record the invalid reason */ protected List getValidQueryStructInfos(Plan queryPlan, CascadesContext cascadesContext, MaterializationContext materializationContext) { List validStructInfos = new ArrayList<>(); - // For every materialized view we should trigger refreshing struct info map + BitSet mvBaseTableIdSet = materializationContext.getBaseTableIdSet(); List uncheckedQueryStructInfos = MaterializedViewUtils.extractStructInfoFuzzy(queryPlan, queryPlan, - cascadesContext, materializationContext.getCommonTableIdSet(cascadesContext.getStatementContext())); - uncheckedQueryStructInfos.forEach(queryStructInfo -> { - boolean valid = checkQueryPattern(queryStructInfo, cascadesContext) && queryStructInfo.isValid(); + cascadesContext, mvBaseTableIdSet); + for (StructInfo queryStructInfo : uncheckedQueryStructInfos) { + boolean patternValid = checkQueryPattern(queryStructInfo, cascadesContext); + boolean structValid = queryStructInfo.isValid(); + boolean valid = patternValid && structValid; if (!valid) { cascadesContext.getMaterializationContexts().forEach(ctx -> ctx.recordFailReason(queryStructInfo, "Query struct info is invalid", @@ -204,7 +213,7 @@ protected List getValidQueryStructInfos(Plan queryPlan, CascadesCont } else { validStructInfos.add(queryStructInfo); } - }); + } return validStructInfos; } @@ -224,13 +233,12 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca return rewriteResults; } SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable(); - int materializedViewRelationMappingMaxCount = sessionVariable.getMaterializedViewRelationMappingMaxCount(); - List queryToViewTableMappings = RelationMapping.generate(queryStructInfo.getRelations(), - viewStructInfo.getRelations(), materializedViewRelationMappingMaxCount); + List queryToViewTableMappings = materializationContext.getRelationMappings( + queryStructInfo, sessionVariable.getMaterializedViewRelationMappingMaxCount()); // if any relation in query and view can not map, bail out. - if (queryToViewTableMappings == null) { + if (queryToViewTableMappings.isEmpty()) { materializationContext.recordFailReason(queryStructInfo, - "Query to view table mapping is null", () -> ""); + "Query to view table mapping is empty", () -> ""); return rewriteResults; } for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) { @@ -274,6 +282,7 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca Plan rewrittenPlan; Plan mvScan = materializationContext.getScanPlan(queryStructInfo, cascadesContext); Plan queryPlan = queryStructInfo.getTopPlan(); + Plan queryOriginalPlan = queryStructInfo.getOriginalPlan(); if (compensatePredicates.isAlwaysTrue()) { rewrittenPlan = mvScan; } else { @@ -416,7 +425,7 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca } } List rewrittenPlanOutput = rewrittenPlan.getOutput(); - rewrittenPlan = MaterializedViewUtils.normalizeExpressions(rewrittenPlan, queryPlan); + rewrittenPlan = MaterializedViewUtils.normalizeExpressions(rewrittenPlan, queryOriginalPlan); if (rewrittenPlan == null) { // maybe virtual slot reference added automatically materializationContext.recordFailReason(queryStructInfo, @@ -424,7 +433,7 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca () -> String.format("materialized view rule normalizeExpressions, output size between " + "origin and rewritten plan is different, rewritten output is %s, " + "origin output is %s", - rewrittenPlanOutput, queryPlan.getOutput())); + rewrittenPlanOutput, queryOriginalPlan.getOutput())); continue; } // Merge project @@ -435,21 +444,17 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca ).execute(); return childContext.getRewritePlan(); }, rewrittenPlan, queryPlan, false); - if (!isOutputValid(queryPlan, rewrittenPlan)) { + if (!isOutputValid(queryOriginalPlan, rewrittenPlan)) { LogicalProperties logicalProperties = rewrittenPlan.getLogicalProperties(); materializationContext.recordFailReason(queryStructInfo, "RewrittenPlan output logical properties is different with target group", () -> String.format("planOutput logical" + " properties = %s,\n groupOutput logical properties = %s", - logicalProperties, queryPlan.getLogicalProperties())); + logicalProperties, queryOriginalPlan.getLogicalProperties())); continue; } - // need to collect table partition again, because the rewritten plan would contain new relation - // and the rewritten plan would part in rewritten later, the table used partition info is needed - // for later rewrite - // record new mv relation id to table in statement context for nest rewrite, because in nest rewrite, - // would get query struct info by mv struct info, if not record, - // MaterializedViewUtils.transformToCommonTableId would fail + // Record partition usage of the rewritten MV scan. Relation-id search metadata is registered later when + // the rewritten plan is copied into Memo. long startTimeMs = TimeUtils.getStartTimeMs(); try { MaterializedViewUtils.collectTableUsedPartitions(rewrittenPlan, cascadesContext); @@ -465,7 +470,7 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca materializationContext.getShuttledExprToScanExprMapping(), viewToQuerySlotMapping, materializationContext); rewriteResults.add(rewrittenPlan); - recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext, cascadesContext); + recordRewriteSuccess(queryStructInfo, materializationContext); resetMaterializationContext(materializationContext, cascadesContext); } return rewriteResults; @@ -572,12 +577,20 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf protected List rewriteExpression(List sourceExpressionsToWrite, Plan sourcePlan, ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping, Map queryExprToInfoMap, CascadesContext cascadesContext) { + List sourceShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( + sourceExpressionsToWrite, sourcePlan); + return rewriteShuttledExpressions(sourceExpressionsToWrite, sourceShuttledExpressions, + targetExpressionMapping, targetToSourceMapping, queryExprToInfoMap, cascadesContext); + } + + protected List rewriteShuttledExpressions(List sourceExpressionsToWrite, + List sourceShuttledExpressions, + ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping, + Map queryExprToInfoMap, CascadesContext cascadesContext) { // Firstly, rewrite the target expression using source with inverse mapping // then try to use the target expression to represent the query. if any of source expressions // could not be represented by target expressions, return null. // generate target to target replacement expression mapping, and change target expression to source based - List sourceShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( - sourceExpressionsToWrite, sourcePlan); ExpressionMapping expressionMappingKeySourceBased = targetExpressionMapping.keyPermute(targetToSourceMapping); // target to target replacement expression mapping, because mv is 1:1 so get the first element List> flattenExpressionMap = expressionMappingKeySourceBased.flattenMap(); @@ -955,20 +968,17 @@ protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesCon return checkQueryPattern(structInfo, cascadesContext); } - protected void recordIfRewritten(Plan plan, MaterializationContext context, CascadesContext cascadesContext) { + protected void recordRewriteSuccess(StructInfo queryStructInfo, MaterializationContext context) { context.setSuccess(true); - cascadesContext.getStatementContext().addMaterializationRewrittenSuccess( - context.generateMaterializationIdentifier()); - if (plan.getGroupExpression().isPresent()) { - context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId(), true); + // This only suppresses repeated attempts of the same MV against the same query group. It does not reopen parent + // failures: current Cascades stack order explores child groups before parent MV rules, so nested child MV scan + // alternatives should already be in Memo when the parent group is matched. + Plan queryPlan = queryStructInfo.getOriginalPlan(); + if (queryPlan.getGroupExpression().isPresent()) { + context.addMatchedGroup(queryPlan.getGroupExpression().get().getOwnerGroup().getGroupId(), true); } } - protected boolean checkIfRewritten(Plan plan, MaterializationContext context) { - return plan.getGroupExpression().isPresent() - && context.alreadyRewrite(plan.getGroupExpression().get().getOwnerGroup().getGroupId()); - } - // check mv plan is valid or not, this can use cache for performance private boolean isMaterializationValid(Plan queryPlan, CascadesContext cascadesContext, MaterializationContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java index e542194f50890f..218d93f43a5b32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PatternCheckResult; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.expressions.Alias; @@ -27,7 +28,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.stream.Collectors; @@ -78,8 +78,9 @@ protected Plan rewriteQueryByView(MatchMode matchMode, */ @Override protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(ImmutableSet.of()); - return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext) + PatternCheckResult checkResult = structInfo.getScanPatternCheckResult(); + PlanCheckContext checkContext = checkResult.getCheckContext(); + return checkResult.isAccepted() && !checkContext.isContainsTopAggregate() && !checkContext.isContainsTopLimit() && !checkContext.isContainsTopTopN() && !checkContext.isContainsTopWindow(); @@ -87,8 +88,9 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca @Override protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); - return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) + PatternCheckResult checkResult = structInfo.getPlanPatternCheckResult(SUPPORTED_JOIN_TYPE_SET); + PlanCheckContext checkContext = checkResult.getCheckContext(); + return checkResult.isAccepted() && !checkContext.isContainsTopAggregate() && !checkContext.isContainsTopLimit() && !checkContext.isContainsTopTopN() && !checkContext.isContainsTopWindow(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 56e6e4b88bf997..40759eac1d3455 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -24,7 +24,6 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.memo.GroupId; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; @@ -69,7 +68,6 @@ */ public abstract class MaterializationContext { private static final Logger LOG = LogManager.getLogger(MaterializationContext.class); - public final Map queryToMaterializationSlotMappingCache = new HashMap<>(); protected List baseTables; protected List
baseViews; // The plan of materialization def sql @@ -79,9 +77,6 @@ public abstract class MaterializationContext { // Should regenerate when materialization is already rewritten successfully because one query may hit repeatedly // make sure output is different in multi using protected Plan scanPlan; - // The materialization plan output shuttled expression, this is used by generate field - // exprToScanExprMapping - protected List planOutputShuttledExpressions; // Generated mapping from materialization plan out expr to materialization scan plan out slot mapping, // this is used for later protected Map exprToScanExprMapping = new HashMap<>(); @@ -98,17 +93,24 @@ public abstract class MaterializationContext { // The materialization plan struct info, construct struct info is expensive, // this should be constructed once for all query for performance protected final StructInfo structInfo; - // Group id set that are rewritten unsuccessfully by this materialization for reducing rewrite times + // Coarse retry guard keyed only by (this materialization, query group). + // It does not distinguish which MV rule or which newly generated logical expression in the same group + // produced the failure, so it is only a practical cache for reducing retries under the current memo state. protected final Set matchedFailGroups = new HashSet<>(); - // Group id set that are rewritten successfully by this materialization for reducing rewrite times + // Coarse success cache keyed only by (this materialization, query group). + // Once one rewrite path for this MV succeeds on the group, later alternatives from other MV rules on the + // same group may be skipped as an engineering tradeoff to reduce repeated work. protected final Set matchedSuccessGroups = new HashSet<>(); // Record the reason, if rewrite by materialization fail. The failReason should be empty if success. // The key is the query belonged group expression objectId, the value is the fail reasons because // for one materialization query may be multi when nested materialized view. protected final Multimap> failReason = HashMultimap.create(); protected List identifier; - // The common table id set which is used in materialization, added for performance consideration - private BitSet commonTableIdSet; + private final Map queryToMaterializationSlotMappingCache = new HashMap<>(); + // RelationMapping depends on the query relation-id set and this materialization's fixed view-side relations. + private final Map> queryToMaterializationRelationMappingCache = new HashMap<>(); + // Statement-scope table ids of relations directly used by this materialization, including normal tables and MVs. + private final BitSet baseTableIdSet; /** * MaterializationContext, this contains necessary info for query rewriting by materialization @@ -120,14 +122,13 @@ public MaterializationContext(Plan plan, Plan originalPlan, StatementBase parsedStatement = cascadesContext.getStatementContext().getParsedStatement(); this.enableRecordFailureDetail = parsedStatement != null && parsedStatement.isExplain() && ExplainLevel.MEMO_PLAN == parsedStatement.getExplainOptions().getExplainLevel(); - // Construct materialization struct info, catch exception which may cause planner roll back this.structInfo = structInfo == null - ? constructStructInfo(plan, originalPlan, cascadesContext).orElseGet(() -> null) + ? constructStructInfo(plan, originalPlan, cascadesContext).orElse(null) : structInfo; this.available = this.structInfo != null; - if (available) { - this.planOutputShuttledExpressions = this.structInfo.getPlanOutputShuttledExpressions(); - } + this.baseTableIdSet = this.structInfo == null + ? new BitSet() + : constructBaseTableIdSet(this.structInfo, cascadesContext.getStatementContext()); } /** @@ -172,7 +173,8 @@ public void tryGenerateScanPlan(CascadesContext cascadesContext) { // Materialization output expression shuttle, this will be used to expression rewrite List scanPlanOutput = this.scanPlan.getOutput(); // generate expression depend on the order of output - this.shuttledExprToScanExprMapping = ExpressionMapping.generate(this.planOutputShuttledExpressions, + this.shuttledExprToScanExprMapping = ExpressionMapping.generate( + this.structInfo.getPlanOutputShuttledExpressions(), scanPlanOutput); // This is used by normalize statistics column expression Map regeneratedMapping = new HashMap<>(); @@ -204,6 +206,13 @@ public SlotMapping getSlotMappingFromCache(RelationMapping relationMapping) { return queryToMaterializationSlotMappingCache.get(relationMapping); } + public List getRelationMappings(StructInfo queryStructInfo, int maxMappingCount) { + BitSet relationIdSet = queryStructInfo.getRelationBitSet(); + return queryToMaterializationRelationMappingCache.computeIfAbsent((BitSet) relationIdSet.clone(), + ignored -> RelationMapping.generate(queryStructInfo.getRelations(), + structInfo.getRelations(), maxMappingCount)); + } + /** * Try to generate scan plan for materialization * if MaterializationContext is already rewritten successfully, then should generate new scan plan in later @@ -343,11 +352,12 @@ public boolean isSuccess() { * Record fail reason when in rewriting by struct info */ public void recordFailReason(StructInfo structInfo, String summary, Supplier failureReasonSupplier) { - // record it's rewritten - if (structInfo.getTopPlan().getGroupExpression().isPresent()) { - this.addMatchedGroup(structInfo.getTopPlan().getGroupExpression().get().getOwnerGroup().getGroupId(), - false); - } + // StructInfo may be built from a detached candidate plan whose top node no longer keeps the original + // groupExpression. For retry suppression we still need to mark the source memo group as failed, so use + // originalPlan here: it still points to the query group that should be skipped until nested-MV state changes. + structInfo.getOriginalPlan().getGroupExpression() + .ifPresent(groupExpression -> this.addMatchedGroup(groupExpression.getOwnerGroup().getGroupId(), + false)); // once success, do not record the fail reason if (this.success) { return; @@ -370,24 +380,20 @@ public void recordFailReason(Plan queryGroupPlan, String summary, Supplier new ObjectId(-1)), + .map(GroupExpression::getId).orElse(new ObjectId(-1)), Pair.of(summary, this.isEnableRecordFailureDetail() ? failureReasonSupplier.get() : "")); } - /** - * get materialization context common table id by current currentQueryStatementContext - */ - public BitSet getCommonTableIdSet(StatementContext currentQueryStatementContext) { - if (commonTableIdSet != null) { - return commonTableIdSet; - } - commonTableIdSet = new BitSet(); - for (StructInfoNode node : structInfo.getRelationIdStructInfoNodeMap().values()) { - for (CatalogRelation catalogRelation : node.getCatalogRelation()) { - commonTableIdSet.set(currentQueryStatementContext.getTableId(catalogRelation.getTable()).asInt()); - } + public BitSet getBaseTableIdSet() { + return baseTableIdSet; + } + + private static BitSet constructBaseTableIdSet(StructInfo structInfo, StatementContext statementContext) { + BitSet baseTableIdSet = new BitSet(); + for (CatalogRelation relation : structInfo.getRelations()) { + baseTableIdSet.set(statementContext.getTableId(relation.getTable()).asInt()); } - return commonTableIdSet; + return baseTableIdSet; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java index 12a822b73885ee..0e2638eadbc71f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewAggregateRule.AggregateExpressionRewriteContext.ExpressionRewriteMode; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PatternCheckResult; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.plans.Plan; @@ -34,7 +35,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Map; @@ -80,19 +80,21 @@ public List buildRules() { @Override protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) { // any check result of join or scan is true, then return true - PlanCheckContext joinCheckContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); - boolean joinCheckResult = structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, joinCheckContext) + PatternCheckResult joinCheckResult = structInfo.getPlanPatternCheckResult(SUPPORTED_JOIN_TYPE_SET); + PlanCheckContext joinCheckContext = joinCheckResult.getCheckContext(); + boolean joinAccepted = joinCheckResult.isAccepted() && !joinCheckContext.isContainsTopAggregate() && !joinCheckContext.isContainsTopLimit() && !joinCheckContext.isContainsTopTopN() && !joinCheckContext.isContainsTopWindow(); - if (joinCheckResult) { + if (joinAccepted) { return true; } - PlanCheckContext scanCheckContext = PlanCheckContext.of(ImmutableSet.of()); - return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, scanCheckContext) + PatternCheckResult scanCheckResult = structInfo.getScanPatternCheckResult(); + PlanCheckContext scanCheckContext = scanCheckResult.getCheckContext(); + return scanCheckResult.isAccepted() && !scanCheckContext.isContainsTopAggregate() - && !joinCheckContext.isContainsTopLimit() && !joinCheckContext.isContainsTopTopN() - && !joinCheckContext.isContainsTopWindow(); + && !scanCheckContext.isContainsTopLimit() && !scanCheckContext.isContainsTopTopN() + && !scanCheckContext.isContainsTopWindow(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitAggregateRule.java index 575e8f406d5768..9ffc2648a7c23f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitAggregateRule.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PatternCheckResult; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.plans.LimitPhase; @@ -66,8 +67,9 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf @Override protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); - return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) + PatternCheckResult checkResult = structInfo.getPlanPatternCheckResult(SUPPORTED_JOIN_TYPE_SET); + PlanCheckContext checkContext = checkResult.getCheckContext(); + return checkResult.isAccepted() && checkContext.isContainsTopAggregate() && checkContext.getTopAggregateNum() == 1 && !checkContext.isContainsTopTopN() && !checkContext.isContainsTopWindow() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitJoinRule.java index 2bfe8fe59efb42..17c47c06bcf75e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitJoinRule.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PatternCheckResult; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.plans.LimitPhase; @@ -66,8 +67,9 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf @Override protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); - return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) + PatternCheckResult checkResult = structInfo.getPlanPatternCheckResult(SUPPORTED_JOIN_TYPE_SET); + PlanCheckContext checkContext = checkResult.getCheckContext(); + return checkResult.isAccepted() && !checkContext.isContainsTopAggregate() && !checkContext.isContainsTopTopN() && !checkContext.isContainsTopWindow() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java index b230d7fbd33d88..1582c20e9d05ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PatternCheckResult; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.plans.LimitPhase; @@ -30,7 +31,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Optional; @@ -67,8 +67,9 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf @Override protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(ImmutableSet.of()); - return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext) + PatternCheckResult checkResult = structInfo.getScanPatternCheckResult(); + PlanCheckContext checkContext = checkResult.getCheckContext(); + return checkResult.isAccepted() && !checkContext.isContainsTopAggregate() && !checkContext.isContainsTopTopN() && !checkContext.isContainsTopWindow() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNAggregateRule.java index 7b96f37ecdc708..b6720ac77893b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNAggregateRule.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PatternCheckResult; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.plans.Plan; @@ -63,8 +64,9 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf @Override protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); - return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) + PatternCheckResult checkResult = structInfo.getPlanPatternCheckResult(SUPPORTED_JOIN_TYPE_SET); + PlanCheckContext checkContext = checkResult.getCheckContext(); + return checkResult.isAccepted() && checkContext.isContainsTopAggregate() && checkContext.getTopAggregateNum() == 1 && !checkContext.isContainsTopLimit() && !checkContext.isContainsTopWindow() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNJoinRule.java index 7c7d6b89fcb2f6..d206c993b3f1db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNJoinRule.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PatternCheckResult; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.plans.Plan; @@ -63,9 +64,9 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf @Override protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); - Boolean accept = structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext); - return accept + PatternCheckResult checkResult = structInfo.getPlanPatternCheckResult(SUPPORTED_JOIN_TYPE_SET); + PlanCheckContext checkContext = checkResult.getCheckContext(); + return checkResult.isAccepted() && !checkContext.isContainsTopAggregate() && !checkContext.isContainsTopLimit() && !checkContext.isContainsTopWindow() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java index 43f5dcf10fb935..d42e2363d9868a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PatternCheckResult; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.plans.Plan; @@ -29,7 +30,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Optional; @@ -64,9 +64,9 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf @Override protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(ImmutableSet.of()); - Boolean accept = structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext); - return accept + PatternCheckResult checkResult = structInfo.getScanPatternCheckResult(); + PlanCheckContext checkContext = checkResult.getCheckContext(); + return checkResult.isAccepted() && !checkContext.isContainsTopAggregate() && !checkContext.isContainsTopLimit() && !checkContext.isContainsTopWindow() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 4c3194703f5153..d8cd1fafab3b35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -252,61 +252,17 @@ public static boolean containTableQueryOperator(Plan analyzedPlan) { return analyzedPlan.accept(TableQueryOperatorChecker.INSTANCE, null); } - /** - * Transform to common table id, this is used by get query struct info, maybe little err when same table occur - * more than once, this is not a problem because the process of query rewrite by mv would consider more - */ - public static BitSet transformToCommonTableId(BitSet relationIdSet, Map relationIdToTableIdMap) { - BitSet transformedBitset = new BitSet(); - for (int i = relationIdSet.nextSetBit(0); i >= 0; i = relationIdSet.nextSetBit(i + 1)) { - Integer commonTableId = relationIdToTableIdMap.get(i); - if (commonTableId != null) { - transformedBitset.set(commonTableId); - } - } - return transformedBitset; - } - - /** - * Extract struct info from plan, support to get struct info from logical plan or plan in group. - * @param plan maybe remove unnecessary plan node, and the logical output maybe wrong - * @param originalPlan original plan, the output is right - * @param cascadesContext the cascadesContext when extractStructInfo - * @param targetTableIdSet the target relation id set which used to filter struct info, - * empty means no struct info match - */ + /** Fuzzily collect StructInfo candidates under the current plan/group that may hit mvBaseTableIdSet. */ public static List extractStructInfoFuzzy(Plan plan, Plan originalPlan, - CascadesContext cascadesContext, BitSet targetTableIdSet) { - // If plan belong to some group, construct it with group struct info + CascadesContext cascadesContext, BitSet mvBaseTableIdSet) { if (plan.getGroupExpression().isPresent()) { Group ownerGroup = plan.getGroupExpression().get().getOwnerGroup(); StructInfoMap structInfoMap = ownerGroup.getStructInfoMap(); - // Refresh struct info in current level plan from top to bottom - SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable(); - int memoVersion = StructInfoMap.getMemoVersion(targetTableIdSet, - cascadesContext.getMemo().getRefreshVersion()); - structInfoMap.refresh(ownerGroup, cascadesContext, targetTableIdSet, new HashSet<>(), - sessionVariable.isEnableMaterializedViewNestRewrite(), memoVersion, true); - structInfoMap.setRefreshVersion(targetTableIdSet, cascadesContext.getMemo().getRefreshVersion()); - Set queryTableIdSets = structInfoMap.getTableMaps(true); - ImmutableList.Builder structInfosBuilder = ImmutableList.builder(); - if (!queryTableIdSets.isEmpty()) { - for (BitSet queryTableIdSet : queryTableIdSets) { - // compare relation id corresponding table id - if (!containsAll(targetTableIdSet, queryTableIdSet)) { - continue; - } - StructInfo structInfo = structInfoMap.getStructInfo(cascadesContext, queryTableIdSet, ownerGroup, - originalPlan, sessionVariable.isEnableMaterializedViewNestRewrite(), true); - if (structInfo != null) { - structInfosBuilder.add(structInfo); - } - } - } - return structInfosBuilder.build(); + return structInfoMap.collectStructInfosByMvBaseTableId(cascadesContext, mvBaseTableIdSet, + originalPlan); } - // if plan doesn't belong to any group, construct it directly - return ImmutableList.of(StructInfo.of(plan, originalPlan, cascadesContext)); + StructInfo structInfo = StructInfo.of(plan, originalPlan, cascadesContext); + return ImmutableList.of(structInfo); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowAggregateRule.java index 7a0dd68692db69..c784dda1ba5892 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowAggregateRule.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PatternCheckResult; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.plans.Plan; @@ -50,8 +51,9 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf */ @Override protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); - return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) + PatternCheckResult checkResult = structInfo.getPlanPatternCheckResult(SUPPORTED_JOIN_TYPE_SET); + PlanCheckContext checkContext = checkResult.getCheckContext(); + return checkResult.isAccepted() && checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow() && checkContext.getTopAggregateNum() <= 1 && checkContext.getTopWindowNum() <= 1 && !checkContext.isWindowUnderAggregate() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowJoinRule.java index 01175d25f78972..f1722de433caaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowJoinRule.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PatternCheckResult; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.plans.Plan; @@ -51,8 +52,9 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf */ @Override protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); - return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) + PatternCheckResult checkResult = structInfo.getPlanPatternCheckResult(SUPPORTED_JOIN_TYPE_SET); + PlanCheckContext checkContext = checkResult.getCheckContext(); + return checkResult.isAccepted() && !checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow() && checkContext.getTopWindowNum() <= 1 && !checkContext.isContainsTopTopN() && !checkContext.isContainsTopLimit(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowScanRule.java index 6df3316887a7a6..2f41c1775d6e28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowScanRule.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PatternCheckResult; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.plans.Plan; @@ -28,7 +29,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import java.util.List; @@ -52,8 +52,9 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf */ @Override protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(ImmutableSet.of()); - return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext) + PatternCheckResult checkResult = structInfo.getScanPatternCheckResult(); + PlanCheckContext checkContext = checkResult.getCheckContext(); + return checkResult.isAccepted() && !checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow() && checkContext.getTopWindowNum() <= 1 && !checkContext.isContainsTopTopN() && !checkContext.isContainsTopLimit(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java index d098c00e40ba67..215246c4fe9383 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java @@ -88,9 +88,9 @@ public static Plan rewrite(CascadesContext cascadesContext) { Pair, MaterializationContext>, BitSet> chosenMaterializationAndUsedTable = MaterializedViewUtils.getChosenMaterializationAndUsedTable(physicalPlan, cascadesContext.getAllMaterializationContexts()); - // Extract logical plan by table id set by the corresponding best physical plan - StructInfo structInfo = root.getStructInfoMap().getStructInfo(cascadesContext, - chosenMaterializationAndUsedTable.value(), root, null, true, false); + // Extract logical plan by relation id set from the corresponding best physical plan. + StructInfo structInfo = root.getStructInfoMap().getStructInfoByRelationIdSet(cascadesContext, + chosenMaterializationAndUsedTable.value(), null); if (structInfo == null) { LOG.error("preMaterializedViewRewriter rewrite structInfo is null, query id is {}", cascadesContext.getConnectContext().getQueryIdentifier()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index a45863aa32c3c7..c37702107b289e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -67,6 +67,7 @@ import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; @@ -135,13 +136,18 @@ public class StructInfo { // this is for building LogicalCompatibilityContext later. private final Map> expressionToShuttledExpressionToMap; - private final List planOutputShuttledExpressions; + // Candidate StructInfo may be built from a reconstructed plan while originalPlan keeps the memo group's output. + // Use this plan to shuttle original output expressions through the candidate tree when rewriting projections. + private final Plan outputLineageSourcePlan; + private List planOutputShuttledExpressions; + private final Map, PatternCheckResult> planPatternCheckResults = new HashMap<>(); + private PatternCheckResult scanPatternCheckResult; /** * The construct method for StructInfo */ - private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGraph, boolean valid, Plan topPlan, - Plan bottomPlan, List relations, + private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGraph, + boolean valid, Plan topPlan, Plan bottomPlan, List relations, Map relationIdStructInfoNodeMap, @Nullable Predicates predicates, Optional groupingId, @@ -149,9 +155,7 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG shuttledExpressionsToExpressionsMap, Map> expressionToShuttledExpressionToMap, BitSet relationIdSet, - SplitPredicate splitPredicate, - EquivalenceClass equivalenceClass, - List planOutputShuttledExpressions) { + Plan outputLineageSourcePlan) { this.originalPlan = originalPlan; this.originalPlanId = originalPlanId; this.hyperGraph = hyperGraph; @@ -163,21 +167,19 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG this.relationIdStructInfoNodeMap = relationIdStructInfoNodeMap; this.predicates = predicates; this.groupingId = groupingId; - this.splitPredicate = splitPredicate; - this.equivalenceClass = equivalenceClass; this.shuttledExpressionsToExpressionsMap = shuttledExpressionsToExpressionsMap; this.expressionToShuttledExpressionToMap = expressionToShuttledExpressionToMap; - this.planOutputShuttledExpressions = planOutputShuttledExpressions; + this.outputLineageSourcePlan = outputLineageSourcePlan; } /** * Construct StructInfo with new predicates */ public StructInfo withPredicates(Predicates predicates) { - return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan, - this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, predicates, this.groupingId, - this.shuttledExpressionsToExpressionsMap, this.expressionToShuttledExpressionToMap, - this.relationBitSet, null, null, this.planOutputShuttledExpressions); + return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, + this.valid, this.topPlan, this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, + predicates, this.groupingId, this.shuttledExpressionsToExpressionsMap, + this.expressionToShuttledExpressionToMap, this.relationBitSet, this.outputLineageSourcePlan); } private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, @@ -329,13 +331,10 @@ public static StructInfo of(Plan derivedPlan, Plan originalPlan, CascadesContext topPlan.accept(PREDICATE_COLLECTOR, predicateCollectorContext); Predicates predicates = Predicates.of(predicateCollectorContext.getCouldPullUpPredicates(), predicateCollectorContext.getCouldNotPullUpPredicates()); - // this should use the output of originalPlan to make sure the output right order - List planOutputShuttledExpressions = - ExpressionUtils.shuttleExpressionWithLineage(originalPlan.getOutput(), originalPlan); return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid, topPlan, bottomPlan, relationList, relationIdStructInfoNodeMap, predicates, planSplitContext.getGroupingId(), shuttledHashConjunctsToConjunctsMap, expressionToShuttledExpressionToMap, - relationBitSet, null, null, planOutputShuttledExpressions); + relationBitSet, derivedPlan); } public List getRelations() { @@ -390,6 +389,35 @@ public Plan getTopPlan() { return topPlan; } + /** + * Get or build the cached plan-pattern check result for the given supported join types. + */ + public PatternCheckResult getPlanPatternCheckResult(Set supportJoinTypes) { + Set cacheKey = ImmutableSet.copyOf(supportJoinTypes); + PatternCheckResult cachedResult = planPatternCheckResults.get(cacheKey); + if (cachedResult != null) { + return cachedResult; + } + PlanCheckContext checkContext = PlanCheckContext.of(cacheKey); + PatternCheckResult result = new PatternCheckResult(topPlan.accept(PLAN_PATTERN_CHECKER, checkContext), + checkContext); + planPatternCheckResults.put(cacheKey, result); + return result; + } + + /** + * Get or build the cached scan-pattern check result for the current top plan. + */ + public PatternCheckResult getScanPatternCheckResult() { + if (scanPatternCheckResult != null) { + return scanPatternCheckResult; + } + PlanCheckContext checkContext = PlanCheckContext.of(ImmutableSet.of()); + scanPatternCheckResult = new PatternCheckResult(topPlan.accept(SCAN_PLAN_PATTERN_CHECKER, checkContext), + checkContext); + return scanPatternCheckResult; + } + public Plan getBottomPlan() { return bottomPlan; } @@ -443,7 +471,12 @@ public BitSet getRelationBitSet() { return relationBitSet; } + /** Lazily derive the query output lineage on top of the current candidate plan. */ public List getPlanOutputShuttledExpressions() { + if (planOutputShuttledExpressions == null) { + planOutputShuttledExpressions = + ExpressionUtils.shuttleExpressionWithLineage(originalPlan.getOutput(), outputLineageSourcePlan); + } return planOutputShuttledExpressions; } @@ -768,6 +801,25 @@ public static PlanCheckContext of(Set supportJoinTypes) { } } + /** The final checker verdict plus the collected top-level shape flags. */ + public static class PatternCheckResult { + private final boolean accepted; + private final PlanCheckContext checkContext; + + public PatternCheckResult(boolean accepted, PlanCheckContext checkContext) { + this.accepted = accepted; + this.checkContext = checkContext; + } + + public boolean isAccepted() { + return accepted; + } + + public PlanCheckContext getCheckContext() { + return checkContext; + } + } + /** * PlanPatternChecker, this is used to check the plan pattern is valid or not */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java index 3952db9b9af160..22027ae2042c36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.rules.exploration.mv.mapping; -import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.common.Pair; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; @@ -28,7 +27,6 @@ import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableBiMap.Builder; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -39,13 +37,9 @@ import java.util.Set; /** - * Relation mapping - * such as query pattern is a1 left join a2 left join b - * view pattern is a1 left join a2 left join b. the mapping will be - * [{a1:a1, a2:a2, b:b}, {a1:a2, a2:a1, b:b}] + * One-to-one mapping set between query relations and view relations. */ public class RelationMapping extends Mapping { - private static final Logger LOG = LogManager.getLogger(RelationMapping.class); private final ImmutableBiMap mappedRelationMap; @@ -62,66 +56,57 @@ public static RelationMapping of(ImmutableBiMap } /** - * Generate mapping according to source and target relation + * Generate all possible RelationMappings by source and target relations. + * + * Every source relation must be mapped. The target side may have more same-table relations + * for query-partial rewrite. */ public static List generate(List sources, List targets, - int relationMappingMaxCount) { - // Construct tmp map, key is the table qualifier, value is the corresponding catalog relations - HashMultimap sourceTableRelationIdMap = HashMultimap.create(); + int maxMappingCount) { + HashMultimap sourceBuckets = HashMultimap.create(); for (CatalogRelation relation : sources) { - sourceTableRelationIdMap.put(getTableIdentifier(relation.getTable()), + sourceBuckets.put(new TableIdentifier(relation.getTable()), MappedRelation.of(relation.getRelationId(), relation)); } - HashMultimap targetTableRelationIdMap = HashMultimap.create(); + HashMultimap targetBuckets = HashMultimap.create(); for (CatalogRelation relation : targets) { - targetTableRelationIdMap.put(getTableIdentifier(relation.getTable()), + targetBuckets.put(new TableIdentifier(relation.getTable()), MappedRelation.of(relation.getRelationId(), relation)); } - Set sourceTableKeySet = sourceTableRelationIdMap.keySet(); List>> mappedRelations = new ArrayList<>(); - for (TableIdentifier tableIdentifier : sourceTableKeySet) { - Set sourceMappedRelations = sourceTableRelationIdMap.get(tableIdentifier); - Set targetMappedRelations = targetTableRelationIdMap.get(tableIdentifier); - if (targetMappedRelations.isEmpty()) { - continue; + for (TableIdentifier relationIdentity : sourceBuckets.keySet()) { + Set sourceMappedRelations = sourceBuckets.get(relationIdentity); + Set targetMappedRelations = targetBuckets.get(relationIdentity); + if (sourceMappedRelations.size() > targetMappedRelations.size()) { + return ImmutableList.of(); } - // if source and target relation appear once, just map them if (targetMappedRelations.size() == 1 && sourceMappedRelations.size() == 1) { + MappedRelation sourceRelation = sourceMappedRelations.iterator().next(); + MappedRelation targetRelation = targetMappedRelations.iterator().next(); ImmutableBiMap.Builder biMapBuilder = ImmutableBiMap.builder(); mappedRelations.add(ImmutableList.of( - biMapBuilder.put(sourceMappedRelations.iterator().next(), - targetMappedRelations.iterator().next()).build())); + biMapBuilder.put(sourceRelation, targetRelation).build())); continue; } - // relation appear more than once, should cartesian them and power set to correct combination - // if query is select * from tableA0, tableA1, materialized view is select * from tableA2, tableA3, - // the relationMappingPowerList in relationMappingPowerList should be bi-direction - // [ - // {tableA0 -> tableA2, tableA1 -> tableA3} - // {tableA0 -> tableA3, tableA1 -> tableA2} - // ] - // query is select * from tableA0, tableA1, tableA4 List> relationMappingPowerList = new ArrayList<>(); List> combinations = getUniquePermutation( sourceMappedRelations.toArray(new MappedRelation[0]), - targetMappedRelations.toArray(new MappedRelation[0]), relationMappingMaxCount); + targetMappedRelations.toArray(new MappedRelation[0]), maxMappingCount); for (Pair combination : combinations) { BiMap combinationBiMap = HashBiMap.create(); MappedRelation[] key = combination.key(); MappedRelation[] value = combination.value(); - int length = Math.min(key.length, value.length); - for (int i = 0; i < length; i++) { + for (int i = 0; i < key.length; i++) { combinationBiMap.put(key[i], value[i]); } relationMappingPowerList.add(combinationBiMap); } mappedRelations.add(relationMappingPowerList); } - // mappedRelations product and merge into each relationMapping - return Lists.cartesianProduct(mappedRelations).stream() - .map(RelationMapping::merge) - .collect(ImmutableList.toImmutableList()); + List relationMappings = new ArrayList<>(); + buildRelationMappings(mappedRelations, 0, new ArrayList<>(), relationMappings, maxMappingCount); + return ImmutableList.copyOf(relationMappings); } public static RelationMapping merge(List> relationMappings) { @@ -132,8 +117,49 @@ public static RelationMapping merge(List> return RelationMapping.of(mappingBuilder.build()); } - private static TableIdentifier getTableIdentifier(TableIf tableIf) { - return new TableIdentifier(tableIf); + /** + * Build full RelationMappings by taking the cartesian product of per-bucket local mappings. + * + * Example: + * mappedRelations = + * [ + * [{q1 -> v1, q2 -> v2}, {q1 -> v2, q2 -> v1}], + * [{q3 -> v3}, {q3 -> v4}] + * ] + * + * The backtracking process does: + * 1. Pick one local mapping from bucket 0. + * 2. Pick one local mapping from bucket 1. + * 3. After all buckets are chosen, merge the current path into one complete RelationMapping. + * + * The example above produces four final mappings: + * - {q1 -> v1, q2 -> v2, q3 -> v3} + * - {q1 -> v1, q2 -> v2, q3 -> v4} + * - {q1 -> v2, q2 -> v1, q3 -> v3} + * - {q1 -> v2, q2 -> v1, q3 -> v4} + */ + private static void buildRelationMappings(List>> mappedRelations, + int offset, List> currentMappings, + List relationMappings, int maxMappingCount) { + if (relationMappings.size() >= maxMappingCount) { + return; + } + if (offset >= mappedRelations.size()) { + // The current path already contains one local mapping from each bucket. + relationMappings.add(merge(currentMappings)); + return; + } + for (BiMap mappedRelation : mappedRelations.get(offset)) { + // Choose one local mapping from the current bucket and continue with the next bucket. + currentMappings.add(mappedRelation); + buildRelationMappings(mappedRelations, offset + 1, currentMappings, relationMappings, + maxMappingCount); + // Backtrack and try the next local mapping in the same bucket. + currentMappings.remove(currentMappings.size() - 1); + if (relationMappings.size() >= maxMappingCount) { + return; + } + } } @Override @@ -195,6 +221,7 @@ public static List> getUniquePermutatio return results; } + /** Pure permutation backtracking; compatibility is filtered by the caller. */ private static void backtrack(MappedRelation[] left, MappedRelation[] right, int index, boolean[] used, MappedRelation[] current, List> results, int maxMappingCount) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java index e0c833f95b14b1..27461d5d0c3d65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java @@ -54,11 +54,6 @@ public Void visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, return null; } StatementContext statementContext = context.getStatementContext(); - // Collect relationId to tableId mapping - Multimap relationIdToTableId = statementContext.getCommonTableIdToRelationIdMap(); - relationIdToTableId.put(statementContext.getTableId(catalogRelation.getTable()).asInt(), - catalogRelation.getRelationId().asInt()); - // Collect table used partition mapping Multimap, Pair>> tableUsedPartitionNameMap = statementContext .getTableUsedPartitionNameMap(); Set tablePartitions = new HashSet<>(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java index 6fbec7f224366a..5ea825b1686405 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java @@ -17,208 +17,125 @@ package org.apache.doris.nereids.memo; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Table; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo; import org.apache.doris.nereids.sqltest.SqlTestBase; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.util.PlanChecker; -import com.google.common.collect.Multimap; +import com.google.common.collect.ImmutableSet; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.BitSet; -import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; class StructInfoMapTest extends SqlTestBase { @Test - void testTableMap() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select T1.id from T1 inner join T2 " - + "on T1.id = T2.id " - + "inner join T3 on T1.id = T3.id", - connectContext - ); - PlanChecker.from(c1) - .analyze() - .rewrite() - .optimize(); - Group root = c1.getMemo().getRoot(); - Set tableMaps = root.getStructInfoMap().getTableMaps(true); - Assertions.assertTrue(tableMaps.isEmpty()); + void testCollectStructInfosByMvBaseTableId() throws Exception { + try { + CascadesContext withoutNestRewriteContext = prepareJoinRewriteContext(false); + Set> withoutNestRewriteRelationNames = collectStructInfos( + withoutNestRewriteContext, "mv1", "T3").stream() + .map(StructInfoMapTest::relationNames) + .collect(Collectors.toSet()); + Assertions.assertFalse(withoutNestRewriteRelationNames.contains(ImmutableSet.of("mv1", "T3"))); + Assertions.assertFalse(withoutNestRewriteRelationNames.stream() + .anyMatch(relationNames -> relationNames.contains("mv1"))); - Multimap commonTableIdToRelationIdMap - = c1.getStatementContext().getCommonTableIdToRelationIdMap(); - BitSet targetBitSet = new BitSet(); - for (Integer tableId : commonTableIdToRelationIdMap.keys()) { - targetBitSet.set(tableId); - } - c1.getMemo().incrementAndGetRefreshVersion(targetBitSet); - int memoVersion = StructInfoMap.getMemoVersion(targetBitSet, c1.getMemo().getRefreshVersion()); - root.getStructInfoMap().refresh(root, c1, targetBitSet, new HashSet<>(), - connectContext.getSessionVariable().enableMaterializedViewNestRewrite, memoVersion, true); - Assertions.assertEquals(1, tableMaps.size()); - installValidRelationManager(); - connectContext.getSessionVariable().enableMaterializedViewRewrite = true; - connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; - connectContext.getSessionVariable().materializedViewRewriteDurationThresholdMs = 1000000; - - dropMvByNereids("drop materialized view if exists mv1"); - createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" - + " DISTRIBUTED BY RANDOM BUCKETS 1\n" - + " PROPERTIES ('replication_num' = '1') \n" - + " as select T1.id from T1 inner join T2 " - + "on T1.id = T2.id;"); - mockCandidateMtmv("mv1"); - c1 = createCascadesContext( - "select T1.id from T1 inner join T2 " - + "on T1.id = T2.id " - + "inner join T3 on T1.id = T3.id", - connectContext - ); - PlanChecker.from(c1) - .setIsQuery() - .analyze() - .rewrite() - .preMvRewrite() - .optimize() - .printlnBestPlanTree(); - root = c1.getMemo().getRoot(); - commonTableIdToRelationIdMap - = c1.getStatementContext().getCommonTableIdToRelationIdMap(); - targetBitSet = new BitSet(); - for (Integer tableId : commonTableIdToRelationIdMap.keys()) { - targetBitSet.set(tableId); + CascadesContext withNestRewriteContext = prepareJoinRewriteContext(true); + Set> baseRelationNames = collectStructInfos(withNestRewriteContext, "T1", "T2", "T3").stream() + .map(StructInfoMapTest::relationNames) + .collect(Collectors.toSet()); + Assertions.assertTrue(baseRelationNames + .contains(ImmutableSet.of("T1", "T2", "T3"))); + Set> nestedMvRelationNames = collectStructInfos(withNestRewriteContext, "mv1", "T3").stream() + .map(StructInfoMapTest::relationNames) + .collect(Collectors.toSet()); + Assertions.assertTrue(nestedMvRelationNames + .contains(ImmutableSet.of("mv1", "T3"))); + } finally { + dropMvByNereids("drop materialized view if exists mv1"); } - c1.getMemo().incrementAndGetRefreshVersion(targetBitSet); - memoVersion = StructInfoMap.getMemoVersion(targetBitSet, c1.getMemo().getRefreshVersion()); - root.getStructInfoMap().refresh(root, c1, targetBitSet, new HashSet<>(), - connectContext.getSessionVariable().enableMaterializedViewNestRewrite, memoVersion, true); - tableMaps = root.getStructInfoMap().getTableMaps(true); - Assertions.assertEquals(2, tableMaps.size()); - dropMvByNereids("drop materialized view mv1"); } @Test - void testLazyRefresh() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select T1.id from T1 inner join T2 " - + "on T1.id = T2.id " - + "inner join T3 on T1.id = T3.id", - connectContext - ); - PlanChecker.from(c1) - .analyze() - .rewrite() - .optimize(); - Group root = c1.getMemo().getRoot(); - Set tableMaps = root.getStructInfoMap().getTableMaps(true); - Assertions.assertTrue(tableMaps.isEmpty()); - root.getStructInfoMap().refresh(root, c1, new BitSet(), new HashSet<>(), - connectContext.getSessionVariable().enableMaterializedViewNestRewrite, 0, true); - root.getStructInfoMap().refresh(root, c1, new BitSet(), new HashSet<>(), - connectContext.getSessionVariable().enableMaterializedViewNestRewrite, 0, true); - Assertions.assertEquals(1, tableMaps.size()); - installValidRelationManager(); - connectContext.getSessionVariable().enableMaterializedViewRewrite = true; - connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; - connectContext.getSessionVariable().materializedViewRewriteDurationThresholdMs = 1000000; - dropMvByNereids("drop materialized view if exists mv1"); - createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" - + " DISTRIBUTED BY RANDOM BUCKETS 1\n" - + " PROPERTIES ('replication_num' = '1') \n" - + " as select T1.id from T1 inner join T2 " - + "on T1.id = T2.id;"); - mockCandidateMtmv("mv1"); - c1 = createCascadesContext( - "select T1.id from T1 inner join T2 " - + "on T1.id = T2.id " - + "inner join T3 on T1.id = T3.id", - connectContext - ); - PlanChecker.from(c1) - .setIsQuery() - .analyze() - .rewrite() - .preMvRewrite() - .optimize() - .printlnBestPlanTree(); - root = c1.getMemo().getRoot(); - Multimap commonTableIdToRelationIdMap - = c1.getStatementContext().getCommonTableIdToRelationIdMap(); - BitSet targetBitSet = new BitSet(); - for (Integer tableId : commonTableIdToRelationIdMap.keys()) { - targetBitSet.set(tableId); + void testGetStructInfoByExactRelationIdSet() throws Exception { + try { + CascadesContext cascadesContext = prepareJoinRewriteContext(true); + StructInfo nestedMvCandidate = collectStructInfos(cascadesContext, "mv1", "T3").stream() + .filter(structInfo -> relationNames(structInfo).equals(ImmutableSet.of("mv1", "T3"))) + .findFirst() + .orElse(null); + Assertions.assertNotNull(nestedMvCandidate); + + Group root = cascadesContext.getMemo().getRoot(); + Plan rootPlan = root.getLogicalExpressions().get(0).getPlan(); + StructInfo exactStructInfo = root.getStructInfoMap().getStructInfoByRelationIdSet( + cascadesContext, nestedMvCandidate.getRelationBitSet(), rootPlan); + Assertions.assertNotNull(exactStructInfo); + Assertions.assertEquals(ImmutableSet.of("mv1", "T3"), relationNames(exactStructInfo)); + + BitSet unknownRelationIdSet = new BitSet(); + unknownRelationIdSet.set(100000); + Assertions.assertNull(root.getStructInfoMap().getStructInfoByRelationIdSet( + cascadesContext, unknownRelationIdSet, rootPlan)); + } finally { + dropMvByNereids("drop materialized view if exists mv1"); } - c1.getMemo().incrementAndGetRefreshVersion(targetBitSet); - int memoVersion = StructInfoMap.getMemoVersion(targetBitSet, c1.getMemo().getRefreshVersion()); - root.getStructInfoMap().refresh(root, c1, targetBitSet, new HashSet<>(), - connectContext.getSessionVariable().enableMaterializedViewNestRewrite, memoVersion, true); - tableMaps = root.getStructInfoMap().getTableMaps(true); - Assertions.assertEquals(2, tableMaps.size()); - dropMvByNereids("drop materialized view mv1"); } - @Test - void testTableChild() throws Exception { + private CascadesContext prepareJoinRewriteContext(boolean enableNestRewrite) throws Exception { connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - CascadesContext c1 = createCascadesContext( - "select T1.id from T1 inner join T2 " - + "on T1.id = T2.id " - + "inner join T3 on T1.id = T3.id", - connectContext - ); installValidRelationManager(); connectContext.getSessionVariable().enableMaterializedViewRewrite = true; - connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; + connectContext.getSessionVariable().enableMaterializedViewNestRewrite = enableNestRewrite; connectContext.getSessionVariable().materializedViewRewriteDurationThresholdMs = 1000000; dropMvByNereids("drop materialized view if exists mv1"); createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + " PROPERTIES ('replication_num' = '1') \n" - + " as select T1.id from T1 inner join T2 " - + "on T1.id = T2.id;"); + + " as select T1.id from T1 inner join T2 on T1.id = T2.id;"); mockCandidateMtmv("mv1"); - c1 = createCascadesContext( - "select T1.id from T1 inner join T2 " - + "on T1.id = T2.id " - + "inner join T3 on T1.id = T3.id", + CascadesContext cascadesContext = createCascadesContext( + "select T1.id from T1 inner join T2 on T1.id = T2.id inner join T3 on T1.id = T3.id", connectContext ); - PlanChecker.from(c1) + PlanChecker.from(cascadesContext) .setIsQuery() .analyze() .rewrite() - .preMvRewrite() .optimize(); - Group root = c1.getMemo().getRoot(); - Multimap commonTableIdToRelationIdMap - = c1.getStatementContext().getCommonTableIdToRelationIdMap(); - BitSet targetBitSet = new BitSet(); - for (Integer tableId : commonTableIdToRelationIdMap.keys()) { - targetBitSet.set(tableId); - } - c1.getMemo().incrementAndGetRefreshVersion(targetBitSet); - int memoVersion = StructInfoMap.getMemoVersion(targetBitSet, c1.getMemo().getRefreshVersion()); - root.getStructInfoMap().refresh(root, c1, targetBitSet, new HashSet<>(), - connectContext.getSessionVariable().enableMaterializedViewNestRewrite, memoVersion, true); - StructInfoMap structInfoMap = root.getStructInfoMap(); - Assertions.assertEquals(2, structInfoMap.getTableMaps(true).size()); - BitSet mvMap = structInfoMap.getTableMaps(true).stream() - .filter(b -> b.cardinality() == 2) - .collect(Collectors.toList()).get(0); - StructInfo structInfo = structInfoMap.getStructInfo(c1, mvMap, root, null, - connectContext.getSessionVariable().enableMaterializedViewNestRewrite, true); - System.out.println(structInfo.getOriginalPlan().treeString()); - BitSet bitSet = new BitSet(); - for (CatalogRelation relation : structInfo.getRelations()) { - bitSet.set(c1.getStatementContext().getTableId(relation.getTable()).asInt()); + return cascadesContext; + } + + private List collectStructInfos(CascadesContext cascadesContext, String... tableNames) { + Group root = cascadesContext.getMemo().getRoot(); + Plan rootPlan = root.getLogicalExpressions().get(0).getPlan(); + return root.getStructInfoMap().collectStructInfosByMvBaseTableId( + cascadesContext, buildTableIdSet(cascadesContext, tableNames), rootPlan); + } + + private BitSet buildTableIdSet(CascadesContext cascadesContext, String... tableNames) { + Database db = (Database) Env.getCurrentEnv().getInternalCatalog().getDbNullable(connectContext.getDatabase()); + BitSet tableIdSet = new BitSet(); + for (String tableName : tableNames) { + Table table = db.getTableNullable(tableName); + tableIdSet.set(cascadesContext.getStatementContext().getTableId(table).asInt()); } - Assertions.assertEquals(bitSet, mvMap); - dropMvByNereids("drop materialized view mv1"); + return tableIdSet; + } + + private static Set relationNames(StructInfo structInfo) { + return structInfo.getRelations().stream() + .map(CatalogRelation::getTable) + .map(table -> table.getName()) + .collect(Collectors.toSet()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PointQueryShouldNotMvRewriteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PointQueryShouldNotMvRewriteTest.java index ea8271c4f8666a..a27f0e2f260c6e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PointQueryShouldNotMvRewriteTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PointQueryShouldNotMvRewriteTest.java @@ -22,8 +22,8 @@ import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.mtmv.MTMVRelationManager; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.sqltest.SqlTestBase; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; @@ -33,8 +33,6 @@ import org.mockito.Mockito; import java.util.BitSet; -import java.util.HashSet; -import java.util.Set; /** * Test that point query should not rewrite to materialized view. @@ -70,16 +68,14 @@ void testShouldNotMvRewriteWhenPointQuery() throws Exception { // set ShortCircuitQuery to true, consider is point query c1.getStatementContext().setShortCircuitQuery(true); - PlanChecker.from(c1) + boolean containsMtmvScan = PlanChecker.from(c1) .analyze() .rewrite() .optimize() - .printlnBestPlanTree(); - Group root = c1.getMemo().getRoot(); - root.getStructInfoMap().refresh(root, c1, new BitSet(), new HashSet<>(), false, 0, - false); - Set tableMaps = root.getStructInfoMap().getTableMaps(false); - Assertions.assertEquals(1, tableMaps.size()); + .getBestPlanTree() + .collect(CatalogRelation.class::isInstance).stream() + .anyMatch(relation -> relation.getTable() instanceof MTMV); + Assertions.assertFalse(containsMtmvScan); dropMvByNereids("drop materialized view mv1"); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java index 6dc5190da3deb5..6d3b8690365cc5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java @@ -2963,9 +2963,8 @@ private void checkIfEquals(String originalSql, List equivalentSqlList) { // extract plan from memo and check is equals or not Memo memo = cascadesContext.getMemo(); for (Map.Entry planEntry : bitSetLogicalPlanMap.entrySet()) { - memo.incrementAndGetRefreshVersion(planEntry.getKey()); - StructInfo structInfo = memo.getRoot().getStructInfoMap().getStructInfo(cascadesContext, - planEntry.getKey(), memo.getRoot(), null, true, false); + StructInfo structInfo = memo.getRoot().getStructInfoMap().getStructInfoByRelationIdSet(cascadesContext, + planEntry.getKey(), null); Assertions.assertNotNull(structInfo); Assertions.assertTrue(structInfo.getOriginalPlan().deepEquals(planEntry.getValue())); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java index 509acfba207339..d11c6b50d7ffc4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java @@ -207,20 +207,7 @@ public void testGenerateMapping3() { List generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations, 8); Assertions.assertNotNull(generateRelationMapping); - Assertions.assertEquals(1, generateRelationMapping.size()); - - // expected slot mapping - BiMap expectedSlotMapping = HashBiMap.create(); - expectedSlotMapping.put(new ExprId(0), new ExprId(2)); - expectedSlotMapping.put(new ExprId(1), new ExprId(3)); - expectedSlotMapping.put(new ExprId(2), new ExprId(4)); - expectedSlotMapping.put(new ExprId(5), new ExprId(0)); - expectedSlotMapping.put(new ExprId(6), new ExprId(1)); - // expected relation mapping - BiMap expectedRelationMapping = HashBiMap.create(); - expectedRelationMapping.put(new RelationId(0), new RelationId(1)); - expectedRelationMapping.put(new RelationId(2), new RelationId(0)); - assertRelationMapping(generateRelationMapping.get(0), expectedRelationMapping, expectedSlotMapping); + Assertions.assertTrue(generateRelationMapping.isEmpty()); } // test table of source query is repeated @@ -254,33 +241,7 @@ public void testGenerateMapping4() { List generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations, 8); Assertions.assertNotNull(generateRelationMapping); - Assertions.assertEquals(2, generateRelationMapping.size()); - - // expected slot mapping - BiMap expectedSlotMapping = HashBiMap.create(); - expectedSlotMapping.put(new ExprId(0), new ExprId(2)); - expectedSlotMapping.put(new ExprId(1), new ExprId(3)); - expectedSlotMapping.put(new ExprId(2), new ExprId(4)); - expectedSlotMapping.put(new ExprId(3), new ExprId(0)); - expectedSlotMapping.put(new ExprId(4), new ExprId(1)); - // expected relation mapping - BiMap expectedRelationMapping = HashBiMap.create(); - expectedRelationMapping.put(new RelationId(0), new RelationId(1)); - expectedRelationMapping.put(new RelationId(1), new RelationId(0)); - assertRelationMapping(generateRelationMapping.get(0), expectedRelationMapping, expectedSlotMapping); - - // expected slot mapping - expectedSlotMapping = HashBiMap.create(); - expectedSlotMapping.put(new ExprId(0), new ExprId(2)); - expectedSlotMapping.put(new ExprId(1), new ExprId(3)); - expectedSlotMapping.put(new ExprId(2), new ExprId(4)); - expectedSlotMapping.put(new ExprId(5), new ExprId(0)); - expectedSlotMapping.put(new ExprId(6), new ExprId(1)); - // expected relation mapping - expectedRelationMapping = HashBiMap.create(); - expectedRelationMapping.put(new RelationId(0), new RelationId(1)); - expectedRelationMapping.put(new RelationId(2), new RelationId(0)); - assertRelationMapping(generateRelationMapping.get(1), expectedRelationMapping, expectedSlotMapping); + Assertions.assertTrue(generateRelationMapping.isEmpty()); } @Test @@ -557,53 +518,7 @@ public void testGenerateMapping8() { List generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations, 8); Assertions.assertNotNull(generateRelationMapping); - Assertions.assertEquals(6, generateRelationMapping.size()); - - // expected table relation mapping is as following - // (1, 0), (2, 2), (0, 1) - // (1, 0), (3, 2), (0, 1) - // (2, 0), (1, 2), (0, 1) - // (3, 0), (2, 2), (0, 1) - // (2, 0), (3, 2), (0, 1) - // (3, 0), (1, 2), (0, 1) - Set> expectedRelationMappingSet = new HashSet<>(); - BiMap expectedRelationMapping = HashBiMap.create(); - expectedRelationMapping.put(new RelationId(1), new RelationId(0)); - expectedRelationMapping.put(new RelationId(2), new RelationId(2)); - expectedRelationMapping.put(new RelationId(0), new RelationId(1)); - expectedRelationMappingSet.add(expectedRelationMapping); - - expectedRelationMapping = HashBiMap.create(); - expectedRelationMapping.put(new RelationId(1), new RelationId(0)); - expectedRelationMapping.put(new RelationId(3), new RelationId(2)); - expectedRelationMapping.put(new RelationId(0), new RelationId(1)); - expectedRelationMappingSet.add(expectedRelationMapping); - - expectedRelationMapping = HashBiMap.create(); - expectedRelationMapping.put(new RelationId(2), new RelationId(0)); - expectedRelationMapping.put(new RelationId(1), new RelationId(2)); - expectedRelationMapping.put(new RelationId(0), new RelationId(1)); - expectedRelationMappingSet.add(expectedRelationMapping); - - expectedRelationMapping = HashBiMap.create(); - expectedRelationMapping.put(new RelationId(3), new RelationId(0)); - expectedRelationMapping.put(new RelationId(2), new RelationId(2)); - expectedRelationMapping.put(new RelationId(0), new RelationId(1)); - expectedRelationMappingSet.add(expectedRelationMapping); - - expectedRelationMapping = HashBiMap.create(); - expectedRelationMapping.put(new RelationId(2), new RelationId(0)); - expectedRelationMapping.put(new RelationId(3), new RelationId(2)); - expectedRelationMapping.put(new RelationId(0), new RelationId(1)); - expectedRelationMappingSet.add(expectedRelationMapping); - - expectedRelationMapping = HashBiMap.create(); - expectedRelationMapping.put(new RelationId(3), new RelationId(0)); - expectedRelationMapping.put(new RelationId(1), new RelationId(2)); - expectedRelationMapping.put(new RelationId(0), new RelationId(1)); - expectedRelationMappingSet.add(expectedRelationMapping); - - assertRelationMapping(new HashSet<>(generateRelationMapping), expectedRelationMappingSet); + Assertions.assertTrue(generateRelationMapping.isEmpty()); } private void assertRelationMapping(RelationMapping relationMapping, diff --git a/regression-test/data/mtmv_p0/test_common_tableid_relationid_rewrite.out b/regression-test/data/mtmv_p0/test_common_tableid_relationid_rewrite.out new file mode 100644 index 00000000000000..9476467f5f4603 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_common_tableid_relationid_rewrite.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !common_tableid_relationid_result -- +2026-02-04 K1 D1 M diff --git a/regression-test/suites/mtmv_p0/test_common_tableid_relationid_rewrite.groovy b/regression-test/suites/mtmv_p0/test_common_tableid_relationid_rewrite.groovy new file mode 100644 index 00000000000000..7a14fc3ad57049 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_common_tableid_relationid_rewrite.groovy @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_common_tableid_relationid_rewrite", "mtmv") { + // verify nested MV rewrite can choose the parent MV when the query starts from + // base tables and the MV definition is built from already-rewritten child MVs. + String suiteName = "test_common_tableid_relationid_rewrite" + String dbName = context.config.getDbNameByFile(context.file) + String factTable = "${suiteName}_fact_src" + String dimTable = "${suiteName}_dim_full" + String dimView = "${suiteName}_v_dim_full_non_double" + String factMv = "${suiteName}_mv_fact" + String dimMv = "${suiteName}_mv_dim_full" + String dimViewMv = "${suiteName}_mv_dim_full_view_non_double" + String targetMv = "${suiteName}_mv_target" + + sql "use ${dbName}" + sql "set enable_nereids_planner = true" + sql "set enable_fallback_to_original_planner = false" + sql "set enable_materialized_view_rewrite = true" + sql "set enable_materialized_view_nest_rewrite = true" + sql "set enable_nereids_timeout = false" + sql "set materialized_view_rewrite_duration_threshold_ms = 1800000" + + sql """drop materialized view if exists ${targetMv}""" + sql """drop materialized view if exists ${dimViewMv}""" + sql """drop materialized view if exists ${dimMv}""" + sql """drop materialized view if exists ${factMv}""" + sql """drop view if exists ${dimView}""" + sql """drop table if exists ${dimTable}""" + sql """drop table if exists ${factTable}""" + + sql """ + create table ${factTable} ( + dt date not null, + k varchar(32) not null, + is_dyn varchar(8), + sku_type varchar(8) + ) duplicate key(dt, k) + partition by range(dt) ( + partition p1 values less than ('2026-02-05') + ) + distributed by hash(k) buckets 1 + properties ( + "replication_num" = "1" + ) + """ + + sql """ + create table ${dimTable} ( + dt date not null, + k varchar(32) not null, + sku_type varchar(8), + is_dyn varchar(8), + bu varchar(32), + mode_flag varchar(8), + double_flag varchar(8) + ) unique key(dt, k, sku_type, is_dyn) + partition by range(dt) ( + partition p1 values less than ('2026-02-05') + ) + distributed by hash(k) buckets 1 + properties ( + "replication_num" = "1" + ) + """ + + sql """ + create view ${dimView} as + select dt, k, mode_flag, sku_type + from ${dimTable} + where double_flag = '0' + """ + + sql """ + insert into ${factTable} values ('2026-02-04', 'K1', '0', '1') + """ + sql """ + insert into ${dimTable} values ('2026-02-04', 'K1', '1', '0', 'D1', 'M', '0') + """ + + sql """ + create materialized view ${factMv} + build deferred refresh complete on manual + partition by (dt) + distributed by hash(k) buckets 1 + properties ("replication_num" = "1") + as + select dt, k, is_dyn, sku_type + from ${factTable} + where sku_type = '1' + """ + sql """refresh materialized view ${factMv} complete""" + waitingMTMVTaskFinishedByMvName(factMv) + + sql """ + create materialized view ${dimMv} + build deferred refresh complete on manual + partition by (dt) + distributed by hash(k) buckets 1 + properties ("replication_num" = "1") + as + select dt, k, bu, is_dyn, sku_type + from ${dimTable} + """ + sql """refresh materialized view ${dimMv} complete""" + waitingMTMVTaskFinishedByMvName(dimMv) + + sql """ + create materialized view ${dimViewMv} + build deferred refresh complete on manual + partition by (dt) + distributed by hash(k) buckets 1 + properties ("replication_num" = "1") + as + select dt, k, mode_flag, sku_type + from ${dimView} + """ + sql """refresh materialized view ${dimViewMv} complete""" + waitingMTMVTaskFinishedByMvName(dimViewMv) + + sql """ + create materialized view ${targetMv} + build deferred refresh complete on manual + partition by (dt) + distributed by hash(k) buckets 1 + properties ("replication_num" = "1") + as + select + t.dt, + t.k, + d0.bu as out_bu, + d1.mode_flag as out_mode + from ${factMv} t + left join ${dimMv} d0 + on t.dt = d0.dt + and t.k = d0.k + and t.sku_type = d0.sku_type + and t.is_dyn = d0.is_dyn + left join ${dimViewMv} d1 + on t.dt = d1.dt + and t.k = d1.k + and t.sku_type = d1.sku_type + """ + sql """refresh materialized view ${targetMv} complete""" + waitingMTMVTaskFinishedByMvName(targetMv) + + String querySql = """ + select + t.dt, + t.k, + d0.bu as out_bu, + d1.mode_flag as out_mode + from ${factTable} t + left join ${dimTable} d0 + on t.dt = d0.dt + and t.k = d0.k + and t.sku_type = d0.sku_type + and t.is_dyn = d0.is_dyn + left join ${dimView} d1 + on t.dt = d1.dt + and t.k = d1.k + and t.sku_type = d1.sku_type + where t.dt = '2026-02-04' + and t.sku_type = '1' + order by t.k + """ + + // First check rewrite success, then require the target MV itself to be chosen. + mv_rewrite_success_without_check_chosen(querySql, targetMv) + explain { + sql("${querySql}") + contains(".${targetMv} chose") + } + order_qt_common_tableid_relationid_result "${querySql}" +}