Skip to content

HIVE-29734: Cleanup Bug.java after Calcite upgrade 1.42.0 #6720

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rubenada wants to merge 12 commits into
base: master
Choose a base branch
from
Open

HIVE-29734: Cleanup Bug.java after Calcite upgrade 1.42.0 #6720

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
4005db3
HIVE-29734: Cleanup Bug.java after Calcite upgrade 1.42.0
rubenada Aug 24, 2026
759753e
Put back HivelRelBuilder withPruneInputOfAggregate(false) due to test…
rubenada Aug 25, 2026
398a1ee
More cleanup
rubenada Aug 25, 2026
4d5100e
Minor changes to fix sonar issues
rubenada Aug 25, 2026
8ef4244
Put back HiveFilterTableFunctionTransposeRule code, since it does not…
rubenada Aug 25, 2026
5dedbc1
More cleanup
rubenada Aug 26, 2026
a6e77f0
More cleanup
rubenada Aug 26, 2026
c38aa7f
Fix LoptOptimizeJoinRule (use the correct RelBuilderFactory)
rubenada Aug 26, 2026
d45e33c
Handle TODOs
rubenada Aug 27, 2026
21e8036
Further cleanup on HiveRemoveEmptySingleRules
rubenada Aug 28, 2026
7a2c844
Delete HiveRelJson
rubenada Aug 28, 2026
30ff3e9
Delete HiveTypeFactory
rubenada Aug 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion data/conf/hive-log4j2.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ logger.CalcitePlanner.name = org.apache.calcite.plan.RelOptPlanner
logger.CalcitePlanner.level = INFO

# Change logger level to DEBUG, to see all application of CBO rules and the operators they produce
logger.CBORuleLogger.name = org.apache.hadoop.hive.ql.optimizer.calcite.RuleEventLogger
logger.CBORuleLogger.name = org.apache.calcite.plan.RuleEventLogger

@zabetak zabetak Aug 28, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you verify that the logs are displayed as expected after the change?

logger.CBORuleLogger.level = OFF
logger.CBORuleLogger.filter.marker.type = MarkerFilter
logger.CBORuleLogger.filter.marker.marker = FULL_PLAN
Expand Down
78 changes: 3 additions & 75 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,83 +41,11 @@ public final class Bug {
* CALCITE-4166</a> is fixed.
*/
public static final boolean CALCITE_4166_FIXED = false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-4499">issue
* CALCITE-4499</a> is fixed.
*/
public static final boolean CALCITE_4499_FIXED = false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-4200">issue
* CALCITE-4200</a> is fixed.
*/
public static final boolean CALCITE_4200_FIXED = false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-4513">issue
* CALCITE-4513</a> is fixed.
*/
public static final boolean CALCITE_4513_FIXED=false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-4574">CALCITE-4574</a> is fixed.
*/
public static final boolean CALCITE_4574_FIXED = false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-4704">CALCITE-4704</a> is fixed.
*/
public static final boolean CALCITE_4704_FIXED = false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-5293">CALCITE-5293</a> is fixed.
*/
public static final boolean CALCITE_5293_FIXED = false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-5294">CALCITE-5294</a> is fixed.
*/
public static final boolean CALCITE_5294_FIXED = false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-5337">CALCITE-5337</a> is fixed.
*/
public static final boolean CALCITE_5337_FIXED = false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-5669">CALCITE-5669</a> is fixed.
*/
public static final boolean CALCITE_5669_FIXED = false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-5669">CALCITE-5985</a> is fixed.
*/
public static final boolean CALCITE_5985_FIXED = false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-6513">CALCITE-6513</a> is fixed.
*/
public static final boolean CALCITE_6513_FIXED = false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-6704">CALCITE-6704</a> is fixed.
*/
public static final boolean CALCITE_6704_FIXED = false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-6513">CALCITE-6737</a> is fixed.
*/
public static final boolean CALCITE_6737_FIXED = false;

/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-6954">CALCITE-6954</a> is fixed.
*/
public static final boolean CALCITE_6954_FIXED = false;
/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-6832">CALCITE-6832</a> is fixed.
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-6391">issue
* CALCITE-6391</a> is fixed.
*/
public static final boolean CALCITE_6832_FIXED = false;
public static final boolean CALCITE_6391_FIXED = false;
/**
* Whether <a href="https://issues.apache.org/jira/browse/CALCITE-7641">CALCITE-7641</a> is fixed.
*/
Expand Down
102 changes: 1 addition & 101 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,13 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite;

import com.google.common.collect.Iterables;
import org.apache.calcite.avatica.util.TimeUnitRange;
import org.apache.calcite.plan.Context;
import org.apache.calcite.plan.Contexts;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptSchema;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.CorrelationId;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCorrelVariable;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.calcite.server.CalciteServerStatement;
Expand All @@ -44,14 +34,8 @@
import org.apache.calcite.tools.Frameworks;
import org.apache.calcite.tools.RelBuilder;
import org.apache.calcite.tools.RelBuilderFactory;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Litmus;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFloorDate;

import java.util.ArrayList;
import java.util.List;
import java.util.Set;

/**
* Builder for relational expressions in Hive.
*
Expand Down Expand Up @@ -90,7 +74,7 @@ public static RelBuilderFactory proto(final Context context) {
return new RelBuilderFactory() {
@Override
public RelBuilder create(RelOptCluster cluster, RelOptSchema schema) {
Context confContext = Contexts.of(Config.DEFAULT.withPruneInputOfAggregate(Bug.CALCITE_4513_FIXED)
Context confContext = Contexts.of(Config.DEFAULT.withPruneInputOfAggregate(Bug.CALCITE_6391_FIXED)
.withSimplifyValues(false)); // disabled to avoid simplifications that can create non-empty HiveValues
return new HiveRelBuilder(Contexts.chain(context, confContext), cluster, schema);
}
Expand Down Expand Up @@ -133,88 +117,4 @@ public static SqlFunction getFloorSqlFunction(TimeUnitRange flag) {
}
return SqlStdOperatorTable.FLOOR;
}

/** Creates a {@link Join} with correlating variables. */
@Override
public RelBuilder join(JoinRelType joinType, RexNode condition,
Set<CorrelationId> variablesSet) {
if (Bug.CALCITE_4574_FIXED) {
throw new IllegalStateException("Method overriding should be removed once CALCITE-4574 is fixed");
}
RelNode right = this.peek(0);
RelNode left = this.peek(1);
final boolean correlate = variablesSet.size() == 1;
RexNode postCondition = literal(true);
if (correlate) {
final CorrelationId id = Iterables.getOnlyElement(variablesSet);
if (!RelOptUtil.notContainsCorrelation(left, id, Litmus.IGNORE)) {
throw new IllegalArgumentException("variable " + id
+ " must not be used by left input to correlation");
}
// Correlate does not have an ON clause.
switch (joinType) {
case LEFT:
case SEMI:
case ANTI:
// For a LEFT/SEMI/ANTI, predicate must be evaluated first.
filter(condition.accept(new Shifter(left, id, right)));
right = this.peek(0);
break;
case INNER:
// For INNER, we can defer.
postCondition = condition;
break;
default:
throw new IllegalArgumentException("Correlated " + joinType + " join is not supported");
}
final ImmutableBitSet requiredColumns = RelOptUtil.correlationColumns(id, right);
List<RexNode> leftFields = this.fields(2, 0);
List<RexNode> requiredFields = new ArrayList<>();
for (int i = 0; i < leftFields.size(); i++) {
if (requiredColumns.get(i)) {
requiredFields.add(leftFields.get(i));
}
}
correlate(joinType, id, requiredFields);
filter(postCondition);
} else {
// When there is no correlation use the default logic which works OK for now
// Cannot copy-paste the respective code here cause we don't have access to stack,
// Frame etc. and we might lose existing aliases in the builder
assert variablesSet.isEmpty();
super.join(joinType,condition, variablesSet);
}
return this;
}

/** Shuttle that shifts a predicate's inputs to the left, replacing early
* ones with references to a
* {@link RexCorrelVariable}. */
private class Shifter extends RexShuttle {
private final RelNode left;
private final CorrelationId id;
private final RelNode right;

Shifter(RelNode left, CorrelationId id, RelNode right) {
this.left = left;
this.id = id;
this.right = right;
if (Bug.CALCITE_4574_FIXED) {
throw new IllegalStateException("Class should be redundant once CALCITE-4574 is fixed");
}
}

public RexNode visitInputRef(RexInputRef inputRef) {
final RelDataType leftRowType = left.getRowType();
final RexBuilder rexBuilder = getRexBuilder();
final int leftCount = leftRowType.getFieldCount();
if (inputRef.getIndex() < leftCount) {
final RexNode v = rexBuilder.makeCorrel(leftRowType, id);
return rexBuilder.makeFieldAccess(v, inputRef.getIndex());
} else {
return rexBuilder.makeInputRef(right, inputRef.getIndex() - leftCount);
}
}
}

}
107 changes: 0 additions & 107 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelJson.java
View file
Open in desktop

This file was deleted.

3 changes: 2 additions & 1 deletion ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelJsonImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.util.List;
import java.util.Map;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.externalize.RelJson;
import org.apache.calcite.rel.externalize.RelJsonWriter;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.util.ImmutableBitSet;
Expand All @@ -47,7 +48,7 @@ public HiveRelJsonImpl() {
try {
final Field fieldRelJson = RelJsonWriter.class.getDeclaredField("relJson");
fieldRelJson.setAccessible(true);
fieldRelJson.set(this, new HiveRelJson(jsonBuilder));
fieldRelJson.set(this, new RelJson(jsonBuilder));
} catch (IllegalAccessException | NoSuchFieldException e) {
throw new RuntimeException(e);
}
Expand Down
14 changes: 0 additions & 14 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -728,20 +728,6 @@ public static PKFKJoinInfo extractPKFKJoin(
return cannotExtract;
}

public static RexNode pushPastProjectUnlessBloat(RexNode node, Project project, int bloat) {
if (Bug.CALCITE_6513_FIXED) {
throw new IllegalStateException("Method is redundant when the fix for CALCITE-6513 is merged into Calcite. " +
"Use RelOptUtil.pushPastProjectUnlessBloat");
}

List<RexNode> newConditions =
RelOptUtil.pushPastProjectUnlessBloat(Collections.singletonList(node), project, bloat);
if (newConditions == null || newConditions.size() != 1) {
return null;
}
return newConditions.get(0);
}

public static class PKFKJoinInfo {
public final boolean isPkFkJoin;
public final Pair<ImmutableBitSet, ImmutableBitSet> pkFkJoinColumns;
Expand Down
3 changes: 2 additions & 1 deletion ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexJsonBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite;

import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexNode;
Expand All @@ -29,7 +30,7 @@
*/
class HiveRexJsonBuilder extends RexBuilder {
HiveRexJsonBuilder() {
super(new HiveTypeFactory());
super(new JavaTypeFactoryImpl(new HiveTypeSystemImpl()));
}

@Override
Expand Down
Loading
Loading