diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableAnalyzer.java index 49e5b5020f07..d98de83bf596 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableAnalyzer.java @@ -137,6 +137,11 @@ protected boolean genResolvedParseTree(ASTNode ast, PlannerContext plannerCtx) return analyzeAndResolveChildTree(child, plannerCtx); } + @Override + public CreateTableDesc getCreatedTableDesc() { + return getQB().getTableDesc(); + } + /** * Checks to see if given partition columns has DEFAULT or CHECK constraints (whether ENABLED or DISABLED) * Or has NOT NULL constraints (only ENABLED) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 986dcb7fcbbb..cae7e29e6483 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -69,7 +69,9 @@ import org.apache.hadoop.hive.ql.cache.results.CacheUsage; import org.apache.hadoop.hive.ql.ddl.DDLDesc.DDLDescWithWriteId; import org.apache.hadoop.hive.ql.ddl.table.constraint.ConstraintsUtils; +import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableDesc; import org.apache.hadoop.hive.ql.exec.FetchTask; +import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -2174,4 +2176,73 @@ protected void setSqlKind(SqlKind sqlKind) { queryState.setSqlKind(sqlKind); } } + + /** + * Returns the sink operator for query plans that produce one. + * @return the sink operator, or throws if not supported + */ + public Operator getSinkOp() { + throw new UnsupportedOperationException( + "getSinkOp not supported for " + getClass().getSimpleName()); + } + + /** + * Accepts CTE context from another analyzer for CTE materialization. + * @param aliasToCTEs the CTE alias to clause mapping to import + */ + public void acceptCTEContext(Map aliasToCTEs) { + throw new UnsupportedOperationException( + "acceptCTEContext not supported for " + getClass().getSimpleName()); + } + + /** + * Returns the table descriptor created during analysis (e.g., for CTAS or CTE materialization). + * @return the created table descriptor, or throws if not supported + */ + public CreateTableDesc getCreatedTableDesc() { + throw new UnsupportedOperationException( + "getCreatedTableDesc not supported for " + getClass().getSimpleName()); + } + + static class CTEClause { + CTEClause(String alias, ASTNode cteNode, ASTNode withColList) { + this.alias = alias; + this.cteNode = cteNode; + this.withColList = withColList; + } + String alias; + ASTNode cteNode; + ASTNode withColList; + boolean materialize; + int reference; + QBExpr qbExpr; + List parents = new ArrayList(); + + // materialized + BaseSemanticAnalyzer source; + + List> getTasks() { + return source == null ? null : source.rootTasks; + } + + List asExecutionOrder() { + List execution = new ArrayList(); + asExecutionOrder(new HashSet(), execution); + return execution; + } + + void asExecutionOrder(Set visited, List execution) { + for (CTEClause parent : parents) { + if (visited.add(parent)) { + parent.asExecutionOrder(visited, execution); + } + } + execution.add(this); + } + + @Override + public String toString() { + return alias == null ? "" : alias; + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index d5c683daa303..fbc32eaa358e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -31,7 +31,6 @@ import java.util.Optional; import java.util.function.Function; import java.util.regex.Pattern; -import org.antlr.runtime.ClassicToken; import org.antlr.runtime.CommonToken; import org.antlr.runtime.tree.Tree; import org.antlr.runtime.tree.TreeVisitor; @@ -144,7 +143,6 @@ import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.QueryState; -import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableAnalyzer; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FunctionInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; @@ -319,7 +317,6 @@ import org.apache.hadoop.hive.ql.parse.type.TypeCheckProcFactory; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.plan.SelectDesc; import org.apache.hadoop.hive.ql.plan.mapper.EmptyStatsSource; import org.apache.hadoop.hive.ql.plan.mapper.StatsSource; @@ -338,7 +335,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.joda.time.Interval; -import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.math.BigDecimal; @@ -1051,51 +1047,6 @@ boolean continueJoinMerge() { return !(runCBO && disableSemJoinReordering); } - @Override - Table materializeCTE(String cteName, CTEClause cte) throws HiveException { - - ASTNode createTable = new ASTNode(new ClassicToken(HiveParser.TOK_CREATETABLE)); - - ASTNode tableName = new ASTNode(new ClassicToken(HiveParser.TOK_TABNAME)); - tableName.addChild(new ASTNode(new ClassicToken(HiveParser.Identifier, cteName))); - - ASTNode temporary = new ASTNode(new ClassicToken(HiveParser.KW_TEMPORARY, MATERIALIZATION_MARKER)); - - createTable.addChild(tableName); - createTable.addChild(temporary); - createTable.addChild(cte.cteNode); - - CreateTableAnalyzer analyzer = new CreateTableAnalyzer(queryState); - analyzer.initCtx(ctx); - analyzer.init(false); - - // should share cte contexts - analyzer.aliasToCTEs.putAll(aliasToCTEs); - - HiveOperation operation = queryState.getHiveOperation(); - try { - analyzer.analyzeInternal(createTable); - } finally { - queryState.setCommandType(operation); - } - - Table table = analyzer.tableDesc.toTable(conf); - Path location = table.getDataLocation(); - try { - location.getFileSystem(conf).mkdirs(location); - } catch (IOException e) { - throw new HiveException(e); - } - table.setMaterializedTable(true); - - LOG.info(cteName + " will be materialized into " + location); - cte.source = analyzer; - - ctx.addMaterializedTable(cteName, table, getMaterializedTableStats(analyzer.getSinkOp())); - - return table; - } - @Override String fixCtasColumnName(String colName) { if (runCBO) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 5c4f049f0350..e5a1c541f6f5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -117,6 +117,7 @@ import org.apache.hadoop.hive.ql.ddl.DDLDescWithTableProperties; import org.apache.hadoop.hive.ql.ddl.DDLWork; import org.apache.hadoop.hive.ql.ddl.misc.hooks.InsertCommitHookDesc; +import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory; import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableDesc; import org.apache.hadoop.hive.ql.ddl.table.misc.preinsert.PreInsertTableDesc; import org.apache.hadoop.hive.ql.ddl.table.misc.properties.AlterTableUnsetPropertiesDesc; @@ -1439,47 +1440,6 @@ public Set getAllOutputs() { return writeEntities; } - class CTEClause { - CTEClause(String alias, ASTNode cteNode, ASTNode withColList) { - this.alias = alias; - this.cteNode = cteNode; - this.withColList = withColList; - } - String alias; - ASTNode cteNode; - ASTNode withColList; - boolean materialize; - int reference; - QBExpr qbExpr; - List parents = new ArrayList(); - - // materialized - SemanticAnalyzer source; - - List> getTasks() { - return source == null ? null : source.rootTasks; - } - - List asExecutionOrder() { - List execution = new ArrayList(); - asExecutionOrder(new HashSet(), execution); - return execution; - } - - void asExecutionOrder(Set visited, List execution) { - for (CTEClause parent : parents) { - if (visited.add(parent)) { - parent.asExecutionOrder(visited, execution); - } - } - execution.add(this); - } - - @Override - public String toString() { - return alias == null ? "" : alias; - } - } private List> getRealTasks(CTEClause cte) { if (cte == rootClause) { @@ -1568,12 +1528,12 @@ Table materializeCTE(String cteName, CTEClause cte) throws HiveException { createTable.addChild(temporary); createTable.addChild(cte.cteNode); - SemanticAnalyzer analyzer = new SemanticAnalyzer(queryState); + BaseSemanticAnalyzer analyzer = DDLSemanticAnalyzerFactory.getAnalyzer(createTable, queryState); analyzer.initCtx(ctx); analyzer.init(false); // should share cte contexts - analyzer.aliasToCTEs.putAll(aliasToCTEs); + analyzer.acceptCTEContext(aliasToCTEs); HiveOperation operation = queryState.getHiveOperation(); try { @@ -1582,7 +1542,7 @@ Table materializeCTE(String cteName, CTEClause cte) throws HiveException { queryState.setCommandType(operation); } - Table table = analyzer.tableDesc.toTable(conf); + Table table = analyzer.getCreatedTableDesc().toTable(conf); Path location = table.getDataLocation(); try { location.getFileSystem(conf).mkdirs(location); @@ -15457,10 +15417,16 @@ public Map getViewAliasToInput() { return viewAliasToInput; } + @Override public Operator getSinkOp() { return sinkOp; } + @Override + public void acceptCTEContext(Map aliasToCTEs) { + this.aliasToCTEs.putAll(aliasToCTEs); + } + protected enum MaterializationRebuildMode { NONE, INSERT_OVERWRITE_REBUILD, diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java index dbdc79769dc8..d0695ff4067b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java @@ -20,12 +20,15 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -46,6 +49,8 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory; +import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableAnalyzer; import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.QueryProperties.QueryType; import org.apache.hadoop.hive.ql.QueryState; @@ -65,6 +70,7 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.mockito.MockedStatic; import org.mockito.stubbing.Answer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -493,4 +499,51 @@ private void checkTablesUsed(String query, Set tables) throws Exception Assert.assertEquals(new TreeSet<>(tables), new TreeSet<>(result)); } + + @Test + public void testMaterializeCTEWithCBODisabled() throws Exception { + testMaterializeCTEUsesDDLFactory(false); + } + + @Test + public void testMaterializeCTEWithCBOEnabled() throws Exception { + testMaterializeCTEUsesDDLFactory(true); + } + + private void testMaterializeCTEUsesDDLFactory(boolean cboEnabled) throws Exception { + HiveConf testConf = new HiveConf(conf); + testConf.setBoolVar(HiveConf.ConfVars.HIVE_CBO_ENABLED, cboEnabled); + + SessionState.start(testConf); + Context ctx = new Context(testConf); + + // Reference CTE 3 times to exceed default materialization threshold of 2 + String query = "WITH cte AS (SELECT COUNT(*) as cnt FROM table1) " + + "SELECT * FROM cte UNION ALL SELECT * FROM cte UNION ALL SELECT * FROM cte"; + + ASTNode astNode = ParseUtils.parse(query, ctx); + QueryState queryState = new QueryState.Builder().withHiveConf(testConf).build(); + BaseSemanticAnalyzer analyzer = SemanticAnalyzerFactory.get(queryState, astNode); + analyzer.initCtx(ctx); + + try (MockedStatic mocked = + mockStatic(DDLSemanticAnalyzerFactory.class, CALLS_REAL_METHODS)) { + BaseSemanticAnalyzer[] cteAnalyzer = new BaseSemanticAnalyzer[1]; + + mocked.when(() -> DDLSemanticAnalyzerFactory.getAnalyzer(any(ASTNode.class), any(QueryState.class))) + .thenAnswer(invocation -> { + BaseSemanticAnalyzer result = (BaseSemanticAnalyzer) invocation.callRealMethod(); + if (invocation.getArgument(0, ASTNode.class).getType() == HiveParser.TOK_CREATETABLE) { + cteAnalyzer[0] = result; + } + return result; + }); + + analyzer.analyze(astNode, ctx); + + assertNotNull("DDLSemanticAnalyzerFactory should be called for CTE materialization", cteAnalyzer[0]); + assertTrue("CTE materialization should use CreateTableAnalyzer", + cteAnalyzer[0] instanceof CreateTableAnalyzer); + } + } } diff --git a/ql/src/test/queries/clientpositive/cte_materialize.q b/ql/src/test/queries/clientpositive/cte_materialize.q new file mode 100644 index 000000000000..b56c0d393b44 --- /dev/null +++ b/ql/src/test/queries/clientpositive/cte_materialize.q @@ -0,0 +1,27 @@ +-- Test CTE materialization with both CBO enabled and disabled +-- Verifies DDLSemanticAnalyzerFactory is used for CTE materialization +-- Also ensures that an NPE is no longer triggered with CBO off (HIVE-28724 regression) + +-- Test with CBO enabled (default) +explain +WITH cte AS ( + SELECT COUNT(*) as cnt FROM (SELECT 1 as id) t +) +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +UNION ALL +SELECT * FROM cte; + +-- Test with CBO disabled +set hive.cbo.enable=false; + +explain +WITH cte AS ( + SELECT COUNT(*) as cnt FROM (SELECT 1 as id) t +) +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +UNION ALL +SELECT * FROM cte; diff --git a/ql/src/test/results/clientpositive/llap/cte_materialize.q.out b/ql/src/test/results/clientpositive/llap/cte_materialize.q.out new file mode 100644 index 000000000000..827963eb8514 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/cte_materialize.q.out @@ -0,0 +1,316 @@ +PREHOOK: query: explain +WITH cte AS ( + SELECT COUNT(*) as cnt FROM (SELECT 1 as id) t +) +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +PREHOOK: type: QUERY +PREHOOK: Input: default@cte +#### A masked pattern was here #### +POSTHOOK: query: explain +WITH cte AS ( + SELECT COUNT(*) as cnt FROM (SELECT 1 as id) t +) +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cte +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-0 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-4 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.4 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.cte + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Tez +#### A masked pattern was here #### + Edges: + Map 3 <- Union 4 (CONTAINS) + Map 5 <- Union 4 (CONTAINS) + Map 6 <- Union 4 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: cte + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cnt (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map 5 + Map Operator Tree: + TableScan + alias: cte + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cnt (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map 6 + Map Operator Tree: + TableScan + alias: cte + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cnt (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-3 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +WITH cte AS ( + SELECT COUNT(*) as cnt FROM (SELECT 1 as id) t +) +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +PREHOOK: type: QUERY +PREHOOK: Input: default@cte +#### A masked pattern was here #### +POSTHOOK: query: explain +WITH cte AS ( + SELECT COUNT(*) as cnt FROM (SELECT 1 as id) t +) +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cte +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-0 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-4 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.4 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.cte + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Tez +#### A masked pattern was here #### + Edges: + Map 3 <- Union 4 (CONTAINS) + Map 5 <- Union 4 (CONTAINS) + Map 6 <- Union 4 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: cte + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cnt (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map 5 + Map Operator Tree: + TableScan + alias: cte + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cnt (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map 6 + Map Operator Tree: + TableScan + alias: cte + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cnt (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-3 + Fetch Operator + limit: -1 + Processor Tree: + ListSink +