PREHOOK: query: -- union10.q -- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink create table tmptable(key string, value int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tmptable POSTHOOK: query: -- union10.q -- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink create table tmptable(key string, value int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@tmptable PREHOOK: query: explain insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2 UNION DISTINCT select 'tst3' as key, count(1) as value from src s3) unionsrc PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2 UNION DISTINCT select 'tst3' as key, count(1) as value from src s3) unionsrc POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Tez Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 6 <- Union 5 (SIMPLE_EDGE) Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 7 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 9 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reducer 10 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst3' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Reducer 6 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable Reducer 8 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst2' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Union 3 Vertex: Union 3 Union 5 Vertex: Union 5 Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable Stage: Stage-3 Stats-Aggr Operator PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2 UNION DISTINCT select 'tst3' as key, count(1) as value from src s3) unionsrc PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@tmptable POSTHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2 UNION DISTINCT select 'tst3' as key, count(1) as value from src s3) unionsrc POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@tmptable POSTHOOK: Lineage: tmptable.key EXPRESSION [] POSTHOOK: Lineage: tmptable.value EXPRESSION [] PREHOOK: query: select * from tmptable x sort by x.key PREHOOK: type: QUERY PREHOOK: Input: default@tmptable #### A masked pattern was here #### POSTHOOK: query: select * from tmptable x sort by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@tmptable #### A masked pattern was here #### tst1 500 tst2 500 tst3 500 PREHOOK: query: -- union11.q -- SORT_BEFORE_DIFF -- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by reducesink explain select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2 UNION DISTINCT select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key PREHOOK: type: QUERY POSTHOOK: query: -- union11.q -- SORT_BEFORE_DIFF -- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by reducesink explain select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2 UNION DISTINCT select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 6 <- Union 5 (SIMPLE_EDGE) Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 7 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 9 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reducer 10 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst3' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string) Reducer 6 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 8 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst2' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Union 3 Vertex: Union 3 Union 5 Vertex: Union 5 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2 UNION DISTINCT select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2 UNION DISTINCT select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### tst1 1 tst2 1 tst3 1 PREHOOK: query: -- union12.q -- union case: all subqueries are a map-reduce jobs, 3 way union, different inputs for all sub-queries, followed by filesink create table tmptable12(key string, value int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tmptable12 POSTHOOK: query: -- union12.q -- union case: all subqueries are a map-reduce jobs, 3 way union, different inputs for all sub-queries, followed by filesink create table tmptable12(key string, value int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@tmptable12 PREHOOK: query: explain insert overwrite table tmptable12 select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src1 s2 UNION DISTINCT select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table tmptable12 select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src1 s2 UNION DISTINCT select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Tez Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 6 <- Union 5 (SIMPLE_EDGE) Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 7 Map Operator Tree: TableScan alias: s2 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 9 Map Operator Tree: TableScan alias: s3 Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reducer 10 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst3' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Reducer 6 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable12 Reducer 8 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst2' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Union 3 Vertex: Union 3 Union 5 Vertex: Union 5 Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable12 Stage: Stage-3 Stats-Aggr Operator PREHOOK: query: insert overwrite table tmptable12 select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src1 s2 UNION DISTINCT select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@src1 PREHOOK: Input: default@srcbucket PREHOOK: Output: default@tmptable12 POSTHOOK: query: insert overwrite table tmptable12 select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src1 s2 UNION DISTINCT select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 POSTHOOK: Input: default@srcbucket POSTHOOK: Output: default@tmptable12 POSTHOOK: Lineage: tmptable12.key EXPRESSION [] POSTHOOK: Lineage: tmptable12.value EXPRESSION [] PREHOOK: query: select * from tmptable12 x sort by x.key PREHOOK: type: QUERY PREHOOK: Input: default@tmptable12 #### A masked pattern was here #### POSTHOOK: query: select * from tmptable12 x sort by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@tmptable12 #### A masked pattern was here #### tst1 500 tst2 25 tst3 1000 PREHOOK: query: -- union13.q -- SORT_BEFORE_DIFF -- union case: both subqueries are a map-only jobs, same input, followed by filesink explain select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc PREHOOK: type: QUERY POSTHOOK: query: -- union13.q -- SORT_BEFORE_DIFF -- union case: both subqueries are a map-only jobs, same input, followed by filesink explain select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 4 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 4 Map Operator Tree: TableScan alias: s1 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 0 val_0 10 val_10 100 val_100 103 val_103 104 val_104 105 val_105 11 val_11 111 val_111 113 val_113 114 val_114 116 val_116 118 val_118 119 val_119 12 val_12 120 val_120 125 val_125 126 val_126 128 val_128 129 val_129 131 val_131 133 val_133 134 val_134 136 val_136 137 val_137 138 val_138 143 val_143 145 val_145 146 val_146 149 val_149 15 val_15 150 val_150 152 val_152 153 val_153 155 val_155 156 val_156 157 val_157 158 val_158 160 val_160 162 val_162 163 val_163 164 val_164 165 val_165 166 val_166 167 val_167 168 val_168 169 val_169 17 val_17 170 val_170 172 val_172 174 val_174 175 val_175 176 val_176 177 val_177 178 val_178 179 val_179 18 val_18 180 val_180 181 val_181 183 val_183 186 val_186 187 val_187 189 val_189 19 val_19 190 val_190 191 val_191 192 val_192 193 val_193 194 val_194 195 val_195 196 val_196 197 val_197 199 val_199 2 val_2 20 val_20 200 val_200 201 val_201 202 val_202 203 val_203 205 val_205 207 val_207 208 val_208 209 val_209 213 val_213 214 val_214 216 val_216 217 val_217 218 val_218 219 val_219 221 val_221 222 val_222 223 val_223 224 val_224 226 val_226 228 val_228 229 val_229 230 val_230 233 val_233 235 val_235 237 val_237 238 val_238 239 val_239 24 val_24 241 val_241 242 val_242 244 val_244 247 val_247 248 val_248 249 val_249 252 val_252 255 val_255 256 val_256 257 val_257 258 val_258 26 val_26 260 val_260 262 val_262 263 val_263 265 val_265 266 val_266 27 val_27 272 val_272 273 val_273 274 val_274 275 val_275 277 val_277 278 val_278 28 val_28 280 val_280 281 val_281 282 val_282 283 val_283 284 val_284 285 val_285 286 val_286 287 val_287 288 val_288 289 val_289 291 val_291 292 val_292 296 val_296 298 val_298 30 val_30 302 val_302 305 val_305 306 val_306 307 val_307 308 val_308 309 val_309 310 val_310 311 val_311 315 val_315 316 val_316 317 val_317 318 val_318 321 val_321 322 val_322 323 val_323 325 val_325 327 val_327 33 val_33 331 val_331 332 val_332 333 val_333 335 val_335 336 val_336 338 val_338 339 val_339 34 val_34 341 val_341 342 val_342 344 val_344 345 val_345 348 val_348 35 val_35 351 val_351 353 val_353 356 val_356 360 val_360 362 val_362 364 val_364 365 val_365 366 val_366 367 val_367 368 val_368 369 val_369 37 val_37 373 val_373 374 val_374 375 val_375 377 val_377 378 val_378 379 val_379 382 val_382 384 val_384 386 val_386 389 val_389 392 val_392 393 val_393 394 val_394 395 val_395 396 val_396 397 val_397 399 val_399 4 val_4 400 val_400 401 val_401 402 val_402 403 val_403 404 val_404 406 val_406 407 val_407 409 val_409 41 val_41 411 val_411 413 val_413 414 val_414 417 val_417 418 val_418 419 val_419 42 val_42 421 val_421 424 val_424 427 val_427 429 val_429 43 val_43 430 val_430 431 val_431 432 val_432 435 val_435 436 val_436 437 val_437 438 val_438 439 val_439 44 val_44 443 val_443 444 val_444 446 val_446 448 val_448 449 val_449 452 val_452 453 val_453 454 val_454 455 val_455 457 val_457 458 val_458 459 val_459 460 val_460 462 val_462 463 val_463 466 val_466 467 val_467 468 val_468 469 val_469 47 val_47 470 val_470 472 val_472 475 val_475 477 val_477 478 val_478 479 val_479 480 val_480 481 val_481 482 val_482 483 val_483 484 val_484 485 val_485 487 val_487 489 val_489 490 val_490 491 val_491 492 val_492 493 val_493 494 val_494 495 val_495 496 val_496 497 val_497 498 val_498 5 val_5 51 val_51 53 val_53 54 val_54 57 val_57 58 val_58 64 val_64 65 val_65 66 val_66 67 val_67 69 val_69 70 val_70 72 val_72 74 val_74 76 val_76 77 val_77 78 val_78 8 val_8 80 val_80 82 val_82 83 val_83 84 val_84 85 val_85 86 val_86 87 val_87 9 val_9 90 val_90 92 val_92 95 val_95 96 val_96 97 val_97 98 val_98 PREHOOK: query: -- union14.q -- SORT_BEFORE_DIFF -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink explain select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 UNION DISTINCT select 'tst1' as key, cast(count(1) as string) as value from src s1) unionsrc group by unionsrc.key PREHOOK: type: QUERY POSTHOOK: query: -- union14.q -- SORT_BEFORE_DIFF -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink explain select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 UNION DISTINCT select 'tst1' as key, cast(count(1) as string) as value from src s1) unionsrc group by unionsrc.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s2 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Map 4 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 5 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Union 2 Vertex: Union 2 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 UNION DISTINCT select 'tst1' as key, cast(count(1) as string) as value from src s1) unionsrc group by unionsrc.key PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@src1 #### A masked pattern was here #### POSTHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 UNION DISTINCT select 'tst1' as key, cast(count(1) as string) as value from src s1) unionsrc group by unionsrc.key POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### 7 128 1 146 1 150 1 213 1 224 1 238 1 255 1 273 1 278 1 311 1 369 1 401 1 406 1 66 1 98 1 tst1 1 PREHOOK: query: -- union15.q -- SORT_BEFORE_DIFF -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink explain select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src1 s2 UNION DISTINCT select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key PREHOOK: type: QUERY POSTHOOK: query: -- union15.q -- SORT_BEFORE_DIFF -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink explain select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src1 s2 UNION DISTINCT select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 7 <- Union 3 (CONTAINS) Map 8 <- Union 5 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 6 <- Union 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 7 Map Operator Tree: TableScan alias: s2 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 8 Map Operator Tree: TableScan alias: s2 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Reducer 6 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 3 Vertex: Union 3 Union 5 Vertex: Union 5 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src1 s2 UNION DISTINCT select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@src1 #### A masked pattern was here #### POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src1 s2 UNION DISTINCT select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### 7 128 1 146 1 150 1 213 1 224 1 238 1 255 1 273 1 278 1 311 1 369 1 401 1 406 1 66 1 98 1 tst1 1 PREHOOK: query: -- union16.q -- SORT_BEFORE_DIFF EXPLAIN SELECT count(1) FROM ( SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src) src PREHOOK: type: QUERY POSTHOOK: query: -- union16.q -- SORT_BEFORE_DIFF EXPLAIN SELECT count(1) FROM ( SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src) src POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 51 <- Union 2 (CONTAINS) Map 52 <- Union 4 (CONTAINS) Map 53 <- Union 6 (CONTAINS) Map 54 <- Union 8 (CONTAINS) Map 55 <- Union 10 (CONTAINS) Map 56 <- Union 12 (CONTAINS) Map 57 <- Union 14 (CONTAINS) Map 58 <- Union 16 (CONTAINS) Map 59 <- Union 18 (CONTAINS) Map 60 <- Union 20 (CONTAINS) Map 61 <- Union 22 (CONTAINS) Map 62 <- Union 24 (CONTAINS) Map 63 <- Union 26 (CONTAINS) Map 64 <- Union 28 (CONTAINS) Map 65 <- Union 30 (CONTAINS) Map 66 <- Union 32 (CONTAINS) Map 67 <- Union 34 (CONTAINS) Map 68 <- Union 36 (CONTAINS) Map 69 <- Union 38 (CONTAINS) Map 70 <- Union 40 (CONTAINS) Map 71 <- Union 42 (CONTAINS) Map 72 <- Union 44 (CONTAINS) Map 73 <- Union 46 (CONTAINS) Map 74 <- Union 48 (CONTAINS) Reducer 11 <- Union 10 (SIMPLE_EDGE), Union 12 (CONTAINS) Reducer 13 <- Union 12 (SIMPLE_EDGE), Union 14 (CONTAINS) Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS) Reducer 17 <- Union 16 (SIMPLE_EDGE), Union 18 (CONTAINS) Reducer 19 <- Union 18 (SIMPLE_EDGE), Union 20 (CONTAINS) Reducer 21 <- Union 20 (SIMPLE_EDGE), Union 22 (CONTAINS) Reducer 23 <- Union 22 (SIMPLE_EDGE), Union 24 (CONTAINS) Reducer 25 <- Union 24 (SIMPLE_EDGE), Union 26 (CONTAINS) Reducer 27 <- Union 26 (SIMPLE_EDGE), Union 28 (CONTAINS) Reducer 29 <- Union 28 (SIMPLE_EDGE), Union 30 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) Reducer 31 <- Union 30 (SIMPLE_EDGE), Union 32 (CONTAINS) Reducer 33 <- Union 32 (SIMPLE_EDGE), Union 34 (CONTAINS) Reducer 35 <- Union 34 (SIMPLE_EDGE), Union 36 (CONTAINS) Reducer 37 <- Union 36 (SIMPLE_EDGE), Union 38 (CONTAINS) Reducer 39 <- Union 38 (SIMPLE_EDGE), Union 40 (CONTAINS) Reducer 41 <- Union 40 (SIMPLE_EDGE), Union 42 (CONTAINS) Reducer 43 <- Union 42 (SIMPLE_EDGE), Union 44 (CONTAINS) Reducer 45 <- Union 44 (SIMPLE_EDGE), Union 46 (CONTAINS) Reducer 47 <- Union 46 (SIMPLE_EDGE), Union 48 (CONTAINS) Reducer 49 <- Union 48 (SIMPLE_EDGE) Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) Reducer 50 <- Reducer 49 (SIMPLE_EDGE) Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) Reducer 9 <- Union 10 (CONTAINS), Union 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 51 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 52 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 53 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 54 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 55 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 56 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 57 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 58 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 59 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 60 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 61 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 62 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 63 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 64 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 65 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 66 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 67 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 68 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 69 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 70 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 71 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 72 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 73 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 74 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 11 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 13 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 15 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 17 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 19 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 21 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 23 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 25 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 27 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 29 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 31 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 33 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 35 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 37 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 39 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 41 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 43 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 45 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 47 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 49 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 5 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 50 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 7 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 9 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Union 10 Vertex: Union 10 Union 12 Vertex: Union 12 Union 14 Vertex: Union 14 Union 16 Vertex: Union 16 Union 18 Vertex: Union 18 Union 2 Vertex: Union 2 Union 20 Vertex: Union 20 Union 22 Vertex: Union 22 Union 24 Vertex: Union 24 Union 26 Vertex: Union 26 Union 28 Vertex: Union 28 Union 30 Vertex: Union 30 Union 32 Vertex: Union 32 Union 34 Vertex: Union 34 Union 36 Vertex: Union 36 Union 38 Vertex: Union 38 Union 4 Vertex: Union 4 Union 40 Vertex: Union 40 Union 42 Vertex: Union 42 Union 44 Vertex: Union 44 Union 46 Vertex: Union 46 Union 48 Vertex: Union 48 Union 6 Vertex: Union 6 Union 8 Vertex: Union 8 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: SELECT count(1) FROM ( SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src) src PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: SELECT count(1) FROM ( SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src UNION DISTINCT SELECT key, value FROM src) src POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 309 PREHOOK: query: -- union17.q CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@DEST1 POSTHOOK: query: -- union17.q CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@DEST1 PREHOOK: query: CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@DEST2 POSTHOOK: query: CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@DEST2 PREHOOK: query: -- SORT_BEFORE_DIFF -- union case:map-reduce sub-queries followed by multi-table insert explain FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value PREHOOK: type: QUERY POSTHOOK: query: -- SORT_BEFORE_DIFF -- union case:map-reduce sub-queries followed by multi-table insert explain FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 Stage-4 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 Tez Edges: Map 6 <- Union 3 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 6 Map Operator Tree: TableScan alias: s2 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Group By Operator aggregations: count(DISTINCT substr(_col1, 5)) keys: _col0 (type: string), substr(_col1, 5) (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Group By Operator aggregations: count(DISTINCT substr(_col1, 5)) keys: _col0 (type: string), _col1 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 Reducer 5 Reduce Operator Tree: Group By Operator aggregations: count(DISTINCT KEY._col1:0._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 Union 3 Vertex: Union 3 Stage: Stage-3 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 Stage: Stage-4 Stats-Aggr Operator Stage: Stage-1 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 Stage: Stage-5 Stats-Aggr Operator PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@dest1 PREHOOK: Output: default@dest2 POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1 POSTHOOK: Output: default@dest2 POSTHOOK: Lineage: dest1.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest1.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest2.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest2.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest2.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: SELECT DEST1.* FROM DEST1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### POSTHOOK: query: SELECT DEST1.* FROM DEST1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### 0 1 10 1 100 1 103 1 104 1 105 1 11 1 111 1 113 1 114 1 116 1 118 1 119 1 12 1 120 1 125 1 126 1 128 1 129 1 131 1 133 1 134 1 136 1 137 1 138 1 143 1 145 1 146 1 149 1 15 1 150 1 152 1 153 1 155 1 156 1 157 1 158 1 160 1 162 1 163 1 164 1 165 1 166 1 167 1 168 1 169 1 17 1 170 1 172 1 174 1 175 1 176 1 177 1 178 1 179 1 18 1 180 1 181 1 183 1 186 1 187 1 189 1 19 1 190 1 191 1 192 1 193 1 194 1 195 1 196 1 197 1 199 1 2 1 20 1 200 1 201 1 202 1 203 1 205 1 207 1 208 1 209 1 213 1 214 1 216 1 217 1 218 1 219 1 221 1 222 1 223 1 224 1 226 1 228 1 229 1 230 1 233 1 235 1 237 1 238 1 239 1 24 1 241 1 242 1 244 1 247 1 248 1 249 1 252 1 255 1 256 1 257 1 258 1 26 1 260 1 262 1 263 1 265 1 266 1 27 1 272 1 273 1 274 1 275 1 277 1 278 1 28 1 280 1 281 1 282 1 283 1 284 1 285 1 286 1 287 1 288 1 289 1 291 1 292 1 296 1 298 1 30 1 302 1 305 1 306 1 307 1 308 1 309 1 310 1 311 1 315 1 316 1 317 1 318 1 321 1 322 1 323 1 325 1 327 1 33 1 331 1 332 1 333 1 335 1 336 1 338 1 339 1 34 1 341 1 342 1 344 1 345 1 348 1 35 1 351 1 353 1 356 1 360 1 362 1 364 1 365 1 366 1 367 1 368 1 369 1 37 1 373 1 374 1 375 1 377 1 378 1 379 1 382 1 384 1 386 1 389 1 392 1 393 1 394 1 395 1 396 1 397 1 399 1 4 1 400 1 401 1 402 1 403 1 404 1 406 1 407 1 409 1 41 1 411 1 413 1 414 1 417 1 418 1 419 1 42 1 421 1 424 1 427 1 429 1 43 1 430 1 431 1 432 1 435 1 436 1 437 1 438 1 439 1 44 1 443 1 444 1 446 1 448 1 449 1 452 1 453 1 454 1 455 1 457 1 458 1 459 1 460 1 462 1 463 1 466 1 467 1 468 1 469 1 47 1 470 1 472 1 475 1 477 1 478 1 479 1 480 1 481 1 482 1 483 1 484 1 485 1 487 1 489 1 490 1 491 1 492 1 493 1 494 1 495 1 496 1 497 1 498 1 5 1 51 1 53 1 54 1 57 1 58 1 64 1 65 1 66 1 67 1 69 1 70 1 72 1 74 1 76 1 77 1 78 1 8 1 80 1 82 1 83 1 84 1 85 1 86 1 87 1 9 1 90 1 92 1 95 1 96 1 97 1 98 1 tst1 1 PREHOOK: query: SELECT DEST2.* FROM DEST2 PREHOOK: type: QUERY PREHOOK: Input: default@dest2 #### A masked pattern was here #### POSTHOOK: query: SELECT DEST2.* FROM DEST2 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest2 #### A masked pattern was here #### 0 val_0 1 10 val_10 1 100 val_100 1 103 val_103 1 104 val_104 1 105 val_105 1 11 val_11 1 111 val_111 1 113 val_113 1 114 val_114 1 116 val_116 1 118 val_118 1 119 val_119 1 12 val_12 1 120 val_120 1 125 val_125 1 126 val_126 1 128 val_128 1 129 val_129 1 131 val_131 1 133 val_133 1 134 val_134 1 136 val_136 1 137 val_137 1 138 val_138 1 143 val_143 1 145 val_145 1 146 val_146 1 149 val_149 1 15 val_15 1 150 val_150 1 152 val_152 1 153 val_153 1 155 val_155 1 156 val_156 1 157 val_157 1 158 val_158 1 160 val_160 1 162 val_162 1 163 val_163 1 164 val_164 1 165 val_165 1 166 val_166 1 167 val_167 1 168 val_168 1 169 val_169 1 17 val_17 1 170 val_170 1 172 val_172 1 174 val_174 1 175 val_175 1 176 val_176 1 177 val_177 1 178 val_178 1 179 val_179 1 18 val_18 1 180 val_180 1 181 val_181 1 183 val_183 1 186 val_186 1 187 val_187 1 189 val_189 1 19 val_19 1 190 val_190 1 191 val_191 1 192 val_192 1 193 val_193 1 194 val_194 1 195 val_195 1 196 val_196 1 197 val_197 1 199 val_199 1 2 val_2 1 20 val_20 1 200 val_200 1 201 val_201 1 202 val_202 1 203 val_203 1 205 val_205 1 207 val_207 1 208 val_208 1 209 val_209 1 213 val_213 1 214 val_214 1 216 val_216 1 217 val_217 1 218 val_218 1 219 val_219 1 221 val_221 1 222 val_222 1 223 val_223 1 224 val_224 1 226 val_226 1 228 val_228 1 229 val_229 1 230 val_230 1 233 val_233 1 235 val_235 1 237 val_237 1 238 val_238 1 239 val_239 1 24 val_24 1 241 val_241 1 242 val_242 1 244 val_244 1 247 val_247 1 248 val_248 1 249 val_249 1 252 val_252 1 255 val_255 1 256 val_256 1 257 val_257 1 258 val_258 1 26 val_26 1 260 val_260 1 262 val_262 1 263 val_263 1 265 val_265 1 266 val_266 1 27 val_27 1 272 val_272 1 273 val_273 1 274 val_274 1 275 val_275 1 277 val_277 1 278 val_278 1 28 val_28 1 280 val_280 1 281 val_281 1 282 val_282 1 283 val_283 1 284 val_284 1 285 val_285 1 286 val_286 1 287 val_287 1 288 val_288 1 289 val_289 1 291 val_291 1 292 val_292 1 296 val_296 1 298 val_298 1 30 val_30 1 302 val_302 1 305 val_305 1 306 val_306 1 307 val_307 1 308 val_308 1 309 val_309 1 310 val_310 1 311 val_311 1 315 val_315 1 316 val_316 1 317 val_317 1 318 val_318 1 321 val_321 1 322 val_322 1 323 val_323 1 325 val_325 1 327 val_327 1 33 val_33 1 331 val_331 1 332 val_332 1 333 val_333 1 335 val_335 1 336 val_336 1 338 val_338 1 339 val_339 1 34 val_34 1 341 val_341 1 342 val_342 1 344 val_344 1 345 val_345 1 348 val_348 1 35 val_35 1 351 val_351 1 353 val_353 1 356 val_356 1 360 val_360 1 362 val_362 1 364 val_364 1 365 val_365 1 366 val_366 1 367 val_367 1 368 val_368 1 369 val_369 1 37 val_37 1 373 val_373 1 374 val_374 1 375 val_375 1 377 val_377 1 378 val_378 1 379 val_379 1 382 val_382 1 384 val_384 1 386 val_386 1 389 val_389 1 392 val_392 1 393 val_393 1 394 val_394 1 395 val_395 1 396 val_396 1 397 val_397 1 399 val_399 1 4 val_4 1 400 val_400 1 401 val_401 1 402 val_402 1 403 val_403 1 404 val_404 1 406 val_406 1 407 val_407 1 409 val_409 1 41 val_41 1 411 val_411 1 413 val_413 1 414 val_414 1 417 val_417 1 418 val_418 1 419 val_419 1 42 val_42 1 421 val_421 1 424 val_424 1 427 val_427 1 429 val_429 1 43 val_43 1 430 val_430 1 431 val_431 1 432 val_432 1 435 val_435 1 436 val_436 1 437 val_437 1 438 val_438 1 439 val_439 1 44 val_44 1 443 val_443 1 444 val_444 1 446 val_446 1 448 val_448 1 449 val_449 1 452 val_452 1 453 val_453 1 454 val_454 1 455 val_455 1 457 val_457 1 458 val_458 1 459 val_459 1 460 val_460 1 462 val_462 1 463 val_463 1 466 val_466 1 467 val_467 1 468 val_468 1 469 val_469 1 47 val_47 1 470 val_470 1 472 val_472 1 475 val_475 1 477 val_477 1 478 val_478 1 479 val_479 1 480 val_480 1 481 val_481 1 482 val_482 1 483 val_483 1 484 val_484 1 485 val_485 1 487 val_487 1 489 val_489 1 490 val_490 1 491 val_491 1 492 val_492 1 493 val_493 1 494 val_494 1 495 val_495 1 496 val_496 1 497 val_497 1 498 val_498 1 5 val_5 1 51 val_51 1 53 val_53 1 54 val_54 1 57 val_57 1 58 val_58 1 64 val_64 1 65 val_65 1 66 val_66 1 67 val_67 1 69 val_69 1 70 val_70 1 72 val_72 1 74 val_74 1 76 val_76 1 77 val_77 1 78 val_78 1 8 val_8 1 80 val_80 1 82 val_82 1 83 val_83 1 84 val_84 1 85 val_85 1 86 val_86 1 87 val_87 1 9 val_9 1 90 val_90 1 92 val_92 1 95 val_95 1 96 val_96 1 97 val_97 1 98 val_98 1 tst1 500 1 PREHOOK: query: -- union18.q CREATE TABLE DEST118(key STRING, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@DEST118 POSTHOOK: query: -- union18.q CREATE TABLE DEST118(key STRING, value STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@DEST118 PREHOOK: query: CREATE TABLE DEST218(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@DEST218 POSTHOOK: query: CREATE TABLE DEST218(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@DEST218 PREHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert explain FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value PREHOOK: type: QUERY POSTHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert explain FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 Stage-4 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 Tez Edges: Map 5 <- Union 3 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 5 Map Operator Tree: TableScan alias: s2 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest118 Select Operator expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest218 Union 3 Vertex: Union 3 Stage: Stage-3 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest118 Stage: Stage-4 Stats-Aggr Operator Stage: Stage-1 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest218 Stage: Stage-5 Stats-Aggr Operator PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@dest118 PREHOOK: Output: default@dest218 POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dest118 POSTHOOK: Output: default@dest218 POSTHOOK: Lineage: dest118.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest118.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest218.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest218.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest218.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: SELECT DEST118.* FROM DEST118 SORT BY DEST118.key, DEST118.value PREHOOK: type: QUERY PREHOOK: Input: default@dest118 #### A masked pattern was here #### POSTHOOK: query: SELECT DEST118.* FROM DEST118 SORT BY DEST118.key, DEST118.value POSTHOOK: type: QUERY POSTHOOK: Input: default@dest118 #### A masked pattern was here #### 0 val_0 10 val_10 100 val_100 103 val_103 104 val_104 105 val_105 11 val_11 111 val_111 113 val_113 114 val_114 116 val_116 118 val_118 119 val_119 12 val_12 120 val_120 125 val_125 126 val_126 128 val_128 129 val_129 131 val_131 133 val_133 134 val_134 136 val_136 137 val_137 138 val_138 143 val_143 145 val_145 146 val_146 149 val_149 15 val_15 150 val_150 152 val_152 153 val_153 155 val_155 156 val_156 157 val_157 158 val_158 160 val_160 162 val_162 163 val_163 164 val_164 165 val_165 166 val_166 167 val_167 168 val_168 169 val_169 17 val_17 170 val_170 172 val_172 174 val_174 175 val_175 176 val_176 177 val_177 178 val_178 179 val_179 18 val_18 180 val_180 181 val_181 183 val_183 186 val_186 187 val_187 189 val_189 19 val_19 190 val_190 191 val_191 192 val_192 193 val_193 194 val_194 195 val_195 196 val_196 197 val_197 199 val_199 2 val_2 20 val_20 200 val_200 201 val_201 202 val_202 203 val_203 205 val_205 207 val_207 208 val_208 209 val_209 213 val_213 214 val_214 216 val_216 217 val_217 218 val_218 219 val_219 221 val_221 222 val_222 223 val_223 224 val_224 226 val_226 228 val_228 229 val_229 230 val_230 233 val_233 235 val_235 237 val_237 238 val_238 239 val_239 24 val_24 241 val_241 242 val_242 244 val_244 247 val_247 248 val_248 249 val_249 252 val_252 255 val_255 256 val_256 257 val_257 258 val_258 26 val_26 260 val_260 262 val_262 263 val_263 265 val_265 266 val_266 27 val_27 272 val_272 273 val_273 274 val_274 275 val_275 277 val_277 278 val_278 28 val_28 280 val_280 281 val_281 282 val_282 283 val_283 284 val_284 285 val_285 286 val_286 287 val_287 288 val_288 289 val_289 291 val_291 292 val_292 296 val_296 298 val_298 30 val_30 302 val_302 305 val_305 306 val_306 307 val_307 308 val_308 309 val_309 310 val_310 311 val_311 315 val_315 316 val_316 317 val_317 318 val_318 321 val_321 322 val_322 323 val_323 325 val_325 327 val_327 33 val_33 331 val_331 332 val_332 333 val_333 335 val_335 336 val_336 338 val_338 339 val_339 34 val_34 341 val_341 342 val_342 344 val_344 345 val_345 348 val_348 35 val_35 351 val_351 353 val_353 356 val_356 360 val_360 362 val_362 364 val_364 365 val_365 366 val_366 367 val_367 368 val_368 369 val_369 37 val_37 373 val_373 374 val_374 375 val_375 377 val_377 378 val_378 379 val_379 382 val_382 384 val_384 386 val_386 389 val_389 392 val_392 393 val_393 394 val_394 395 val_395 396 val_396 397 val_397 399 val_399 4 val_4 400 val_400 401 val_401 402 val_402 403 val_403 404 val_404 406 val_406 407 val_407 409 val_409 41 val_41 411 val_411 413 val_413 414 val_414 417 val_417 418 val_418 419 val_419 42 val_42 421 val_421 424 val_424 427 val_427 429 val_429 43 val_43 430 val_430 431 val_431 432 val_432 435 val_435 436 val_436 437 val_437 438 val_438 439 val_439 44 val_44 443 val_443 444 val_444 446 val_446 448 val_448 449 val_449 452 val_452 453 val_453 454 val_454 455 val_455 457 val_457 458 val_458 459 val_459 460 val_460 462 val_462 463 val_463 466 val_466 467 val_467 468 val_468 469 val_469 47 val_47 470 val_470 472 val_472 475 val_475 477 val_477 478 val_478 479 val_479 480 val_480 481 val_481 482 val_482 483 val_483 484 val_484 485 val_485 487 val_487 489 val_489 490 val_490 491 val_491 492 val_492 493 val_493 494 val_494 495 val_495 496 val_496 497 val_497 498 val_498 5 val_5 51 val_51 53 val_53 54 val_54 57 val_57 58 val_58 64 val_64 65 val_65 66 val_66 67 val_67 69 val_69 70 val_70 72 val_72 74 val_74 76 val_76 77 val_77 78 val_78 8 val_8 80 val_80 82 val_82 83 val_83 84 val_84 85 val_85 86 val_86 87 val_87 9 val_9 90 val_90 92 val_92 95 val_95 96 val_96 97 val_97 98 val_98 tst1 500 PREHOOK: query: SELECT DEST218.* FROM DEST218 SORT BY DEST218.key, DEST218.val1, DEST218.val2 PREHOOK: type: QUERY PREHOOK: Input: default@dest218 #### A masked pattern was here #### POSTHOOK: query: SELECT DEST218.* FROM DEST218 SORT BY DEST218.key, DEST218.val1, DEST218.val2 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest218 #### A masked pattern was here #### 0 val_0 val_0 10 val_10 val_10 100 val_100 val_100 103 val_103 val_103 104 val_104 val_104 105 val_105 val_105 11 val_11 val_11 111 val_111 val_111 113 val_113 val_113 114 val_114 val_114 116 val_116 val_116 118 val_118 val_118 119 val_119 val_119 12 val_12 val_12 120 val_120 val_120 125 val_125 val_125 126 val_126 val_126 128 val_128 val_128 129 val_129 val_129 131 val_131 val_131 133 val_133 val_133 134 val_134 val_134 136 val_136 val_136 137 val_137 val_137 138 val_138 val_138 143 val_143 val_143 145 val_145 val_145 146 val_146 val_146 149 val_149 val_149 15 val_15 val_15 150 val_150 val_150 152 val_152 val_152 153 val_153 val_153 155 val_155 val_155 156 val_156 val_156 157 val_157 val_157 158 val_158 val_158 160 val_160 val_160 162 val_162 val_162 163 val_163 val_163 164 val_164 val_164 165 val_165 val_165 166 val_166 val_166 167 val_167 val_167 168 val_168 val_168 169 val_169 val_169 17 val_17 val_17 170 val_170 val_170 172 val_172 val_172 174 val_174 val_174 175 val_175 val_175 176 val_176 val_176 177 val_177 val_177 178 val_178 val_178 179 val_179 val_179 18 val_18 val_18 180 val_180 val_180 181 val_181 val_181 183 val_183 val_183 186 val_186 val_186 187 val_187 val_187 189 val_189 val_189 19 val_19 val_19 190 val_190 val_190 191 val_191 val_191 192 val_192 val_192 193 val_193 val_193 194 val_194 val_194 195 val_195 val_195 196 val_196 val_196 197 val_197 val_197 199 val_199 val_199 2 val_2 val_2 20 val_20 val_20 200 val_200 val_200 201 val_201 val_201 202 val_202 val_202 203 val_203 val_203 205 val_205 val_205 207 val_207 val_207 208 val_208 val_208 209 val_209 val_209 213 val_213 val_213 214 val_214 val_214 216 val_216 val_216 217 val_217 val_217 218 val_218 val_218 219 val_219 val_219 221 val_221 val_221 222 val_222 val_222 223 val_223 val_223 224 val_224 val_224 226 val_226 val_226 228 val_228 val_228 229 val_229 val_229 230 val_230 val_230 233 val_233 val_233 235 val_235 val_235 237 val_237 val_237 238 val_238 val_238 239 val_239 val_239 24 val_24 val_24 241 val_241 val_241 242 val_242 val_242 244 val_244 val_244 247 val_247 val_247 248 val_248 val_248 249 val_249 val_249 252 val_252 val_252 255 val_255 val_255 256 val_256 val_256 257 val_257 val_257 258 val_258 val_258 26 val_26 val_26 260 val_260 val_260 262 val_262 val_262 263 val_263 val_263 265 val_265 val_265 266 val_266 val_266 27 val_27 val_27 272 val_272 val_272 273 val_273 val_273 274 val_274 val_274 275 val_275 val_275 277 val_277 val_277 278 val_278 val_278 28 val_28 val_28 280 val_280 val_280 281 val_281 val_281 282 val_282 val_282 283 val_283 val_283 284 val_284 val_284 285 val_285 val_285 286 val_286 val_286 287 val_287 val_287 288 val_288 val_288 289 val_289 val_289 291 val_291 val_291 292 val_292 val_292 296 val_296 val_296 298 val_298 val_298 30 val_30 val_30 302 val_302 val_302 305 val_305 val_305 306 val_306 val_306 307 val_307 val_307 308 val_308 val_308 309 val_309 val_309 310 val_310 val_310 311 val_311 val_311 315 val_315 val_315 316 val_316 val_316 317 val_317 val_317 318 val_318 val_318 321 val_321 val_321 322 val_322 val_322 323 val_323 val_323 325 val_325 val_325 327 val_327 val_327 33 val_33 val_33 331 val_331 val_331 332 val_332 val_332 333 val_333 val_333 335 val_335 val_335 336 val_336 val_336 338 val_338 val_338 339 val_339 val_339 34 val_34 val_34 341 val_341 val_341 342 val_342 val_342 344 val_344 val_344 345 val_345 val_345 348 val_348 val_348 35 val_35 val_35 351 val_351 val_351 353 val_353 val_353 356 val_356 val_356 360 val_360 val_360 362 val_362 val_362 364 val_364 val_364 365 val_365 val_365 366 val_366 val_366 367 val_367 val_367 368 val_368 val_368 369 val_369 val_369 37 val_37 val_37 373 val_373 val_373 374 val_374 val_374 375 val_375 val_375 377 val_377 val_377 378 val_378 val_378 379 val_379 val_379 382 val_382 val_382 384 val_384 val_384 386 val_386 val_386 389 val_389 val_389 392 val_392 val_392 393 val_393 val_393 394 val_394 val_394 395 val_395 val_395 396 val_396 val_396 397 val_397 val_397 399 val_399 val_399 4 val_4 val_4 400 val_400 val_400 401 val_401 val_401 402 val_402 val_402 403 val_403 val_403 404 val_404 val_404 406 val_406 val_406 407 val_407 val_407 409 val_409 val_409 41 val_41 val_41 411 val_411 val_411 413 val_413 val_413 414 val_414 val_414 417 val_417 val_417 418 val_418 val_418 419 val_419 val_419 42 val_42 val_42 421 val_421 val_421 424 val_424 val_424 427 val_427 val_427 429 val_429 val_429 43 val_43 val_43 430 val_430 val_430 431 val_431 val_431 432 val_432 val_432 435 val_435 val_435 436 val_436 val_436 437 val_437 val_437 438 val_438 val_438 439 val_439 val_439 44 val_44 val_44 443 val_443 val_443 444 val_444 val_444 446 val_446 val_446 448 val_448 val_448 449 val_449 val_449 452 val_452 val_452 453 val_453 val_453 454 val_454 val_454 455 val_455 val_455 457 val_457 val_457 458 val_458 val_458 459 val_459 val_459 460 val_460 val_460 462 val_462 val_462 463 val_463 val_463 466 val_466 val_466 467 val_467 val_467 468 val_468 val_468 469 val_469 val_469 47 val_47 val_47 470 val_470 val_470 472 val_472 val_472 475 val_475 val_475 477 val_477 val_477 478 val_478 val_478 479 val_479 val_479 480 val_480 val_480 481 val_481 val_481 482 val_482 val_482 483 val_483 val_483 484 val_484 val_484 485 val_485 val_485 487 val_487 val_487 489 val_489 val_489 490 val_490 val_490 491 val_491 val_491 492 val_492 val_492 493 val_493 val_493 494 val_494 val_494 495 val_495 val_495 496 val_496 val_496 497 val_497 val_497 498 val_498 val_498 5 val_5 val_5 51 val_51 val_51 53 val_53 val_53 54 val_54 val_54 57 val_57 val_57 58 val_58 val_58 64 val_64 val_64 65 val_65 val_65 66 val_66 val_66 67 val_67 val_67 69 val_69 val_69 70 val_70 val_70 72 val_72 val_72 74 val_74 val_74 76 val_76 val_76 77 val_77 val_77 78 val_78 val_78 8 val_8 val_8 80 val_80 val_80 82 val_82 val_82 83 val_83 val_83 84 val_84 val_84 85 val_85 val_85 86 val_86 val_86 87 val_87 val_87 9 val_9 val_9 90 val_90 val_90 92 val_92 val_92 95 val_95 val_95 96 val_96 val_96 97 val_97 val_97 98 val_98 val_98 tst1 500 500 PREHOOK: query: -- union19.q CREATE TABLE DEST119(key STRING, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@DEST119 POSTHOOK: query: -- union19.q CREATE TABLE DEST119(key STRING, value STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@DEST119 PREHOOK: query: CREATE TABLE DEST219(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@DEST219 POSTHOOK: query: CREATE TABLE DEST219(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@DEST219 PREHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert explain FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value PREHOOK: type: QUERY POSTHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert explain FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 Stage-4 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 Tez Edges: Map 5 <- Union 3 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 5 Map Operator Tree: TableScan alias: s2 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Group By Operator aggregations: count(_col1) keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest119 Select Operator expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest219 Union 3 Vertex: Union 3 Stage: Stage-3 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest119 Stage: Stage-4 Stats-Aggr Operator Stage: Stage-1 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest219 Stage: Stage-5 Stats-Aggr Operator PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@dest119 PREHOOK: Output: default@dest219 POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dest119 POSTHOOK: Output: default@dest219 POSTHOOK: Lineage: dest119.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest119.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest219.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest219.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest219.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: SELECT DEST119.* FROM DEST119 SORT BY DEST119.key, DEST119.value PREHOOK: type: QUERY PREHOOK: Input: default@dest119 #### A masked pattern was here #### POSTHOOK: query: SELECT DEST119.* FROM DEST119 SORT BY DEST119.key, DEST119.value POSTHOOK: type: QUERY POSTHOOK: Input: default@dest119 #### A masked pattern was here #### 0 1 10 1 100 1 103 1 104 1 105 1 11 1 111 1 113 1 114 1 116 1 118 1 119 1 12 1 120 1 125 1 126 1 128 1 129 1 131 1 133 1 134 1 136 1 137 1 138 1 143 1 145 1 146 1 149 1 15 1 150 1 152 1 153 1 155 1 156 1 157 1 158 1 160 1 162 1 163 1 164 1 165 1 166 1 167 1 168 1 169 1 17 1 170 1 172 1 174 1 175 1 176 1 177 1 178 1 179 1 18 1 180 1 181 1 183 1 186 1 187 1 189 1 19 1 190 1 191 1 192 1 193 1 194 1 195 1 196 1 197 1 199 1 2 1 20 1 200 1 201 1 202 1 203 1 205 1 207 1 208 1 209 1 213 1 214 1 216 1 217 1 218 1 219 1 221 1 222 1 223 1 224 1 226 1 228 1 229 1 230 1 233 1 235 1 237 1 238 1 239 1 24 1 241 1 242 1 244 1 247 1 248 1 249 1 252 1 255 1 256 1 257 1 258 1 26 1 260 1 262 1 263 1 265 1 266 1 27 1 272 1 273 1 274 1 275 1 277 1 278 1 28 1 280 1 281 1 282 1 283 1 284 1 285 1 286 1 287 1 288 1 289 1 291 1 292 1 296 1 298 1 30 1 302 1 305 1 306 1 307 1 308 1 309 1 310 1 311 1 315 1 316 1 317 1 318 1 321 1 322 1 323 1 325 1 327 1 33 1 331 1 332 1 333 1 335 1 336 1 338 1 339 1 34 1 341 1 342 1 344 1 345 1 348 1 35 1 351 1 353 1 356 1 360 1 362 1 364 1 365 1 366 1 367 1 368 1 369 1 37 1 373 1 374 1 375 1 377 1 378 1 379 1 382 1 384 1 386 1 389 1 392 1 393 1 394 1 395 1 396 1 397 1 399 1 4 1 400 1 401 1 402 1 403 1 404 1 406 1 407 1 409 1 41 1 411 1 413 1 414 1 417 1 418 1 419 1 42 1 421 1 424 1 427 1 429 1 43 1 430 1 431 1 432 1 435 1 436 1 437 1 438 1 439 1 44 1 443 1 444 1 446 1 448 1 449 1 452 1 453 1 454 1 455 1 457 1 458 1 459 1 460 1 462 1 463 1 466 1 467 1 468 1 469 1 47 1 470 1 472 1 475 1 477 1 478 1 479 1 480 1 481 1 482 1 483 1 484 1 485 1 487 1 489 1 490 1 491 1 492 1 493 1 494 1 495 1 496 1 497 1 498 1 5 1 51 1 53 1 54 1 57 1 58 1 64 1 65 1 66 1 67 1 69 1 70 1 72 1 74 1 76 1 77 1 78 1 8 1 80 1 82 1 83 1 84 1 85 1 86 1 87 1 9 1 90 1 92 1 95 1 96 1 97 1 98 1 tst1 1 PREHOOK: query: SELECT DEST219.* FROM DEST219 SORT BY DEST219.key, DEST219.val1, DEST219.val2 PREHOOK: type: QUERY PREHOOK: Input: default@dest219 #### A masked pattern was here #### POSTHOOK: query: SELECT DEST219.* FROM DEST219 SORT BY DEST219.key, DEST219.val1, DEST219.val2 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest219 #### A masked pattern was here #### 0 val_0 val_0 10 val_10 val_10 100 val_100 val_100 103 val_103 val_103 104 val_104 val_104 105 val_105 val_105 11 val_11 val_11 111 val_111 val_111 113 val_113 val_113 114 val_114 val_114 116 val_116 val_116 118 val_118 val_118 119 val_119 val_119 12 val_12 val_12 120 val_120 val_120 125 val_125 val_125 126 val_126 val_126 128 val_128 val_128 129 val_129 val_129 131 val_131 val_131 133 val_133 val_133 134 val_134 val_134 136 val_136 val_136 137 val_137 val_137 138 val_138 val_138 143 val_143 val_143 145 val_145 val_145 146 val_146 val_146 149 val_149 val_149 15 val_15 val_15 150 val_150 val_150 152 val_152 val_152 153 val_153 val_153 155 val_155 val_155 156 val_156 val_156 157 val_157 val_157 158 val_158 val_158 160 val_160 val_160 162 val_162 val_162 163 val_163 val_163 164 val_164 val_164 165 val_165 val_165 166 val_166 val_166 167 val_167 val_167 168 val_168 val_168 169 val_169 val_169 17 val_17 val_17 170 val_170 val_170 172 val_172 val_172 174 val_174 val_174 175 val_175 val_175 176 val_176 val_176 177 val_177 val_177 178 val_178 val_178 179 val_179 val_179 18 val_18 val_18 180 val_180 val_180 181 val_181 val_181 183 val_183 val_183 186 val_186 val_186 187 val_187 val_187 189 val_189 val_189 19 val_19 val_19 190 val_190 val_190 191 val_191 val_191 192 val_192 val_192 193 val_193 val_193 194 val_194 val_194 195 val_195 val_195 196 val_196 val_196 197 val_197 val_197 199 val_199 val_199 2 val_2 val_2 20 val_20 val_20 200 val_200 val_200 201 val_201 val_201 202 val_202 val_202 203 val_203 val_203 205 val_205 val_205 207 val_207 val_207 208 val_208 val_208 209 val_209 val_209 213 val_213 val_213 214 val_214 val_214 216 val_216 val_216 217 val_217 val_217 218 val_218 val_218 219 val_219 val_219 221 val_221 val_221 222 val_222 val_222 223 val_223 val_223 224 val_224 val_224 226 val_226 val_226 228 val_228 val_228 229 val_229 val_229 230 val_230 val_230 233 val_233 val_233 235 val_235 val_235 237 val_237 val_237 238 val_238 val_238 239 val_239 val_239 24 val_24 val_24 241 val_241 val_241 242 val_242 val_242 244 val_244 val_244 247 val_247 val_247 248 val_248 val_248 249 val_249 val_249 252 val_252 val_252 255 val_255 val_255 256 val_256 val_256 257 val_257 val_257 258 val_258 val_258 26 val_26 val_26 260 val_260 val_260 262 val_262 val_262 263 val_263 val_263 265 val_265 val_265 266 val_266 val_266 27 val_27 val_27 272 val_272 val_272 273 val_273 val_273 274 val_274 val_274 275 val_275 val_275 277 val_277 val_277 278 val_278 val_278 28 val_28 val_28 280 val_280 val_280 281 val_281 val_281 282 val_282 val_282 283 val_283 val_283 284 val_284 val_284 285 val_285 val_285 286 val_286 val_286 287 val_287 val_287 288 val_288 val_288 289 val_289 val_289 291 val_291 val_291 292 val_292 val_292 296 val_296 val_296 298 val_298 val_298 30 val_30 val_30 302 val_302 val_302 305 val_305 val_305 306 val_306 val_306 307 val_307 val_307 308 val_308 val_308 309 val_309 val_309 310 val_310 val_310 311 val_311 val_311 315 val_315 val_315 316 val_316 val_316 317 val_317 val_317 318 val_318 val_318 321 val_321 val_321 322 val_322 val_322 323 val_323 val_323 325 val_325 val_325 327 val_327 val_327 33 val_33 val_33 331 val_331 val_331 332 val_332 val_332 333 val_333 val_333 335 val_335 val_335 336 val_336 val_336 338 val_338 val_338 339 val_339 val_339 34 val_34 val_34 341 val_341 val_341 342 val_342 val_342 344 val_344 val_344 345 val_345 val_345 348 val_348 val_348 35 val_35 val_35 351 val_351 val_351 353 val_353 val_353 356 val_356 val_356 360 val_360 val_360 362 val_362 val_362 364 val_364 val_364 365 val_365 val_365 366 val_366 val_366 367 val_367 val_367 368 val_368 val_368 369 val_369 val_369 37 val_37 val_37 373 val_373 val_373 374 val_374 val_374 375 val_375 val_375 377 val_377 val_377 378 val_378 val_378 379 val_379 val_379 382 val_382 val_382 384 val_384 val_384 386 val_386 val_386 389 val_389 val_389 392 val_392 val_392 393 val_393 val_393 394 val_394 val_394 395 val_395 val_395 396 val_396 val_396 397 val_397 val_397 399 val_399 val_399 4 val_4 val_4 400 val_400 val_400 401 val_401 val_401 402 val_402 val_402 403 val_403 val_403 404 val_404 val_404 406 val_406 val_406 407 val_407 val_407 409 val_409 val_409 41 val_41 val_41 411 val_411 val_411 413 val_413 val_413 414 val_414 val_414 417 val_417 val_417 418 val_418 val_418 419 val_419 val_419 42 val_42 val_42 421 val_421 val_421 424 val_424 val_424 427 val_427 val_427 429 val_429 val_429 43 val_43 val_43 430 val_430 val_430 431 val_431 val_431 432 val_432 val_432 435 val_435 val_435 436 val_436 val_436 437 val_437 val_437 438 val_438 val_438 439 val_439 val_439 44 val_44 val_44 443 val_443 val_443 444 val_444 val_444 446 val_446 val_446 448 val_448 val_448 449 val_449 val_449 452 val_452 val_452 453 val_453 val_453 454 val_454 val_454 455 val_455 val_455 457 val_457 val_457 458 val_458 val_458 459 val_459 val_459 460 val_460 val_460 462 val_462 val_462 463 val_463 val_463 466 val_466 val_466 467 val_467 val_467 468 val_468 val_468 469 val_469 val_469 47 val_47 val_47 470 val_470 val_470 472 val_472 val_472 475 val_475 val_475 477 val_477 val_477 478 val_478 val_478 479 val_479 val_479 480 val_480 val_480 481 val_481 val_481 482 val_482 val_482 483 val_483 val_483 484 val_484 val_484 485 val_485 val_485 487 val_487 val_487 489 val_489 val_489 490 val_490 val_490 491 val_491 val_491 492 val_492 val_492 493 val_493 val_493 494 val_494 val_494 495 val_495 val_495 496 val_496 val_496 497 val_497 val_497 498 val_498 val_498 5 val_5 val_5 51 val_51 val_51 53 val_53 val_53 54 val_54 val_54 57 val_57 val_57 58 val_58 val_58 64 val_64 val_64 65 val_65 val_65 66 val_66 val_66 67 val_67 val_67 69 val_69 val_69 70 val_70 val_70 72 val_72 val_72 74 val_74 val_74 76 val_76 val_76 77 val_77 val_77 78 val_78 val_78 8 val_8 val_8 80 val_80 val_80 82 val_82 val_82 83 val_83 val_83 84 val_84 val_84 85 val_85 val_85 86 val_86 val_86 87 val_87 val_87 9 val_9 val_9 90 val_90 val_90 92 val_92 val_92 95 val_95 val_95 96 val_96 val_96 97 val_97 val_97 98 val_98 val_98 tst1 500 500 PREHOOK: query: -- union2.q -- SORT_BEFORE_DIFF -- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink explain select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc PREHOOK: type: QUERY POSTHOOK: query: -- union2.q -- SORT_BEFORE_DIFF -- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink explain select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 5 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 5 Map Operator Tree: TableScan alias: s1 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 309 PREHOOK: query: -- union20.q -- SORT_BEFORE_DIFF -- union :map-reduce sub-queries followed by join explain SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 JOIN (select 'tst1' as key, cast(count(1) as string) as value from src s3 UNION DISTINCT select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 ON (unionsrc1.key = unionsrc2.key) PREHOOK: type: QUERY POSTHOOK: query: -- union20.q -- SORT_BEFORE_DIFF -- union :map-reduce sub-queries followed by join explain SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 JOIN (select 'tst1' as key, cast(count(1) as string) as value from src s3 UNION DISTINCT select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 ON (unionsrc1.key = unionsrc2.key) POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 11 <- Union 9 (CONTAINS) Map 6 <- Union 3 (CONTAINS) Reducer 10 <- Union 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 9 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 11 Map Operator Tree: TableScan alias: s1 Filter Operator predicate: (UDFToDouble(key) < 10.0) (type: boolean) Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 6 Map Operator Tree: TableScan alias: s1 Filter Operator predicate: (UDFToDouble(key) < 10.0) (type: boolean) Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 7 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reducer 10 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL value expressions: _col1 (type: string) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL value expressions: _col1 (type: string) Reducer 5 Reduce Operator Tree: Merge Join Operator condition map: Inner Join 0 to 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 8 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Union 3 Vertex: Union 3 Union 9 Vertex: Union 9 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 JOIN (select 'tst1' as key, cast(count(1) as string) as value from src s3 UNION DISTINCT select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 ON (unionsrc1.key = unionsrc2.key) PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 JOIN (select 'tst1' as key, cast(count(1) as string) as value from src s3 UNION DISTINCT select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 ON (unionsrc1.key = unionsrc2.key) POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 0 val_0 0 val_0 2 val_2 2 val_2 4 val_4 4 val_4 5 val_5 5 val_5 8 val_8 8 val_8 9 val_9 9 val_9 tst1 500 tst1 500 PREHOOK: query: -- union21.q -- SORT_BEFORE_DIFF -- union of constants, udf outputs, and columns from text table and thrift table explain SELECT key, count(1) FROM ( SELECT '1' as key from src UNION DISTINCT SELECT reverse(key) as key from src UNION DISTINCT SELECT key as key from src UNION DISTINCT SELECT astring as key from src_thrift UNION DISTINCT SELECT lstring[0] as key from src_thrift ) union_output GROUP BY key PREHOOK: type: QUERY POSTHOOK: query: -- union21.q -- SORT_BEFORE_DIFF -- union of constants, udf outputs, and columns from text table and thrift table explain SELECT key, count(1) FROM ( SELECT '1' as key from src UNION DISTINCT SELECT reverse(key) as key from src UNION DISTINCT SELECT key as key from src UNION DISTINCT SELECT astring as key from src_thrift UNION DISTINCT SELECT lstring[0] as key from src_thrift ) union_output GROUP BY key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 10 <- Union 2 (CONTAINS) Map 11 <- Union 4 (CONTAINS) Map 12 <- Union 6 (CONTAINS) Map 13 <- Union 8 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) Reducer 9 <- Union 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src Select Operator expressions: '1' (type: string) outputColumnNames: _col0 Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Map 10 Map Operator Tree: TableScan alias: src Select Operator expressions: reverse(key) (type: string) outputColumnNames: _col0 Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Map 11 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string) outputColumnNames: _col0 Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Map 12 Map Operator Tree: TableScan alias: src_thrift Select Operator expressions: astring (type: string) outputColumnNames: _col0 Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Map 13 Map Operator Tree: TableScan alias: src_thrift Select Operator expressions: lstring[0] (type: string) outputColumnNames: _col0 Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Reducer 5 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Reducer 7 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Reducer 9 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 Union 4 Vertex: Union 4 Union 6 Vertex: Union 6 Union 8 Vertex: Union 8 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: SELECT key, count(1) FROM ( SELECT '1' as key from src UNION DISTINCT SELECT reverse(key) as key from src UNION DISTINCT SELECT key as key from src UNION DISTINCT SELECT astring as key from src_thrift UNION DISTINCT SELECT lstring[0] as key from src_thrift ) union_output GROUP BY key PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@src_thrift #### A masked pattern was here #### POSTHOOK: query: SELECT key, count(1) FROM ( SELECT '1' as key from src UNION DISTINCT SELECT reverse(key) as key from src UNION DISTINCT SELECT key as key from src UNION DISTINCT SELECT astring as key from src_thrift UNION DISTINCT SELECT lstring[0] as key from src_thrift ) union_output GROUP BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src_thrift #### A masked pattern was here #### NULL 1 0 1 001 1 002 1 004 1 01 1 013 1 02 1 021 1 03 1 032 1 034 1 051 1 061 1 062 1 063 1 064 1 07 1 071 1 074 1 08 1 081 1 082 1 084 1 09 1 091 1 094 1 1 1 10 1 100 1 102 1 103 1 104 1 105 1 11 1 111 1 113 1 114 1 116 1 118 1 119 1 12 1 120 1 122 1 123 1 124 1 125 1 126 1 128 1 129 1 131 1 133 1 134 1 136 1 137 1 138 1 14 1 142 1 143 1 145 1 146 1 149 1 15 1 150 1 152 1 153 1 155 1 156 1 157 1 158 1 160 1 162 1 163 1 164 1 165 1 166 1 167 1 168 1 169 1 17 1 170 1 172 1 174 1 175 1 176 1 177 1 178 1 179 1 18 1 180 1 181 1 182 1 183 1 184 1 186 1 187 1 189 1 19 1 190 1 191 1 192 1 193 1 194 1 195 1 196 1 197 1 199 1 2 1 20 1 200 1 201 1 202 1 203 1 204 1 205 1 207 1 208 1 209 1 21 1 213 1 214 1 216 1 217 1 218 1 219 1 221 1 222 1 223 1 224 1 226 1 228 1 229 1 230 1 233 1 234 1 235 1 237 1 238 1 239 1 24 1 241 1 242 1 243 1 244 1 247 1 248 1 249 1 251 1 252 1 254 1 255 1 256 1 257 1 258 1 26 1 260 1 261 1 262 1 263 1 264 1 265 1 266 1 27 1 271 1 272 1 273 1 274 1 275 1 277 1 278 1 28 1 280 1 281 1 282 1 283 1 284 1 285 1 286 1 287 1 288 1 289 1 29 1 291 1 292 1 293 1 294 1 296 1 298 1 30 1 301 1 302 1 304 1 305 1 306 1 307 1 308 1 309 1 310 1 311 1 312 1 314 1 315 1 316 1 317 1 318 1 321 1 322 1 323 1 325 1 327 1 33 1 331 1 332 1 333 1 335 1 336 1 338 1 339 1 34 1 341 1 342 1 344 1 345 1 348 1 35 1 351 1 353 1 354 1 356 1 360 1 361 1 362 1 364 1 365 1 366 1 367 1 368 1 369 1 37 1 372 1 373 1 374 1 375 1 377 1 378 1 379 1 38 1 381 1 382 1 384 1 386 1 389 1 391 1 392 1 393 1 394 1 395 1 396 1 397 1 399 1 4 1 40 1 400 1 401 1 402 1 403 1 404 1 406 1 407 1 409 1 41 1 411 1 412 1 413 1 414 1 417 1 418 1 419 1 42 1 421 1 422 1 424 1 427 1 429 1 43 1 430 1 431 1 432 1 435 1 436 1 437 1 438 1 439 1 44 1 442 1 443 1 444 1 446 1 448 1 449 1 45 1 452 1 453 1 454 1 455 1 457 1 458 1 459 1 46 1 460 1 461 1 462 1 463 1 466 1 467 1 468 1 469 1 47 1 470 1 471 1 472 1 473 1 475 1 477 1 478 1 479 1 48 1 480 1 481 1 482 1 483 1 484 1 485 1 487 1 489 1 490 1 491 1 492 1 493 1 494 1 495 1 496 1 497 1 498 1 5 1 50 1 501 1 502 1 503 1 51 1 513 1 521 1 523 1 53 1 532 1 533 1 534 1 54 1 541 1 543 1 551 1 552 1 554 1 56 1 561 1 562 1 563 1 57 1 571 1 572 1 573 1 574 1 58 1 582 1 584 1 59 1 591 1 593 1 594 1 60 1 603 1 604 1 611 1 612 1 613 1 62 1 621 1 622 1 631 1 633 1 634 1 64 1 641 1 644 1 65 1 651 1 652 1 653 1 66 1 661 1 662 1 663 1 664 1 67 1 671 1 68 1 681 1 682 1 683 1 69 1 691 1 692 1 693 1 694 1 70 1 702 1 703 1 704 1 71 1 712 1 713 1 714 1 72 1 723 1 724 1 73 1 731 1 732 1 734 1 74 1 742 1 75 1 751 1 752 1 754 1 76 1 761 1 763 1 764 1 77 1 771 1 772 1 773 1 774 1 78 1 781 1 782 1 784 1 79 1 791 1 793 1 794 1 8 1 80 1 802 1 803 1 81 1 811 1 812 1 813 1 814 1 82 1 821 1 822 1 83 1 831 1 832 1 833 1 834 1 84 1 842 1 843 1 844 1 85 1 851 1 852 1 854 1 86 1 861 1 863 1 864 1 87 1 871 1 872 1 873 1 874 1 882 1 89 1 892 1 894 1 9 1 90 1 902 1 903 1 904 1 91 1 911 1 912 1 914 1 92 1 921 1 922 1 924 1 932 1 933 1 934 1 941 1 942 1 944 1 95 1 954 1 96 1 961 1 963 1 964 1 97 1 971 1 973 1 974 1 98 1 981 1 982 1 983 1 984 1 991 1 993 1 record_0 1 record_1 1 record_2 1 record_3 1 record_4 1 record_5 1 record_6 1 record_7 1 record_8 1 record_9 1 PREHOOK: query: -- union22.q -- SORT_QUERY_RESULTS create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@dst_union22 POSTHOOK: query: -- union22.q -- SORT_QUERY_RESULTS create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dst_union22 PREHOOK: query: create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@dst_union22_delta POSTHOOK: query: create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dst_union22_delta PREHOOK: query: insert overwrite table dst_union22 partition (ds='1') select key, value, key , value from src PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@dst_union22@ds=1 POSTHOOK: query: insert overwrite table dst_union22 partition (ds='1') select key, value, key , value from src POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dst_union22@ds=1 POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: insert overwrite table dst_union22_delta partition (ds='1') select key, key, value, key, value, value from src PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@dst_union22_delta@ds=1 POSTHOOK: query: insert overwrite table dst_union22_delta partition (ds='1') select key, key, value, key, value, value from src POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dst_union22_delta@ds=1 POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k5 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: -- Since the inputs are small, it should be automatically converted to mapjoin explain extended insert overwrite table dst_union22 partition (ds='2') select * from ( select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 UNION DISTINCT select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on a.k1 = b.k1 and a.ds='1' where a.k1 > 20 ) subq PREHOOK: type: QUERY POSTHOOK: query: -- Since the inputs are small, it should be automatically converted to mapjoin explain extended insert overwrite table dst_union22 partition (ds='2') select * from ( select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 UNION DISTINCT select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on a.k1 = b.k1 and a.ds='1' where a.k1 > 20 ) subq POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY TOK_QUERY TOK_FROM TOK_SUBQUERY TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF TOK_TABNAME dst_union22_delta TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR TOK_TABLE_OR_COL k1 k1 TOK_SELEXPR TOK_TABLE_OR_COL k2 k2 TOK_SELEXPR TOK_TABLE_OR_COL k3 k3 TOK_SELEXPR TOK_TABLE_OR_COL k4 k4 TOK_WHERE and = TOK_TABLE_OR_COL ds '1' <= TOK_TABLE_OR_COL k0 50 TOK_QUERY TOK_FROM TOK_LEFTOUTERJOIN TOK_TABREF TOK_TABNAME dst_union22 a TOK_SUBQUERY TOK_QUERY TOK_FROM TOK_TABREF TOK_TABNAME dst_union22_delta TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR TOK_ALLCOLREF TOK_WHERE and = TOK_TABLE_OR_COL ds '1' > TOK_TABLE_OR_COL k0 50 b and = . TOK_TABLE_OR_COL a k1 . TOK_TABLE_OR_COL b k1 = . TOK_TABLE_OR_COL a ds '1' TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR . TOK_TABLE_OR_COL a k1 k1 TOK_SELEXPR . TOK_TABLE_OR_COL a k2 k2 TOK_SELEXPR . TOK_TABLE_OR_COL b k3 k3 TOK_SELEXPR . TOK_TABLE_OR_COL b k4 k4 TOK_WHERE > . TOK_TABLE_OR_COL a k1 20 _u1 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECTDI TOK_SELEXPR TOK_ALLCOLREF subq TOK_INSERT TOK_DESTINATION TOK_TAB TOK_TABNAME dst_union22 TOK_PARTSPEC TOK_PARTVAL ds '2' TOK_SELECT TOK_SELEXPR TOK_ALLCOLREF STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 5 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: dst_union22_delta GatherStats: false Filter Operator isSamplingPred: false predicate: (k0 <= 50) (type: boolean) Select Operator expressions: k1 (type: string), k2 (type: string), k3 (type: string), k4 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Group By Operator keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) tag: -1 auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: ds=1 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: ds 1 properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns k0,k1,k2,k3,k4,k5 columns.comments columns.types string:string:string:string:string:string #### A masked pattern was here #### name default.dst_union22_delta numFiles 1 numRows 500 partition_columns ds partition_columns.types string rawDataSize 16936 serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 17436 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 columns k0,k1,k2,k3,k4,k5 columns.comments columns.types string:string:string:string:string:string #### A masked pattern was here #### name default.dst_union22_delta partition_columns ds partition_columns.types string serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dst_union22_delta name: default.dst_union22_delta Truncated Path -> Alias: /dst_union22_delta/ds=1 [dst_union22_delta] Map 4 Map Operator Tree: TableScan alias: dst_union22_delta Statistics: Num rows: 500 Data size: 16936 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: ((k0 > 50) and (k1 > 20)) (type: boolean) Statistics: Num rows: 55 Data size: 1862 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: k1 (type: string), k3 (type: string), k4 (type: string) outputColumnNames: _col1, _col3, _col4 Statistics: Num rows: 55 Data size: 1862 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 55 Data size: 1862 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col3 (type: string), _col4 (type: string) auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: ds=1 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: ds 1 properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns k0,k1,k2,k3,k4,k5 columns.comments columns.types string:string:string:string:string:string #### A masked pattern was here #### name default.dst_union22_delta numFiles 1 numRows 500 partition_columns ds partition_columns.types string rawDataSize 16936 serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 17436 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 columns k0,k1,k2,k3,k4,k5 columns.comments columns.types string:string:string:string:string:string #### A masked pattern was here #### name default.dst_union22_delta partition_columns ds partition_columns.types string serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dst_union22_delta name: default.dst_union22_delta Truncated Path -> Alias: /dst_union22_delta/ds=1 [dst_union22_delta] Map 5 Map Operator Tree: TableScan alias: a GatherStats: false Filter Operator isSamplingPred: false predicate: (k1 > 20) (type: boolean) Map Join Operator condition map: Left Outer Join0 to 1 filter mappings: 0 [1, 1] filter predicates: 0 {(ds = '1')} 1 Estimated key counts: Map 4 => 55 keys: 0 k1 (type: string) 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col11, _col12 input vertices: 1 Map 4 Position of Big Table: 0 Select Operator expressions: _col0 (type: string), _col1 (type: string), _col11 (type: string), _col12 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Group By Operator keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) tag: -1 auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: ds=1 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: ds 1 properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns k1,k2,k3,k4 columns.comments columns.types string:string:string:string #### A masked pattern was here #### name default.dst_union22 numFiles 1 numRows 500 partition_columns ds partition_columns.types string rawDataSize 11124 serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 columns k1,k2,k3,k4 columns.comments columns.types string:string:string:string #### A masked pattern was here #### name default.dst_union22 partition_columns ds partition_columns.types string serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dst_union22 name: default.dst_union22 Truncated Path -> Alias: /dst_union22/ds=1 [a] Reducer 3 Needs Tagging: false Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 174 Data size: 4842 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2/ Statistics: Num rows: 174 Data size: 4842 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 columns k1,k2,k3,k4 columns.comments columns.types string:string:string:string #### A masked pattern was here #### name default.dst_union22 partition_columns ds partition_columns.types string serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dst_union22 TotalFiles: 1 GatherStats: true MultiFileSpray: false Union 2 Vertex: Union 2 Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: partition: ds 2 replace: true #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 columns k1,k2,k3,k4 columns.comments columns.types string:string:string:string #### A masked pattern was here #### name default.dst_union22 partition_columns ds partition_columns.types string serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dst_union22 Stage: Stage-3 Stats-Aggr Operator #### A masked pattern was here #### PREHOOK: query: insert overwrite table dst_union22 partition (ds='2') select * from ( select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 UNION DISTINCT select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on a.k1 = b.k1 and a.ds='1' where a.k1 > 20 ) subq PREHOOK: type: QUERY PREHOOK: Input: default@dst_union22 PREHOOK: Input: default@dst_union22@ds=1 PREHOOK: Input: default@dst_union22_delta PREHOOK: Input: default@dst_union22_delta@ds=1 PREHOOK: Output: default@dst_union22@ds=2 POSTHOOK: query: insert overwrite table dst_union22 partition (ds='2') select * from ( select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 UNION DISTINCT select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on a.k1 = b.k1 and a.ds='1' where a.k1 > 20 ) subq POSTHOOK: type: QUERY POSTHOOK: Input: default@dst_union22 POSTHOOK: Input: default@dst_union22@ds=1 POSTHOOK: Input: default@dst_union22_delta POSTHOOK: Input: default@dst_union22_delta@ds=1 POSTHOOK: Output: default@dst_union22@ds=2 POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k1 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k1, type:string, comment:null), (dst_union22)a.FieldSchema(name:k1, type:string, comment:null), ] POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k2 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k2, type:string, comment:null), (dst_union22)a.FieldSchema(name:k2, type:string, comment:null), ] POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k3 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k3, type:string, comment:null), (dst_union22_delta)dst_union22_delta.FieldSchema(name:k3, type:string, comment:null), ] POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k4 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k4, type:string, comment:null), (dst_union22_delta)dst_union22_delta.FieldSchema(name:k4, type:string, comment:null), ] PREHOOK: query: select * from dst_union22 where ds = '2' PREHOOK: type: QUERY PREHOOK: Input: default@dst_union22 PREHOOK: Input: default@dst_union22@ds=2 #### A masked pattern was here #### POSTHOOK: query: select * from dst_union22 where ds = '2' POSTHOOK: type: QUERY POSTHOOK: Input: default@dst_union22 POSTHOOK: Input: default@dst_union22@ds=2 #### A masked pattern was here #### 0 val_0 0 val_0 2 10 val_10 10 val_10 2 100 val_100 100 val_100 2 103 val_103 103 val_103 2 104 val_104 104 val_104 2 105 val_105 105 val_105 2 11 val_11 11 val_11 2 111 val_111 111 val_111 2 113 val_113 113 val_113 2 114 val_114 114 val_114 2 116 val_116 116 val_116 2 118 val_118 118 val_118 2 119 val_119 119 val_119 2 12 val_12 12 val_12 2 120 val_120 120 val_120 2 125 val_125 125 val_125 2 126 val_126 126 val_126 2 128 val_128 128 val_128 2 129 val_129 129 val_129 2 131 val_131 131 val_131 2 133 val_133 133 val_133 2 134 val_134 134 val_134 2 136 val_136 136 val_136 2 137 val_137 137 val_137 2 138 val_138 138 val_138 2 143 val_143 143 val_143 2 145 val_145 145 val_145 2 146 val_146 146 val_146 2 149 val_149 149 val_149 2 15 val_15 15 val_15 2 150 val_150 150 val_150 2 152 val_152 152 val_152 2 153 val_153 153 val_153 2 155 val_155 155 val_155 2 156 val_156 156 val_156 2 157 val_157 157 val_157 2 158 val_158 158 val_158 2 160 val_160 160 val_160 2 162 val_162 162 val_162 2 163 val_163 163 val_163 2 164 val_164 164 val_164 2 165 val_165 165 val_165 2 166 val_166 166 val_166 2 167 val_167 167 val_167 2 168 val_168 168 val_168 2 169 val_169 169 val_169 2 17 val_17 17 val_17 2 170 val_170 170 val_170 2 172 val_172 172 val_172 2 174 val_174 174 val_174 2 175 val_175 175 val_175 2 176 val_176 176 val_176 2 177 val_177 177 val_177 2 178 val_178 178 val_178 2 179 val_179 179 val_179 2 18 val_18 18 val_18 2 180 val_180 180 val_180 2 181 val_181 181 val_181 2 183 val_183 183 val_183 2 186 val_186 186 val_186 2 187 val_187 187 val_187 2 189 val_189 189 val_189 2 19 val_19 19 val_19 2 190 val_190 190 val_190 2 191 val_191 191 val_191 2 192 val_192 192 val_192 2 193 val_193 193 val_193 2 194 val_194 194 val_194 2 195 val_195 195 val_195 2 196 val_196 196 val_196 2 197 val_197 197 val_197 2 199 val_199 199 val_199 2 2 val_2 2 val_2 2 20 val_20 20 val_20 2 200 val_200 200 val_200 2 201 val_201 201 val_201 2 202 val_202 202 val_202 2 203 val_203 203 val_203 2 205 val_205 205 val_205 2 207 val_207 207 val_207 2 208 val_208 208 val_208 2 209 val_209 209 val_209 2 213 val_213 213 val_213 2 214 val_214 214 val_214 2 216 val_216 216 val_216 2 217 val_217 217 val_217 2 218 val_218 218 val_218 2 219 val_219 219 val_219 2 221 val_221 221 val_221 2 222 val_222 222 val_222 2 223 val_223 223 val_223 2 224 val_224 224 val_224 2 226 val_226 226 val_226 2 228 val_228 228 val_228 2 229 val_229 229 val_229 2 230 val_230 230 val_230 2 233 val_233 233 val_233 2 235 val_235 235 val_235 2 237 val_237 237 val_237 2 238 val_238 238 val_238 2 239 val_239 239 val_239 2 24 val_24 NULL NULL 2 24 val_24 24 val_24 2 241 val_241 241 val_241 2 242 val_242 242 val_242 2 244 val_244 244 val_244 2 247 val_247 247 val_247 2 248 val_248 248 val_248 2 249 val_249 249 val_249 2 252 val_252 252 val_252 2 255 val_255 255 val_255 2 256 val_256 256 val_256 2 257 val_257 257 val_257 2 258 val_258 258 val_258 2 26 val_26 NULL NULL 2 26 val_26 26 val_26 2 260 val_260 260 val_260 2 262 val_262 262 val_262 2 263 val_263 263 val_263 2 265 val_265 265 val_265 2 266 val_266 266 val_266 2 27 val_27 NULL NULL 2 27 val_27 27 val_27 2 272 val_272 272 val_272 2 273 val_273 273 val_273 2 274 val_274 274 val_274 2 275 val_275 275 val_275 2 277 val_277 277 val_277 2 278 val_278 278 val_278 2 28 val_28 NULL NULL 2 28 val_28 28 val_28 2 280 val_280 280 val_280 2 281 val_281 281 val_281 2 282 val_282 282 val_282 2 283 val_283 283 val_283 2 284 val_284 284 val_284 2 285 val_285 285 val_285 2 286 val_286 286 val_286 2 287 val_287 287 val_287 2 288 val_288 288 val_288 2 289 val_289 289 val_289 2 291 val_291 291 val_291 2 292 val_292 292 val_292 2 296 val_296 296 val_296 2 298 val_298 298 val_298 2 30 val_30 NULL NULL 2 30 val_30 30 val_30 2 302 val_302 302 val_302 2 305 val_305 305 val_305 2 306 val_306 306 val_306 2 307 val_307 307 val_307 2 308 val_308 308 val_308 2 309 val_309 309 val_309 2 310 val_310 310 val_310 2 311 val_311 311 val_311 2 315 val_315 315 val_315 2 316 val_316 316 val_316 2 317 val_317 317 val_317 2 318 val_318 318 val_318 2 321 val_321 321 val_321 2 322 val_322 322 val_322 2 323 val_323 323 val_323 2 325 val_325 325 val_325 2 327 val_327 327 val_327 2 33 val_33 NULL NULL 2 33 val_33 33 val_33 2 331 val_331 331 val_331 2 332 val_332 332 val_332 2 333 val_333 333 val_333 2 335 val_335 335 val_335 2 336 val_336 336 val_336 2 338 val_338 338 val_338 2 339 val_339 339 val_339 2 34 val_34 NULL NULL 2 34 val_34 34 val_34 2 341 val_341 341 val_341 2 342 val_342 342 val_342 2 344 val_344 344 val_344 2 345 val_345 345 val_345 2 348 val_348 348 val_348 2 35 val_35 NULL NULL 2 35 val_35 35 val_35 2 351 val_351 351 val_351 2 353 val_353 353 val_353 2 356 val_356 356 val_356 2 360 val_360 360 val_360 2 362 val_362 362 val_362 2 364 val_364 364 val_364 2 365 val_365 365 val_365 2 366 val_366 366 val_366 2 367 val_367 367 val_367 2 368 val_368 368 val_368 2 369 val_369 369 val_369 2 37 val_37 NULL NULL 2 37 val_37 37 val_37 2 373 val_373 373 val_373 2 374 val_374 374 val_374 2 375 val_375 375 val_375 2 377 val_377 377 val_377 2 378 val_378 378 val_378 2 379 val_379 379 val_379 2 382 val_382 382 val_382 2 384 val_384 384 val_384 2 386 val_386 386 val_386 2 389 val_389 389 val_389 2 392 val_392 392 val_392 2 393 val_393 393 val_393 2 394 val_394 394 val_394 2 395 val_395 395 val_395 2 396 val_396 396 val_396 2 397 val_397 397 val_397 2 399 val_399 399 val_399 2 4 val_4 4 val_4 2 400 val_400 400 val_400 2 401 val_401 401 val_401 2 402 val_402 402 val_402 2 403 val_403 403 val_403 2 404 val_404 404 val_404 2 406 val_406 406 val_406 2 407 val_407 407 val_407 2 409 val_409 409 val_409 2 41 val_41 NULL NULL 2 41 val_41 41 val_41 2 411 val_411 411 val_411 2 413 val_413 413 val_413 2 414 val_414 414 val_414 2 417 val_417 417 val_417 2 418 val_418 418 val_418 2 419 val_419 419 val_419 2 42 val_42 NULL NULL 2 42 val_42 42 val_42 2 421 val_421 421 val_421 2 424 val_424 424 val_424 2 427 val_427 427 val_427 2 429 val_429 429 val_429 2 43 val_43 NULL NULL 2 43 val_43 43 val_43 2 430 val_430 430 val_430 2 431 val_431 431 val_431 2 432 val_432 432 val_432 2 435 val_435 435 val_435 2 436 val_436 436 val_436 2 437 val_437 437 val_437 2 438 val_438 438 val_438 2 439 val_439 439 val_439 2 44 val_44 NULL NULL 2 44 val_44 44 val_44 2 443 val_443 443 val_443 2 444 val_444 444 val_444 2 446 val_446 446 val_446 2 448 val_448 448 val_448 2 449 val_449 449 val_449 2 452 val_452 452 val_452 2 453 val_453 453 val_453 2 454 val_454 454 val_454 2 455 val_455 455 val_455 2 457 val_457 457 val_457 2 458 val_458 458 val_458 2 459 val_459 459 val_459 2 460 val_460 460 val_460 2 462 val_462 462 val_462 2 463 val_463 463 val_463 2 466 val_466 466 val_466 2 467 val_467 467 val_467 2 468 val_468 468 val_468 2 469 val_469 469 val_469 2 47 val_47 NULL NULL 2 47 val_47 47 val_47 2 470 val_470 470 val_470 2 472 val_472 472 val_472 2 475 val_475 475 val_475 2 477 val_477 477 val_477 2 478 val_478 478 val_478 2 479 val_479 479 val_479 2 480 val_480 480 val_480 2 481 val_481 481 val_481 2 482 val_482 482 val_482 2 483 val_483 483 val_483 2 484 val_484 484 val_484 2 485 val_485 485 val_485 2 487 val_487 487 val_487 2 489 val_489 489 val_489 2 490 val_490 490 val_490 2 491 val_491 491 val_491 2 492 val_492 492 val_492 2 493 val_493 493 val_493 2 494 val_494 494 val_494 2 495 val_495 495 val_495 2 496 val_496 496 val_496 2 497 val_497 497 val_497 2 498 val_498 498 val_498 2 5 val_5 5 val_5 2 51 val_51 51 val_51 2 53 val_53 53 val_53 2 54 val_54 54 val_54 2 57 val_57 57 val_57 2 58 val_58 58 val_58 2 64 val_64 64 val_64 2 65 val_65 65 val_65 2 66 val_66 66 val_66 2 67 val_67 67 val_67 2 69 val_69 69 val_69 2 70 val_70 70 val_70 2 72 val_72 72 val_72 2 74 val_74 74 val_74 2 76 val_76 76 val_76 2 77 val_77 77 val_77 2 78 val_78 78 val_78 2 8 val_8 8 val_8 2 80 val_80 80 val_80 2 82 val_82 82 val_82 2 83 val_83 83 val_83 2 84 val_84 84 val_84 2 85 val_85 85 val_85 2 86 val_86 86 val_86 2 87 val_87 87 val_87 2 9 val_9 9 val_9 2 90 val_90 90 val_90 2 92 val_92 92 val_92 2 95 val_95 95 val_95 2 96 val_96 96 val_96 2 97 val_97 97 val_97 2 98 val_98 98 val_98 2 PREHOOK: query: -- union23.q explain select s.key2, s.value2 from ( select transform(key, value) using 'cat' as (key2, value2) from src UNION DISTINCT select key as key2, value as value2 from src) s order by s.key2, s.value2 PREHOOK: type: QUERY POSTHOOK: query: -- union23.q explain select s.key2, s.value2 from ( select transform(key, value) using 'cat' as (key2, value2) from src UNION DISTINCT select key as key2, value as value2 from src) s order by s.key2, s.value2 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 5 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Transform Operator command: cat output info: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 5 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reducer 4 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select s.key2, s.value2 from ( select transform(key, value) using 'cat' as (key2, value2) from src UNION DISTINCT select key as key2, value as value2 from src) s order by s.key2, s.value2 PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: select s.key2, s.value2 from ( select transform(key, value) using 'cat' as (key2, value2) from src UNION DISTINCT select key as key2, value as value2 from src) s order by s.key2, s.value2 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 0 val_0 10 val_10 100 val_100 103 val_103 104 val_104 105 val_105 11 val_11 111 val_111 113 val_113 114 val_114 116 val_116 118 val_118 119 val_119 12 val_12 120 val_120 125 val_125 126 val_126 128 val_128 129 val_129 131 val_131 133 val_133 134 val_134 136 val_136 137 val_137 138 val_138 143 val_143 145 val_145 146 val_146 149 val_149 15 val_15 150 val_150 152 val_152 153 val_153 155 val_155 156 val_156 157 val_157 158 val_158 160 val_160 162 val_162 163 val_163 164 val_164 165 val_165 166 val_166 167 val_167 168 val_168 169 val_169 17 val_17 170 val_170 172 val_172 174 val_174 175 val_175 176 val_176 177 val_177 178 val_178 179 val_179 18 val_18 180 val_180 181 val_181 183 val_183 186 val_186 187 val_187 189 val_189 19 val_19 190 val_190 191 val_191 192 val_192 193 val_193 194 val_194 195 val_195 196 val_196 197 val_197 199 val_199 2 val_2 20 val_20 200 val_200 201 val_201 202 val_202 203 val_203 205 val_205 207 val_207 208 val_208 209 val_209 213 val_213 214 val_214 216 val_216 217 val_217 218 val_218 219 val_219 221 val_221 222 val_222 223 val_223 224 val_224 226 val_226 228 val_228 229 val_229 230 val_230 233 val_233 235 val_235 237 val_237 238 val_238 239 val_239 24 val_24 241 val_241 242 val_242 244 val_244 247 val_247 248 val_248 249 val_249 252 val_252 255 val_255 256 val_256 257 val_257 258 val_258 26 val_26 260 val_260 262 val_262 263 val_263 265 val_265 266 val_266 27 val_27 272 val_272 273 val_273 274 val_274 275 val_275 277 val_277 278 val_278 28 val_28 280 val_280 281 val_281 282 val_282 283 val_283 284 val_284 285 val_285 286 val_286 287 val_287 288 val_288 289 val_289 291 val_291 292 val_292 296 val_296 298 val_298 30 val_30 302 val_302 305 val_305 306 val_306 307 val_307 308 val_308 309 val_309 310 val_310 311 val_311 315 val_315 316 val_316 317 val_317 318 val_318 321 val_321 322 val_322 323 val_323 325 val_325 327 val_327 33 val_33 331 val_331 332 val_332 333 val_333 335 val_335 336 val_336 338 val_338 339 val_339 34 val_34 341 val_341 342 val_342 344 val_344 345 val_345 348 val_348 35 val_35 351 val_351 353 val_353 356 val_356 360 val_360 362 val_362 364 val_364 365 val_365 366 val_366 367 val_367 368 val_368 369 val_369 37 val_37 373 val_373 374 val_374 375 val_375 377 val_377 378 val_378 379 val_379 382 val_382 384 val_384 386 val_386 389 val_389 392 val_392 393 val_393 394 val_394 395 val_395 396 val_396 397 val_397 399 val_399 4 val_4 400 val_400 401 val_401 402 val_402 403 val_403 404 val_404 406 val_406 407 val_407 409 val_409 41 val_41 411 val_411 413 val_413 414 val_414 417 val_417 418 val_418 419 val_419 42 val_42 421 val_421 424 val_424 427 val_427 429 val_429 43 val_43 430 val_430 431 val_431 432 val_432 435 val_435 436 val_436 437 val_437 438 val_438 439 val_439 44 val_44 443 val_443 444 val_444 446 val_446 448 val_448 449 val_449 452 val_452 453 val_453 454 val_454 455 val_455 457 val_457 458 val_458 459 val_459 460 val_460 462 val_462 463 val_463 466 val_466 467 val_467 468 val_468 469 val_469 47 val_47 470 val_470 472 val_472 475 val_475 477 val_477 478 val_478 479 val_479 480 val_480 481 val_481 482 val_482 483 val_483 484 val_484 485 val_485 487 val_487 489 val_489 490 val_490 491 val_491 492 val_492 493 val_493 494 val_494 495 val_495 496 val_496 497 val_497 498 val_498 5 val_5 51 val_51 53 val_53 54 val_54 57 val_57 58 val_58 64 val_64 65 val_65 66 val_66 67 val_67 69 val_69 70 val_70 72 val_72 74 val_74 76 val_76 77 val_77 78 val_78 8 val_8 80 val_80 82 val_82 83 val_83 84 val_84 85 val_85 86 val_86 87 val_87 9 val_9 90 val_90 92 val_92 95 val_95 96 val_96 97 val_97 98 val_98 PREHOOK: query: -- union24.q -- SORT_QUERY_RESULTS create table src2 as select key, count(1) as count from src group by key PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@src2 POSTHOOK: query: -- union24.q -- SORT_QUERY_RESULTS create table src2 as select key, count(1) as count from src group by key POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@src2 PREHOOK: query: create table src3 as select * from src2 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src2 PREHOOK: Output: database:default PREHOOK: Output: default@src3 POSTHOOK: query: create table src3 as select * from src2 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src2 POSTHOOK: Output: database:default POSTHOOK: Output: default@src3 PREHOOK: query: create table src4 as select * from src2 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src2 PREHOOK: Output: database:default PREHOOK: Output: default@src4 POSTHOOK: query: create table src4 as select * from src2 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src2 POSTHOOK: Output: database:default POSTHOOK: Output: default@src4 PREHOOK: query: create table src5 as select * from src2 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src2 PREHOOK: Output: database:default PREHOOK: Output: default@src5 POSTHOOK: query: create table src5 as select * from src2 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src2 POSTHOOK: Output: database:default POSTHOOK: Output: default@src5 PREHOOK: query: explain extended select s.key, s.count from ( select key, count from src2 where key < 10 UNION DISTINCT select key, count from src3 where key < 10 UNION DISTINCT select key, count from src4 where key < 10 UNION DISTINCT select key, count(1) as count from src5 where key < 10 group by key )s PREHOOK: type: QUERY POSTHOOK: query: explain extended select s.key, s.count from ( select key, count from src2 where key < 10 UNION DISTINCT select key, count from src3 where key < 10 UNION DISTINCT select key, count from src4 where key < 10 UNION DISTINCT select key, count(1) as count from src5 where key < 10 group by key )s POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY TOK_QUERY TOK_FROM TOK_SUBQUERY TOK_UNIONALL TOK_QUERY TOK_FROM TOK_SUBQUERY TOK_UNIONALL TOK_QUERY TOK_FROM TOK_SUBQUERY TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF TOK_TABNAME src2 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR TOK_TABLE_OR_COL key TOK_SELEXPR TOK_TABLE_OR_COL count TOK_WHERE < TOK_TABLE_OR_COL key 10 TOK_QUERY TOK_FROM TOK_TABREF TOK_TABNAME src3 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR TOK_TABLE_OR_COL key TOK_SELEXPR TOK_TABLE_OR_COL count TOK_WHERE < TOK_TABLE_OR_COL key 10 _u1 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECTDI TOK_SELEXPR TOK_ALLCOLREF TOK_QUERY TOK_FROM TOK_TABREF TOK_TABNAME src4 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR TOK_TABLE_OR_COL key TOK_SELEXPR TOK_TABLE_OR_COL count TOK_WHERE < TOK_TABLE_OR_COL key 10 _u2 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECTDI TOK_SELEXPR TOK_ALLCOLREF TOK_QUERY TOK_FROM TOK_TABREF TOK_TABNAME src5 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR TOK_TABLE_OR_COL key TOK_SELEXPR TOK_FUNCTION count 1 count TOK_WHERE < TOK_TABLE_OR_COL key 10 TOK_GROUPBY TOK_TABLE_OR_COL key _u3 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECTDI TOK_SELEXPR TOK_ALLCOLREF s TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR . TOK_TABLE_OR_COL s key TOK_SELEXPR . TOK_TABLE_OR_COL s count STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 8 <- Union 2 (CONTAINS) Map 9 <- Union 4 (CONTAINS) Reducer 11 <- Map 10 (SIMPLE_EDGE), Union 6 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) Reducer 7 <- Union 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src2 GatherStats: false Filter Operator isSamplingPred: false predicate: (UDFToDouble(key) < 10.0) (type: boolean) Select Operator expressions: key (type: string), count (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: src2 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src2 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src2 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src2 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src2 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src2 name: default.src2 Truncated Path -> Alias: /src2 [src2] Map 10 Map Operator Tree: TableScan alias: src5 Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (UDFToDouble(key) < 10.0) (type: boolean) Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: src5 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src5 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src5 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src5 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src5 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src5 name: default.src5 Truncated Path -> Alias: /src5 [src5] Map 8 Map Operator Tree: TableScan alias: src3 GatherStats: false Filter Operator isSamplingPred: false predicate: (UDFToDouble(key) < 10.0) (type: boolean) Select Operator expressions: key (type: string), count (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: src3 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src3 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src3 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src3 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src3 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src3 name: default.src3 Truncated Path -> Alias: /src3 [src3] Map 9 Map Operator Tree: TableScan alias: src4 GatherStats: false Filter Operator isSamplingPred: false predicate: (UDFToDouble(key) < 10.0) (type: boolean) Select Operator expressions: key (type: string), count (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: src4 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src4 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src4 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src4 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src4 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src4 name: default.src4 Truncated Path -> Alias: /src4 [src4] Reducer 11 Needs Tagging: false Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Reducer 5 Needs Tagging: false Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Reducer 7 Needs Tagging: false Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 77 Data size: 369 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 Statistics: Num rows: 77 Data size: 369 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: columns _col0,_col1 columns.types string:bigint escape.delim \ hive.serialization.extend.additional.nesting.levels true serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TotalFiles: 1 GatherStats: false MultiFileSpray: false Union 2 Vertex: Union 2 Union 4 Vertex: Union 4 Union 6 Vertex: Union 6 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select s.key, s.count from ( select key, count from src2 where key < 10 UNION DISTINCT select key, count from src3 where key < 10 UNION DISTINCT select key, count from src4 where key < 10 UNION DISTINCT select key, count(1) as count from src5 where key < 10 group by key )s PREHOOK: type: QUERY PREHOOK: Input: default@src2 PREHOOK: Input: default@src3 PREHOOK: Input: default@src4 PREHOOK: Input: default@src5 #### A masked pattern was here #### POSTHOOK: query: select s.key, s.count from ( select key, count from src2 where key < 10 UNION DISTINCT select key, count from src3 where key < 10 UNION DISTINCT select key, count from src4 where key < 10 UNION DISTINCT select key, count(1) as count from src5 where key < 10 group by key )s POSTHOOK: type: QUERY POSTHOOK: Input: default@src2 POSTHOOK: Input: default@src3 POSTHOOK: Input: default@src4 POSTHOOK: Input: default@src5 #### A masked pattern was here #### 0 1 0 3 2 1 4 1 5 1 5 3 8 1 9 1 PREHOOK: query: explain extended select s.key, s.count from ( select key, count from src2 where key < 10 UNION DISTINCT select key, count from src3 where key < 10 UNION DISTINCT select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 )s PREHOOK: type: QUERY POSTHOOK: query: explain extended select s.key, s.count from ( select key, count from src2 where key < 10 UNION DISTINCT select key, count from src3 where key < 10 UNION DISTINCT select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 )s POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY TOK_QUERY TOK_FROM TOK_SUBQUERY TOK_UNIONALL TOK_QUERY TOK_FROM TOK_SUBQUERY TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF TOK_TABNAME src2 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR TOK_TABLE_OR_COL key TOK_SELEXPR TOK_TABLE_OR_COL count TOK_WHERE < TOK_TABLE_OR_COL key 10 TOK_QUERY TOK_FROM TOK_TABREF TOK_TABNAME src3 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR TOK_TABLE_OR_COL key TOK_SELEXPR TOK_TABLE_OR_COL count TOK_WHERE < TOK_TABLE_OR_COL key 10 _u1 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECTDI TOK_SELEXPR TOK_ALLCOLREF TOK_QUERY TOK_FROM TOK_JOIN TOK_TABREF TOK_TABNAME src4 a TOK_TABREF TOK_TABNAME src5 b = . TOK_TABLE_OR_COL a key . TOK_TABLE_OR_COL b key TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR . TOK_TABLE_OR_COL a key key TOK_SELEXPR . TOK_TABLE_OR_COL b count count TOK_WHERE < . TOK_TABLE_OR_COL a key 10 _u2 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECTDI TOK_SELEXPR TOK_ALLCOLREF s TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR . TOK_TABLE_OR_COL s key TOK_SELEXPR . TOK_TABLE_OR_COL s count STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 6 <- Union 2 (CONTAINS) Map 7 <- Map 8 (BROADCAST_EDGE), Union 4 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) Reducer 5 <- Union 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src2 GatherStats: false Filter Operator isSamplingPred: false predicate: (key < 10) (type: boolean) Select Operator expressions: key (type: string), count (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: src2 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src2 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src2 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src2 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src2 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src2 name: default.src2 Truncated Path -> Alias: /src2 [src2] Map 6 Map Operator Tree: TableScan alias: src3 GatherStats: false Filter Operator isSamplingPred: false predicate: (key < 10) (type: boolean) Select Operator expressions: key (type: string), count (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: src3 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src3 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src3 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src3 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src3 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src3 name: default.src3 Truncated Path -> Alias: /src3 [src3] Map 7 Map Operator Tree: TableScan alias: a GatherStats: false Filter Operator isSamplingPred: false predicate: (key < 10) (type: boolean) Map Join Operator condition map: Inner Join 0 to 1 Estimated key counts: Map 8 => 103 keys: 0 key (type: string) 1 key (type: string) outputColumnNames: _col0, _col6 input vertices: 1 Map 8 Position of Big Table: 0 Select Operator expressions: _col0 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: src4 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src4 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src4 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src4 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src4 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src4 name: default.src4 Truncated Path -> Alias: /src4 [a] Map 8 Map Operator Tree: TableScan alias: b Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key < 10) (type: boolean) Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: count (type: bigint) auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: src5 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src5 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src5 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src5 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src5 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src5 name: default.src5 Truncated Path -> Alias: /src5 [b] Reducer 3 Needs Tagging: false Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Reducer 5 Needs Tagging: false Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 108 Data size: 518 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 Statistics: Num rows: 108 Data size: 518 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: columns _col0,_col1 columns.types string:bigint escape.delim \ hive.serialization.extend.additional.nesting.levels true serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TotalFiles: 1 GatherStats: false MultiFileSpray: false Union 2 Vertex: Union 2 Union 4 Vertex: Union 4 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select s.key, s.count from ( select key, count from src2 where key < 10 UNION DISTINCT select key, count from src3 where key < 10 UNION DISTINCT select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 )s PREHOOK: type: QUERY PREHOOK: Input: default@src2 PREHOOK: Input: default@src3 PREHOOK: Input: default@src4 PREHOOK: Input: default@src5 #### A masked pattern was here #### POSTHOOK: query: select s.key, s.count from ( select key, count from src2 where key < 10 UNION DISTINCT select key, count from src3 where key < 10 UNION DISTINCT select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 )s POSTHOOK: type: QUERY POSTHOOK: Input: default@src2 POSTHOOK: Input: default@src3 POSTHOOK: Input: default@src4 POSTHOOK: Input: default@src5 #### A masked pattern was here #### 0 3 2 1 4 1 5 3 8 1 9 1 PREHOOK: query: explain extended select s.key, s.count from ( select key, count from src2 where key < 10 UNION DISTINCT select key, count from src3 where key < 10 UNION DISTINCT select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key )s PREHOOK: type: QUERY POSTHOOK: query: explain extended select s.key, s.count from ( select key, count from src2 where key < 10 UNION DISTINCT select key, count from src3 where key < 10 UNION DISTINCT select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key )s POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY TOK_QUERY TOK_FROM TOK_SUBQUERY TOK_UNIONALL TOK_QUERY TOK_FROM TOK_SUBQUERY TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF TOK_TABNAME src2 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR TOK_TABLE_OR_COL key TOK_SELEXPR TOK_TABLE_OR_COL count TOK_WHERE < TOK_TABLE_OR_COL key 10 TOK_QUERY TOK_FROM TOK_TABREF TOK_TABNAME src3 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR TOK_TABLE_OR_COL key TOK_SELEXPR TOK_TABLE_OR_COL count TOK_WHERE < TOK_TABLE_OR_COL key 10 _u1 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECTDI TOK_SELEXPR TOK_ALLCOLREF TOK_QUERY TOK_FROM TOK_JOIN TOK_TABREF TOK_TABNAME src4 a TOK_TABREF TOK_TABNAME src5 b = . TOK_TABLE_OR_COL a key . TOK_TABLE_OR_COL b key TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR . TOK_TABLE_OR_COL a key key TOK_SELEXPR TOK_FUNCTION count 1 count TOK_WHERE < . TOK_TABLE_OR_COL a key 10 TOK_GROUPBY . TOK_TABLE_OR_COL a key _u2 TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECTDI TOK_SELEXPR TOK_ALLCOLREF s TOK_INSERT TOK_DESTINATION TOK_DIR TOK_TMP_FILE TOK_SELECT TOK_SELEXPR . TOK_TABLE_OR_COL s key TOK_SELEXPR . TOK_TABLE_OR_COL s count STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 6 <- Union 2 (CONTAINS) Map 7 <- Map 9 (BROADCAST_EDGE) Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) Reducer 5 <- Union 4 (SIMPLE_EDGE) Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 4 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src2 GatherStats: false Filter Operator isSamplingPred: false predicate: (key < 10) (type: boolean) Select Operator expressions: key (type: string), count (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: src2 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src2 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src2 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src2 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src2 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src2 name: default.src2 Truncated Path -> Alias: /src2 [src2] Map 6 Map Operator Tree: TableScan alias: src3 GatherStats: false Filter Operator isSamplingPred: false predicate: (key < 10) (type: boolean) Select Operator expressions: key (type: string), count (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: src3 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src3 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src3 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src3 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src3 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src3 name: default.src3 Truncated Path -> Alias: /src3 [src3] Map 7 Map Operator Tree: TableScan alias: a Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key < 10) (type: boolean) Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 Estimated key counts: Map 9 => 103 keys: 0 key (type: string) 1 key (type: string) outputColumnNames: _col0 input vertices: 1 Map 9 Position of Big Table: 0 Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: src4 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src4 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src4 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src4 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src4 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src4 name: default.src4 Truncated Path -> Alias: /src4 [a] Map 9 Map Operator Tree: TableScan alias: b Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key < 10) (type: boolean) Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE tag: 1 auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition base file name: src5 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src5 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src5 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true bucket_count -1 columns key,count columns.comments columns.types string:bigint #### A masked pattern was here #### name default.src5 numFiles 1 numRows 309 rawDataSize 1482 serialization.ddl struct src5 { string key, i64 count} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 1791 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src5 name: default.src5 Truncated Path -> Alias: /src5 [b] Reducer 3 Needs Tagging: false Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Reducer 5 Needs Tagging: false Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 79 Data size: 379 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 Statistics: Num rows: 79 Data size: 379 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: columns _col0,_col1 columns.types string:bigint escape.delim \ hive.serialization.extend.additional.nesting.levels true serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TotalFiles: 1 GatherStats: false MultiFileSpray: false Reducer 8 Needs Tagging: false Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) tag: -1 auto parallelism: true Union 2 Vertex: Union 2 Union 4 Vertex: Union 4 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select s.key, s.count from ( select key, count from src2 where key < 10 UNION DISTINCT select key, count from src3 where key < 10 UNION DISTINCT select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key )s PREHOOK: type: QUERY PREHOOK: Input: default@src2 PREHOOK: Input: default@src3 PREHOOK: Input: default@src4 PREHOOK: Input: default@src5 #### A masked pattern was here #### POSTHOOK: query: select s.key, s.count from ( select key, count from src2 where key < 10 UNION DISTINCT select key, count from src3 where key < 10 UNION DISTINCT select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key )s POSTHOOK: type: QUERY POSTHOOK: Input: default@src2 POSTHOOK: Input: default@src3 POSTHOOK: Input: default@src4 POSTHOOK: Input: default@src5 #### A masked pattern was here #### 0 1 0 3 2 1 4 1 5 1 5 3 8 1 9 1 PREHOOK: query: -- union25.q create table tmp_srcpart like srcpart PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tmp_srcpart POSTHOOK: query: -- union25.q create table tmp_srcpart like srcpart POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@tmp_srcpart PREHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') select key, value from srcpart where ds='2008-04-08' and hr='11' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 POSTHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') select key, value from srcpart where ds='2008-04-08' and hr='11' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: explain create table tmp_unionall as SELECT count(1) as counts, key, value FROM ( SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' UNION DISTINCT SELECT key, key as value FROM ( SELECT distinct key FROM ( SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' UNION DISTINCT SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' )t ) master_table ) a GROUP BY key, value PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: explain create table tmp_unionall as SELECT count(1) as counts, key, value FROM ( SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' UNION DISTINCT SELECT key, key as value FROM ( SELECT distinct key FROM ( SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' UNION DISTINCT SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' )t ) master_table ) a GROUP BY key, value POSTHOOK: type: CREATETABLE_AS_SELECT STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-4 depends on stages: Stage-2, Stage-0 Stage-3 depends on stages: Stage-4 Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 4 <- Union 5 (CONTAINS) Map 7 <- Union 5 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 6 <- Union 2 (CONTAINS), Union 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: a Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 4 Map Operator Tree: TableScan alias: a Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Map 7 Map Operator Tree: TableScan alias: a Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 375 Data size: 3984 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 187 Data size: 1986 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 187 Data size: 1986 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 187 Data size: 1986 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmp_unionall Reducer 6 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 Group By Operator keys: _col0 (type: string) mode: complete outputColumnNames: _col0 Select Operator expressions: _col0 (type: string), _col0 (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Union 2 Vertex: Union 2 Union 5 Vertex: Union 5 Stage: Stage-2 Dependency Collection Stage: Stage-4 Create Table Operator: Create Table columns: counts bigint, key string, value string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmp_unionall Stage: Stage-3 Stats-Aggr Operator Stage: Stage-0 Move Operator files: hdfs directory: true #### A masked pattern was here #### PREHOOK: query: -- union26.q -- SORT_QUERY_RESULTS EXPLAIN SELECT count(1) as counts, key, value FROM ( SELECT a.key, a.value FROM srcpart a JOIN srcpart b ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' AND a.key = b.key UNION DISTINCT select key, value FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE ds='2008-04-08' and hr='11' ) a group by key, value PREHOOK: type: QUERY POSTHOOK: query: -- union26.q -- SORT_QUERY_RESULTS EXPLAIN SELECT count(1) as counts, key, value FROM ( SELECT a.key, a.value FROM srcpart a JOIN srcpart b ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' AND a.key = b.key UNION DISTINCT select key, value FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE ds='2008-04-08' and hr='11' ) a group by key, value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) Map 5 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: a Filter Operator predicate: key is not null (type: boolean) Map Join Operator condition map: Inner Join 0 to 1 keys: 0 key (type: string) 1 key (type: string) outputColumnNames: _col0, _col1 input vertices: 1 Map 4 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 4 Map Operator Tree: TableScan alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan alias: srcpart Lateral View Forward Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value Lateral View Join Operator outputColumnNames: _col0, _col1, _col7 Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Select Operator expressions: array(1,2,3) (type: array) outputColumnNames: _col0 UDTF Operator function name: explode Lateral View Join Operator outputColumnNames: _col0, _col1, _col7 Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 637 Data size: 6767 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 318 Data size: 3378 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 318 Data size: 3378 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 318 Data size: 3378 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: SELECT count(1) as counts, key, value FROM ( SELECT a.key, a.value FROM srcpart a JOIN srcpart b ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' AND a.key = b.key UNION DISTINCT select key, value FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE ds='2008-04-08' and hr='11' ) a group by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### POSTHOOK: query: SELECT count(1) as counts, key, value FROM ( SELECT a.key, a.value FROM srcpart a JOIN srcpart b ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' AND a.key = b.key UNION DISTINCT select key, value FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE ds='2008-04-08' and hr='11' ) a group by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1 0 val_0 1 10 val_10 1 100 val_100 1 103 val_103 1 104 val_104 1 105 val_105 1 11 val_11 1 111 val_111 1 113 val_113 1 114 val_114 1 116 val_116 1 118 val_118 1 119 val_119 1 12 val_12 1 120 val_120 1 125 val_125 1 126 val_126 1 128 val_128 1 129 val_129 1 131 val_131 1 133 val_133 1 134 val_134 1 136 val_136 1 137 val_137 1 138 val_138 1 143 val_143 1 145 val_145 1 146 val_146 1 149 val_149 1 15 val_15 1 150 val_150 1 152 val_152 1 153 val_153 1 155 val_155 1 156 val_156 1 157 val_157 1 158 val_158 1 160 val_160 1 162 val_162 1 163 val_163 1 164 val_164 1 165 val_165 1 166 val_166 1 167 val_167 1 168 val_168 1 169 val_169 1 17 val_17 1 170 val_170 1 172 val_172 1 174 val_174 1 175 val_175 1 176 val_176 1 177 val_177 1 178 val_178 1 179 val_179 1 18 val_18 1 180 val_180 1 181 val_181 1 183 val_183 1 186 val_186 1 187 val_187 1 189 val_189 1 19 val_19 1 190 val_190 1 191 val_191 1 192 val_192 1 193 val_193 1 194 val_194 1 195 val_195 1 196 val_196 1 197 val_197 1 199 val_199 1 2 val_2 1 20 val_20 1 200 val_200 1 201 val_201 1 202 val_202 1 203 val_203 1 205 val_205 1 207 val_207 1 208 val_208 1 209 val_209 1 213 val_213 1 214 val_214 1 216 val_216 1 217 val_217 1 218 val_218 1 219 val_219 1 221 val_221 1 222 val_222 1 223 val_223 1 224 val_224 1 226 val_226 1 228 val_228 1 229 val_229 1 230 val_230 1 233 val_233 1 235 val_235 1 237 val_237 1 238 val_238 1 239 val_239 1 24 val_24 1 241 val_241 1 242 val_242 1 244 val_244 1 247 val_247 1 248 val_248 1 249 val_249 1 252 val_252 1 255 val_255 1 256 val_256 1 257 val_257 1 258 val_258 1 26 val_26 1 260 val_260 1 262 val_262 1 263 val_263 1 265 val_265 1 266 val_266 1 27 val_27 1 272 val_272 1 273 val_273 1 274 val_274 1 275 val_275 1 277 val_277 1 278 val_278 1 28 val_28 1 280 val_280 1 281 val_281 1 282 val_282 1 283 val_283 1 284 val_284 1 285 val_285 1 286 val_286 1 287 val_287 1 288 val_288 1 289 val_289 1 291 val_291 1 292 val_292 1 296 val_296 1 298 val_298 1 30 val_30 1 302 val_302 1 305 val_305 1 306 val_306 1 307 val_307 1 308 val_308 1 309 val_309 1 310 val_310 1 311 val_311 1 315 val_315 1 316 val_316 1 317 val_317 1 318 val_318 1 321 val_321 1 322 val_322 1 323 val_323 1 325 val_325 1 327 val_327 1 33 val_33 1 331 val_331 1 332 val_332 1 333 val_333 1 335 val_335 1 336 val_336 1 338 val_338 1 339 val_339 1 34 val_34 1 341 val_341 1 342 val_342 1 344 val_344 1 345 val_345 1 348 val_348 1 35 val_35 1 351 val_351 1 353 val_353 1 356 val_356 1 360 val_360 1 362 val_362 1 364 val_364 1 365 val_365 1 366 val_366 1 367 val_367 1 368 val_368 1 369 val_369 1 37 val_37 1 373 val_373 1 374 val_374 1 375 val_375 1 377 val_377 1 378 val_378 1 379 val_379 1 382 val_382 1 384 val_384 1 386 val_386 1 389 val_389 1 392 val_392 1 393 val_393 1 394 val_394 1 395 val_395 1 396 val_396 1 397 val_397 1 399 val_399 1 4 val_4 1 400 val_400 1 401 val_401 1 402 val_402 1 403 val_403 1 404 val_404 1 406 val_406 1 407 val_407 1 409 val_409 1 41 val_41 1 411 val_411 1 413 val_413 1 414 val_414 1 417 val_417 1 418 val_418 1 419 val_419 1 42 val_42 1 421 val_421 1 424 val_424 1 427 val_427 1 429 val_429 1 43 val_43 1 430 val_430 1 431 val_431 1 432 val_432 1 435 val_435 1 436 val_436 1 437 val_437 1 438 val_438 1 439 val_439 1 44 val_44 1 443 val_443 1 444 val_444 1 446 val_446 1 448 val_448 1 449 val_449 1 452 val_452 1 453 val_453 1 454 val_454 1 455 val_455 1 457 val_457 1 458 val_458 1 459 val_459 1 460 val_460 1 462 val_462 1 463 val_463 1 466 val_466 1 467 val_467 1 468 val_468 1 469 val_469 1 47 val_47 1 470 val_470 1 472 val_472 1 475 val_475 1 477 val_477 1 478 val_478 1 479 val_479 1 480 val_480 1 481 val_481 1 482 val_482 1 483 val_483 1 484 val_484 1 485 val_485 1 487 val_487 1 489 val_489 1 490 val_490 1 491 val_491 1 492 val_492 1 493 val_493 1 494 val_494 1 495 val_495 1 496 val_496 1 497 val_497 1 498 val_498 1 5 val_5 1 51 val_51 1 53 val_53 1 54 val_54 1 57 val_57 1 58 val_58 1 64 val_64 1 65 val_65 1 66 val_66 1 67 val_67 1 69 val_69 1 70 val_70 1 72 val_72 1 74 val_74 1 76 val_76 1 77 val_77 1 78 val_78 1 8 val_8 1 80 val_80 1 82 val_82 1 83 val_83 1 84 val_84 1 85 val_85 1 86 val_86 1 87 val_87 1 9 val_9 1 90 val_90 1 92 val_92 1 95 val_95 1 96 val_96 1 97 val_97 1 98 val_98 PREHOOK: query: SELECT count(1) as counts, key, value FROM ( SELECT a.key, a.value FROM srcpart a JOIN srcpart b ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' AND a.key = b.key UNION DISTINCT select key, value FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE ds='2008-04-08' and hr='11' ) a group by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### POSTHOOK: query: SELECT count(1) as counts, key, value FROM ( SELECT a.key, a.value FROM srcpart a JOIN srcpart b ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' AND a.key = b.key UNION DISTINCT select key, value FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE ds='2008-04-08' and hr='11' ) a group by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1 0 val_0 1 10 val_10 1 100 val_100 1 103 val_103 1 104 val_104 1 105 val_105 1 11 val_11 1 111 val_111 1 113 val_113 1 114 val_114 1 116 val_116 1 118 val_118 1 119 val_119 1 12 val_12 1 120 val_120 1 125 val_125 1 126 val_126 1 128 val_128 1 129 val_129 1 131 val_131 1 133 val_133 1 134 val_134 1 136 val_136 1 137 val_137 1 138 val_138 1 143 val_143 1 145 val_145 1 146 val_146 1 149 val_149 1 15 val_15 1 150 val_150 1 152 val_152 1 153 val_153 1 155 val_155 1 156 val_156 1 157 val_157 1 158 val_158 1 160 val_160 1 162 val_162 1 163 val_163 1 164 val_164 1 165 val_165 1 166 val_166 1 167 val_167 1 168 val_168 1 169 val_169 1 17 val_17 1 170 val_170 1 172 val_172 1 174 val_174 1 175 val_175 1 176 val_176 1 177 val_177 1 178 val_178 1 179 val_179 1 18 val_18 1 180 val_180 1 181 val_181 1 183 val_183 1 186 val_186 1 187 val_187 1 189 val_189 1 19 val_19 1 190 val_190 1 191 val_191 1 192 val_192 1 193 val_193 1 194 val_194 1 195 val_195 1 196 val_196 1 197 val_197 1 199 val_199 1 2 val_2 1 20 val_20 1 200 val_200 1 201 val_201 1 202 val_202 1 203 val_203 1 205 val_205 1 207 val_207 1 208 val_208 1 209 val_209 1 213 val_213 1 214 val_214 1 216 val_216 1 217 val_217 1 218 val_218 1 219 val_219 1 221 val_221 1 222 val_222 1 223 val_223 1 224 val_224 1 226 val_226 1 228 val_228 1 229 val_229 1 230 val_230 1 233 val_233 1 235 val_235 1 237 val_237 1 238 val_238 1 239 val_239 1 24 val_24 1 241 val_241 1 242 val_242 1 244 val_244 1 247 val_247 1 248 val_248 1 249 val_249 1 252 val_252 1 255 val_255 1 256 val_256 1 257 val_257 1 258 val_258 1 26 val_26 1 260 val_260 1 262 val_262 1 263 val_263 1 265 val_265 1 266 val_266 1 27 val_27 1 272 val_272 1 273 val_273 1 274 val_274 1 275 val_275 1 277 val_277 1 278 val_278 1 28 val_28 1 280 val_280 1 281 val_281 1 282 val_282 1 283 val_283 1 284 val_284 1 285 val_285 1 286 val_286 1 287 val_287 1 288 val_288 1 289 val_289 1 291 val_291 1 292 val_292 1 296 val_296 1 298 val_298 1 30 val_30 1 302 val_302 1 305 val_305 1 306 val_306 1 307 val_307 1 308 val_308 1 309 val_309 1 310 val_310 1 311 val_311 1 315 val_315 1 316 val_316 1 317 val_317 1 318 val_318 1 321 val_321 1 322 val_322 1 323 val_323 1 325 val_325 1 327 val_327 1 33 val_33 1 331 val_331 1 332 val_332 1 333 val_333 1 335 val_335 1 336 val_336 1 338 val_338 1 339 val_339 1 34 val_34 1 341 val_341 1 342 val_342 1 344 val_344 1 345 val_345 1 348 val_348 1 35 val_35 1 351 val_351 1 353 val_353 1 356 val_356 1 360 val_360 1 362 val_362 1 364 val_364 1 365 val_365 1 366 val_366 1 367 val_367 1 368 val_368 1 369 val_369 1 37 val_37 1 373 val_373 1 374 val_374 1 375 val_375 1 377 val_377 1 378 val_378 1 379 val_379 1 382 val_382 1 384 val_384 1 386 val_386 1 389 val_389 1 392 val_392 1 393 val_393 1 394 val_394 1 395 val_395 1 396 val_396 1 397 val_397 1 399 val_399 1 4 val_4 1 400 val_400 1 401 val_401 1 402 val_402 1 403 val_403 1 404 val_404 1 406 val_406 1 407 val_407 1 409 val_409 1 41 val_41 1 411 val_411 1 413 val_413 1 414 val_414 1 417 val_417 1 418 val_418 1 419 val_419 1 42 val_42 1 421 val_421 1 424 val_424 1 427 val_427 1 429 val_429 1 43 val_43 1 430 val_430 1 431 val_431 1 432 val_432 1 435 val_435 1 436 val_436 1 437 val_437 1 438 val_438 1 439 val_439 1 44 val_44 1 443 val_443 1 444 val_444 1 446 val_446 1 448 val_448 1 449 val_449 1 452 val_452 1 453 val_453 1 454 val_454 1 455 val_455 1 457 val_457 1 458 val_458 1 459 val_459 1 460 val_460 1 462 val_462 1 463 val_463 1 466 val_466 1 467 val_467 1 468 val_468 1 469 val_469 1 47 val_47 1 470 val_470 1 472 val_472 1 475 val_475 1 477 val_477 1 478 val_478 1 479 val_479 1 480 val_480 1 481 val_481 1 482 val_482 1 483 val_483 1 484 val_484 1 485 val_485 1 487 val_487 1 489 val_489 1 490 val_490 1 491 val_491 1 492 val_492 1 493 val_493 1 494 val_494 1 495 val_495 1 496 val_496 1 497 val_497 1 498 val_498 1 5 val_5 1 51 val_51 1 53 val_53 1 54 val_54 1 57 val_57 1 58 val_58 1 64 val_64 1 65 val_65 1 66 val_66 1 67 val_67 1 69 val_69 1 70 val_70 1 72 val_72 1 74 val_74 1 76 val_76 1 77 val_77 1 78 val_78 1 8 val_8 1 80 val_80 1 82 val_82 1 83 val_83 1 84 val_84 1 85 val_85 1 86 val_86 1 87 val_87 1 9 val_9 1 90 val_90 1 92 val_92 1 95 val_95 1 96 val_96 1 97 val_97 1 98 val_98 PREHOOK: query: SELECT count(1) as counts, key, value FROM ( SELECT a.key, a.value FROM srcpart a JOIN srcpart b ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' AND a.key = b.key UNION DISTINCT select key, value FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE ds='2008-04-08' and hr='11' ) a group by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### POSTHOOK: query: SELECT count(1) as counts, key, value FROM ( SELECT a.key, a.value FROM srcpart a JOIN srcpart b ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' AND a.key = b.key UNION DISTINCT select key, value FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE ds='2008-04-08' and hr='11' ) a group by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1 0 val_0 1 10 val_10 1 100 val_100 1 103 val_103 1 104 val_104 1 105 val_105 1 11 val_11 1 111 val_111 1 113 val_113 1 114 val_114 1 116 val_116 1 118 val_118 1 119 val_119 1 12 val_12 1 120 val_120 1 125 val_125 1 126 val_126 1 128 val_128 1 129 val_129 1 131 val_131 1 133 val_133 1 134 val_134 1 136 val_136 1 137 val_137 1 138 val_138 1 143 val_143 1 145 val_145 1 146 val_146 1 149 val_149 1 15 val_15 1 150 val_150 1 152 val_152 1 153 val_153 1 155 val_155 1 156 val_156 1 157 val_157 1 158 val_158 1 160 val_160 1 162 val_162 1 163 val_163 1 164 val_164 1 165 val_165 1 166 val_166 1 167 val_167 1 168 val_168 1 169 val_169 1 17 val_17 1 170 val_170 1 172 val_172 1 174 val_174 1 175 val_175 1 176 val_176 1 177 val_177 1 178 val_178 1 179 val_179 1 18 val_18 1 180 val_180 1 181 val_181 1 183 val_183 1 186 val_186 1 187 val_187 1 189 val_189 1 19 val_19 1 190 val_190 1 191 val_191 1 192 val_192 1 193 val_193 1 194 val_194 1 195 val_195 1 196 val_196 1 197 val_197 1 199 val_199 1 2 val_2 1 20 val_20 1 200 val_200 1 201 val_201 1 202 val_202 1 203 val_203 1 205 val_205 1 207 val_207 1 208 val_208 1 209 val_209 1 213 val_213 1 214 val_214 1 216 val_216 1 217 val_217 1 218 val_218 1 219 val_219 1 221 val_221 1 222 val_222 1 223 val_223 1 224 val_224 1 226 val_226 1 228 val_228 1 229 val_229 1 230 val_230 1 233 val_233 1 235 val_235 1 237 val_237 1 238 val_238 1 239 val_239 1 24 val_24 1 241 val_241 1 242 val_242 1 244 val_244 1 247 val_247 1 248 val_248 1 249 val_249 1 252 val_252 1 255 val_255 1 256 val_256 1 257 val_257 1 258 val_258 1 26 val_26 1 260 val_260 1 262 val_262 1 263 val_263 1 265 val_265 1 266 val_266 1 27 val_27 1 272 val_272 1 273 val_273 1 274 val_274 1 275 val_275 1 277 val_277 1 278 val_278 1 28 val_28 1 280 val_280 1 281 val_281 1 282 val_282 1 283 val_283 1 284 val_284 1 285 val_285 1 286 val_286 1 287 val_287 1 288 val_288 1 289 val_289 1 291 val_291 1 292 val_292 1 296 val_296 1 298 val_298 1 30 val_30 1 302 val_302 1 305 val_305 1 306 val_306 1 307 val_307 1 308 val_308 1 309 val_309 1 310 val_310 1 311 val_311 1 315 val_315 1 316 val_316 1 317 val_317 1 318 val_318 1 321 val_321 1 322 val_322 1 323 val_323 1 325 val_325 1 327 val_327 1 33 val_33 1 331 val_331 1 332 val_332 1 333 val_333 1 335 val_335 1 336 val_336 1 338 val_338 1 339 val_339 1 34 val_34 1 341 val_341 1 342 val_342 1 344 val_344 1 345 val_345 1 348 val_348 1 35 val_35 1 351 val_351 1 353 val_353 1 356 val_356 1 360 val_360 1 362 val_362 1 364 val_364 1 365 val_365 1 366 val_366 1 367 val_367 1 368 val_368 1 369 val_369 1 37 val_37 1 373 val_373 1 374 val_374 1 375 val_375 1 377 val_377 1 378 val_378 1 379 val_379 1 382 val_382 1 384 val_384 1 386 val_386 1 389 val_389 1 392 val_392 1 393 val_393 1 394 val_394 1 395 val_395 1 396 val_396 1 397 val_397 1 399 val_399 1 4 val_4 1 400 val_400 1 401 val_401 1 402 val_402 1 403 val_403 1 404 val_404 1 406 val_406 1 407 val_407 1 409 val_409 1 41 val_41 1 411 val_411 1 413 val_413 1 414 val_414 1 417 val_417 1 418 val_418 1 419 val_419 1 42 val_42 1 421 val_421 1 424 val_424 1 427 val_427 1 429 val_429 1 43 val_43 1 430 val_430 1 431 val_431 1 432 val_432 1 435 val_435 1 436 val_436 1 437 val_437 1 438 val_438 1 439 val_439 1 44 val_44 1 443 val_443 1 444 val_444 1 446 val_446 1 448 val_448 1 449 val_449 1 452 val_452 1 453 val_453 1 454 val_454 1 455 val_455 1 457 val_457 1 458 val_458 1 459 val_459 1 460 val_460 1 462 val_462 1 463 val_463 1 466 val_466 1 467 val_467 1 468 val_468 1 469 val_469 1 47 val_47 1 470 val_470 1 472 val_472 1 475 val_475 1 477 val_477 1 478 val_478 1 479 val_479 1 480 val_480 1 481 val_481 1 482 val_482 1 483 val_483 1 484 val_484 1 485 val_485 1 487 val_487 1 489 val_489 1 490 val_490 1 491 val_491 1 492 val_492 1 493 val_493 1 494 val_494 1 495 val_495 1 496 val_496 1 497 val_497 1 498 val_498 1 5 val_5 1 51 val_51 1 53 val_53 1 54 val_54 1 57 val_57 1 58 val_58 1 64 val_64 1 65 val_65 1 66 val_66 1 67 val_67 1 69 val_69 1 70 val_70 1 72 val_72 1 74 val_74 1 76 val_76 1 77 val_77 1 78 val_78 1 8 val_8 1 80 val_80 1 82 val_82 1 83 val_83 1 84 val_84 1 85 val_85 1 86 val_86 1 87 val_87 1 9 val_9 1 90 val_90 1 92 val_92 1 95 val_95 1 96 val_96 1 97 val_97 1 98 val_98 PREHOOK: query: -- union27.q -- SORT_BEFORE_DIFF create table jackson_sev_same as select * from src PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@jackson_sev_same POSTHOOK: query: -- union27.q -- SORT_BEFORE_DIFF create table jackson_sev_same as select * from src POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@jackson_sev_same PREHOOK: query: create table dim_pho as select * from src PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@dim_pho POSTHOOK: query: create table dim_pho as select * from src POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@dim_pho PREHOOK: query: create table jackson_sev_add as select * from src PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@jackson_sev_add POSTHOOK: query: create table jackson_sev_add as select * from src POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@jackson_sev_add PREHOOK: query: explain select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 PREHOOK: type: QUERY POSTHOOK: query: explain select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 4 <- Union 2 (CONTAINS) Map 5 <- Reducer 3 (BROADCAST_EDGE) Reducer 3 <- Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: dim_pho Filter Operator predicate: (key = 97) (type: boolean) Select Operator expressions: value (type: string) outputColumnNames: _col1 Select Operator expressions: '97' (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 4 Map Operator Tree: TableScan alias: jackson_sev_add Filter Operator predicate: (key = 97) (type: boolean) Select Operator expressions: value (type: string) outputColumnNames: _col1 Select Operator expressions: '97' (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 5 Map Operator Tree: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key = 97) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 '97' (type: string) 1 '97' (type: string) outputColumnNames: _col6 input vertices: 1 Reducer 3 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: '97' (type: string), _col6 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '97' (type: string) sort order: + Map-reduce partition columns: '97' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Union 2 Vertex: Union 2 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 PREHOOK: type: QUERY PREHOOK: Input: default@dim_pho PREHOOK: Input: default@jackson_sev_add PREHOOK: Input: default@jackson_sev_same #### A masked pattern was here #### POSTHOOK: query: select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 POSTHOOK: type: QUERY POSTHOOK: Input: default@dim_pho POSTHOOK: Input: default@jackson_sev_add POSTHOOK: Input: default@jackson_sev_same #### A masked pattern was here #### 97 val_97 97 val_97 PREHOOK: query: -- union28.q create table union_subq_union(key int, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@union_subq_union POSTHOOK: query: -- union28.q create table union_subq_union(key int, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@union_subq_union PREHOOK: query: explain insert overwrite table union_subq_union select * from ( select key, value from src UNION DISTINCT select key, value from ( select key, value, count(1) from src group by key, value UNION DISTINCT select key, value, count(1) from src group by key, value ) subq ) a PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table union_subq_union select * from ( select key, value from src UNION DISTINCT select key, value from ( select key, value, count(1) from src group by key, value UNION DISTINCT select key, value, count(1) from src group by key, value ) subq ) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 6 (CONTAINS) Reducer 7 <- Union 2 (CONTAINS), Union 6 (SIMPLE_EDGE) Reducer 9 <- Map 8 (SIMPLE_EDGE), Union 6 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 4 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Map 8 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 375 Data size: 3984 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 375 Data size: 3984 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 375 Data size: 3984 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union Reducer 5 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Group By Operator keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) mode: hash outputColumnNames: _col0, _col1, _col2 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) Reducer 7 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 9 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Group By Operator keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) mode: hash outputColumnNames: _col0, _col1, _col2 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) Union 2 Vertex: Union 2 Union 6 Vertex: Union 6 Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union Stage: Stage-3 Stats-Aggr Operator PREHOOK: query: insert overwrite table union_subq_union select * from ( select key, value from src UNION DISTINCT select key, value from ( select key, value, count(1) from src group by key, value UNION DISTINCT select key, value, count(1) from src group by key, value ) subq ) a PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@union_subq_union POSTHOOK: query: insert overwrite table union_subq_union select * from ( select key, value from src UNION DISTINCT select key, value from ( select key, value, count(1) from src group by key, value UNION DISTINCT select key, value, count(1) from src group by key, value ) subq ) a POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@union_subq_union POSTHOOK: Lineage: union_subq_union.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: union_subq_union.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: select * from union_subq_union order by key, value limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@union_subq_union #### A masked pattern was here #### POSTHOOK: query: select * from union_subq_union order by key, value limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@union_subq_union #### A masked pattern was here #### 0 val_0 2 val_2 4 val_4 5 val_5 8 val_8 9 val_9 10 val_10 11 val_11 12 val_12 15 val_15 17 val_17 18 val_18 19 val_19 20 val_20 24 val_24 26 val_26 27 val_27 28 val_28 30 val_30 33 val_33 PREHOOK: query: -- union29.q create table union_subq_union29(key int, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@union_subq_union29 POSTHOOK: query: -- union29.q create table union_subq_union29(key int, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@union_subq_union29 PREHOOK: query: explain insert overwrite table union_subq_union29 select * from ( select key, value from src UNION DISTINCT select key, value from ( select key, value from src UNION DISTINCT select key, value from src ) subq ) a PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table union_subq_union29 select * from ( select key, value from src UNION DISTINCT select key, value from ( select key, value from src UNION DISTINCT select key, value from src ) subq ) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 4 <- Union 5 (CONTAINS) Map 7 <- Union 5 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 6 <- Union 2 (CONTAINS), Union 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 4 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 7 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union29 Reducer 6 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Union 2 Vertex: Union 2 Union 5 Vertex: Union 5 Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union29 Stage: Stage-3 Stats-Aggr Operator PREHOOK: query: insert overwrite table union_subq_union29 select * from ( select key, value from src UNION DISTINCT select key, value from ( select key, value from src UNION DISTINCT select key, value from src ) subq ) a PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@union_subq_union29 POSTHOOK: query: insert overwrite table union_subq_union29 select * from ( select key, value from src UNION DISTINCT select key, value from ( select key, value from src UNION DISTINCT select key, value from src ) subq ) a POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@union_subq_union29 POSTHOOK: Lineage: union_subq_union29.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: union_subq_union29.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: select * from union_subq_union29 order by key, value limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@union_subq_union29 #### A masked pattern was here #### POSTHOOK: query: select * from union_subq_union29 order by key, value limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@union_subq_union29 #### A masked pattern was here #### 0 val_0 2 val_2 4 val_4 5 val_5 8 val_8 9 val_9 10 val_10 11 val_11 12 val_12 15 val_15 17 val_17 18 val_18 19 val_19 20 val_20 24 val_24 26 val_26 27 val_27 28 val_28 30 val_30 33 val_33 PREHOOK: query: -- union3.q -- SORT_BEFORE_DIFF explain SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 UNION DISTINCT SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 UNION DISTINCT SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION DISTINCT SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 CLUSTER BY id ) a PREHOOK: type: QUERY POSTHOOK: query: -- union3.q -- SORT_BEFORE_DIFF explain SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 UNION DISTINCT SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 UNION DISTINCT SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION DISTINCT SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 CLUSTER BY id ) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 14 <- Map 13 (SIMPLE_EDGE), Union 7 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) Reducer 8 <- Union 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Map 11 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Map 13 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Map 9 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reducer 10 Reduce Operator Tree: Limit Number of rows: 1 Select Operator expressions: 2 (type: int) outputColumnNames: _col0 Group By Operator keys: _col0 (type: int) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) Reducer 12 Reduce Operator Tree: Limit Number of rows: 1 Select Operator expressions: 3 (type: int) outputColumnNames: _col0 Group By Operator keys: _col0 (type: int) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) Reducer 14 Reduce Operator Tree: Limit Number of rows: 1 Select Operator expressions: 4 (type: int) outputColumnNames: _col0 Group By Operator keys: _col0 (type: int) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) Reducer 2 Reduce Operator Tree: Limit Number of rows: 1 Select Operator expressions: 1 (type: int) outputColumnNames: _col0 Group By Operator keys: _col0 (type: int) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 Group By Operator keys: _col0 (type: int) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) Reducer 6 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 Group By Operator keys: _col0 (type: int) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) Reducer 8 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE File Output Operator compressed: false Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 3 Vertex: Union 3 Union 5 Vertex: Union 5 Union 7 Vertex: Union 7 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: CREATE TABLE union_out (id int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@union_out POSTHOOK: query: CREATE TABLE union_out (id int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@union_out PREHOOK: query: insert overwrite table union_out SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 UNION DISTINCT SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 UNION DISTINCT SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION DISTINCT SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 CLUSTER BY id ) a PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@union_out POSTHOOK: query: insert overwrite table union_out SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 UNION DISTINCT SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 UNION DISTINCT SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION DISTINCT SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 CLUSTER BY id ) a POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@union_out POSTHOOK: Lineage: union_out.id EXPRESSION [] PREHOOK: query: select * from union_out PREHOOK: type: QUERY PREHOOK: Input: default@union_out #### A masked pattern was here #### POSTHOOK: query: select * from union_out POSTHOOK: type: QUERY POSTHOOK: Input: default@union_out #### A masked pattern was here #### 1 2 3 4 PREHOOK: query: -- union30.q create table union_subq_union30(key int, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@union_subq_union30 POSTHOOK: query: -- union30.q create table union_subq_union30(key int, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@union_subq_union30 PREHOOK: query: explain insert overwrite table union_subq_union30 select * from ( select * from ( select key, value from src UNION DISTINCT select key, value from ( select key, value, count(1) from src group by key, value UNION DISTINCT select key, value, count(1) from src group by key, value ) subq ) a UNION DISTINCT select key, value from src ) aa PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table union_subq_union30 select * from ( select * from ( select key, value from src UNION DISTINCT select key, value from ( select key, value, count(1) from src group by key, value UNION DISTINCT select key, value, count(1) from src group by key, value ) subq ) a UNION DISTINCT select key, value from src ) aa POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 12 <- Union 4 (CONTAINS) Reducer 11 <- Map 10 (SIMPLE_EDGE), Union 8 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) Reducer 5 <- Union 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 8 (CONTAINS) Reducer 9 <- Union 2 (CONTAINS), Union 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 10 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Map 12 Map Operator Tree: TableScan alias: src Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 6 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reducer 11 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Group By Operator keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) mode: hash outputColumnNames: _col0, _col1, _col2 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 5 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 437 Data size: 4642 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 437 Data size: 4642 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 437 Data size: 4642 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union30 Reducer 7 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Group By Operator keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) mode: hash outputColumnNames: _col0, _col1, _col2 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) Reducer 9 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Union 2 Vertex: Union 2 Union 4 Vertex: Union 4 Union 8 Vertex: Union 8 Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union30 Stage: Stage-3 Stats-Aggr Operator PREHOOK: query: insert overwrite table union_subq_union30 select * from ( select * from ( select key, value from src UNION DISTINCT select key, value from ( select key, value, count(1) from src group by key, value UNION DISTINCT select key, value, count(1) from src group by key, value ) subq ) a UNION DISTINCT select key, value from src ) aa PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@union_subq_union30 POSTHOOK: query: insert overwrite table union_subq_union30 select * from ( select * from ( select key, value from src UNION DISTINCT select key, value from ( select key, value, count(1) from src group by key, value UNION DISTINCT select key, value, count(1) from src group by key, value ) subq ) a UNION DISTINCT select key, value from src ) aa POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@union_subq_union30 POSTHOOK: Lineage: union_subq_union30.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: union_subq_union30.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: select * from union_subq_union30 order by key, value limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@union_subq_union30 #### A masked pattern was here #### POSTHOOK: query: select * from union_subq_union30 order by key, value limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@union_subq_union30 #### A masked pattern was here #### 0 val_0 2 val_2 4 val_4 5 val_5 8 val_8 9 val_9 10 val_10 11 val_11 12 val_12 15 val_15 17 val_17 18 val_18 19 val_19 20 val_20 24 val_24 26 val_26 27 val_27 28 val_28 30 val_30 33 val_33 PREHOOK: query: -- union31.q -- SORT_QUERY_RESULTS drop table t1 PREHOOK: type: DROPTABLE POSTHOOK: query: -- union31.q -- SORT_QUERY_RESULTS drop table t1 POSTHOOK: type: DROPTABLE PREHOOK: query: drop table t2 PREHOOK: type: DROPTABLE POSTHOOK: query: drop table t2 POSTHOOK: type: DROPTABLE PREHOOK: query: create table t1 as select * from src where key < 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@t1 POSTHOOK: query: create table t1 as select * from src where key < 10 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@t1 PREHOOK: query: create table t2 as select * from src where key < 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@t2 POSTHOOK: query: create table t2 as select * from src where key < 10 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@t2 PREHOOK: query: create table t3(key string, cnt int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@t3 POSTHOOK: query: create table t3(key string, cnt int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@t3 PREHOOK: query: create table t4(value string, cnt int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@t4 POSTHOOK: query: create table t4(value string, cnt int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@t4 PREHOOK: query: explain from (select * from t1 UNION DISTINCT select * from t2 ) x insert overwrite table t3 select key, count(1) group by key insert overwrite table t4 select value, count(1) group by value PREHOOK: type: QUERY POSTHOOK: query: explain from (select * from t1 UNION DISTINCT select * from t2 ) x insert overwrite table t3 select key, count(1) group by key insert overwrite table t4 select value, count(1) group by value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 Stage-4 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 5 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: t1 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Map 5 Map Operator Tree: TableScan alias: t2 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t3 Select Operator expressions: _col1 (type: string) outputColumnNames: _col1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t4 Union 2 Vertex: Union 2 Stage: Stage-3 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t3 Stage: Stage-4 Stats-Aggr Operator Stage: Stage-1 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t4 Stage: Stage-5 Stats-Aggr Operator PREHOOK: query: from (select * from t1 UNION DISTINCT select * from t2 ) x insert overwrite table t3 select key, count(1) group by key insert overwrite table t4 select value, count(1) group by value PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Output: default@t3 PREHOOK: Output: default@t4 POSTHOOK: query: from (select * from t1 UNION DISTINCT select * from t2 ) x insert overwrite table t3 select key, count(1) group by key insert overwrite table t4 select value, count(1) group by value POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Output: default@t3 POSTHOOK: Output: default@t4 POSTHOOK: Lineage: t3.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] POSTHOOK: Lineage: t3.key EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: t4.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] POSTHOOK: Lineage: t4.value EXPRESSION [(t1)t1.FieldSchema(name:value, type:string, comment:null), (t2)t2.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: select * from t3 PREHOOK: type: QUERY PREHOOK: Input: default@t3 #### A masked pattern was here #### POSTHOOK: query: select * from t3 POSTHOOK: type: QUERY POSTHOOK: Input: default@t3 #### A masked pattern was here #### 0 1 2 1 4 1 5 1 8 1 9 1 PREHOOK: query: select * from t4 PREHOOK: type: QUERY PREHOOK: Input: default@t4 #### A masked pattern was here #### POSTHOOK: query: select * from t4 POSTHOOK: type: QUERY POSTHOOK: Input: default@t4 #### A masked pattern was here #### val_0 1 val_2 1 val_4 1 val_5 1 val_8 1 val_9 1 PREHOOK: query: create table t5(c1 string, cnt int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@t5 POSTHOOK: query: create table t5(c1 string, cnt int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@t5 PREHOOK: query: create table t6(c1 string, cnt int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@t6 POSTHOOK: query: create table t6(c1 string, cnt int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@t6 PREHOOK: query: explain from ( select key as c1, count(1) as cnt from t1 group by key UNION DISTINCT select key as c1, count(1) as cnt from t2 group by key ) x insert overwrite table t5 select c1, sum(cnt) group by c1 insert overwrite table t6 select c1, sum(cnt) group by c1 PREHOOK: type: QUERY POSTHOOK: query: explain from ( select key as c1, count(1) as cnt from t1 group by key UNION DISTINCT select key as c1, count(1) as cnt from t2 group by key ) x insert overwrite table t5 select c1, sum(cnt) group by c1 insert overwrite table t6 select c1, sum(cnt) group by c1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 Stage-4 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: t1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Map 6 Map Operator Tree: TableScan alias: t2 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reducer 5 Reduce Operator Tree: Forward Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(VALUE._col0) keys: KEY._col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t5 Group By Operator aggregations: sum(VALUE._col0) keys: KEY._col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t6 Reducer 7 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Union 3 Vertex: Union 3 Stage: Stage-3 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t5 Stage: Stage-4 Stats-Aggr Operator Stage: Stage-1 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t6 Stage: Stage-5 Stats-Aggr Operator PREHOOK: query: from ( select key as c1, count(1) as cnt from t1 group by key UNION DISTINCT select key as c1, count(1) as cnt from t2 group by key ) x insert overwrite table t5 select c1, sum(cnt) group by c1 insert overwrite table t6 select c1, sum(cnt) group by c1 PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Output: default@t5 PREHOOK: Output: default@t6 POSTHOOK: query: from ( select key as c1, count(1) as cnt from t1 group by key UNION DISTINCT select key as c1, count(1) as cnt from t2 group by key ) x insert overwrite table t5 select c1, sum(cnt) group by c1 insert overwrite table t6 select c1, sum(cnt) group by c1 POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Output: default@t5 POSTHOOK: Output: default@t6 POSTHOOK: Lineage: t5.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: t5.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] POSTHOOK: Lineage: t6.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: t6.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] PREHOOK: query: select * from t5 PREHOOK: type: QUERY PREHOOK: Input: default@t5 #### A masked pattern was here #### POSTHOOK: query: select * from t5 POSTHOOK: type: QUERY POSTHOOK: Input: default@t5 #### A masked pattern was here #### 0 3 2 1 4 1 5 3 8 1 9 1 PREHOOK: query: select * from t6 PREHOOK: type: QUERY PREHOOK: Input: default@t6 #### A masked pattern was here #### POSTHOOK: query: select * from t6 POSTHOOK: type: QUERY POSTHOOK: Input: default@t6 #### A masked pattern was here #### 0 3 2 1 4 1 5 3 8 1 9 1 PREHOOK: query: drop table t1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@t1 PREHOOK: Output: default@t1 POSTHOOK: query: drop table t1 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@t1 POSTHOOK: Output: default@t1 PREHOOK: query: drop table t2 PREHOOK: type: DROPTABLE PREHOOK: Input: default@t2 PREHOOK: Output: default@t2 POSTHOOK: query: drop table t2 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@t2 POSTHOOK: Output: default@t2 PREHOOK: query: create table t1 as select * from src where key < 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@t1 POSTHOOK: query: create table t1 as select * from src where key < 10 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@t1 PREHOOK: query: create table t2 as select key, count(1) as cnt from src where key < 10 group by key PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@t2 POSTHOOK: query: create table t2 as select key, count(1) as cnt from src where key < 10 group by key POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@t2 PREHOOK: query: create table t7(c1 string, cnt int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@t7 POSTHOOK: query: create table t7(c1 string, cnt int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@t7 PREHOOK: query: create table t8(c1 string, cnt int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@t8 POSTHOOK: query: create table t8(c1 string, cnt int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@t8 PREHOOK: query: explain from ( select key as c1, count(1) as cnt from t1 group by key UNION DISTINCT select key as c1, cnt from t2 ) x insert overwrite table t7 select c1, count(1) group by c1 insert overwrite table t8 select c1, count(1) group by c1 PREHOOK: type: QUERY POSTHOOK: query: explain from ( select key as c1, count(1) as cnt from t1 group by key UNION DISTINCT select key as c1, cnt from t2 ) x insert overwrite table t7 select c1, count(1) group by c1 insert overwrite table t8 select c1, count(1) group by c1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 Stage-4 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 Tez Edges: Map 6 <- Union 3 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: t1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Map 6 Map Operator Tree: TableScan alias: t2 Select Operator expressions: key (type: string), cnt (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 5 Reduce Operator Tree: Forward Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: KEY._col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t7 Group By Operator aggregations: count(1) keys: KEY._col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t8 Union 3 Vertex: Union 3 Stage: Stage-3 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t7 Stage: Stage-4 Stats-Aggr Operator Stage: Stage-1 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t8 Stage: Stage-5 Stats-Aggr Operator PREHOOK: query: from ( select key as c1, count(1) as cnt from t1 group by key UNION DISTINCT select key as c1, cnt from t2 ) x insert overwrite table t7 select c1, count(1) group by c1 insert overwrite table t8 select c1, count(1) group by c1 PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Output: default@t7 PREHOOK: Output: default@t8 POSTHOOK: query: from ( select key as c1, count(1) as cnt from t1 group by key UNION DISTINCT select key as c1, cnt from t2 ) x insert overwrite table t7 select c1, count(1) group by c1 insert overwrite table t8 select c1, count(1) group by c1 POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Output: default@t7 POSTHOOK: Output: default@t8 POSTHOOK: Lineage: t7.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: t7.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] POSTHOOK: Lineage: t8.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: t8.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] PREHOOK: query: select * from t7 PREHOOK: type: QUERY PREHOOK: Input: default@t7 #### A masked pattern was here #### POSTHOOK: query: select * from t7 POSTHOOK: type: QUERY POSTHOOK: Input: default@t7 #### A masked pattern was here #### 0 1 2 1 4 1 5 1 8 1 9 1 PREHOOK: query: select * from t8 PREHOOK: type: QUERY PREHOOK: Input: default@t8 #### A masked pattern was here #### POSTHOOK: query: select * from t8 POSTHOOK: type: QUERY POSTHOOK: Input: default@t8 #### A masked pattern was here #### 0 1 2 1 4 1 5 1 8 1 9 1 PREHOOK: query: -- union32.q -- SORT_QUERY_RESULTS -- This tests various union queries which have columns on one side of the query -- being of double type and those on the other side another drop table if exists t1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@t1 PREHOOK: Output: default@t1 POSTHOOK: query: -- union32.q -- SORT_QUERY_RESULTS -- This tests various union queries which have columns on one side of the query -- being of double type and those on the other side another drop table if exists t1 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@t1 POSTHOOK: Output: default@t1 PREHOOK: query: drop table if exists t2 PREHOOK: type: DROPTABLE PREHOOK: Input: default@t2 PREHOOK: Output: default@t2 POSTHOOK: query: drop table if exists t2 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@t2 POSTHOOK: Output: default@t2 PREHOOK: query: CREATE TABLE t1 AS SELECT * FROM src WHERE key < 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@t1 POSTHOOK: query: CREATE TABLE t1 AS SELECT * FROM src WHERE key < 10 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@t1 PREHOOK: query: CREATE TABLE t2 AS SELECT * FROM src WHERE key < 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@t2 POSTHOOK: query: CREATE TABLE t2 AS SELECT * FROM src WHERE key < 10 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@t2 PREHOOK: query: -- Test simple union with double EXPLAIN SELECT * FROM (SELECT CAST(key AS DOUBLE) AS key FROM t1 UNION DISTINCT SELECT CAST(key AS BIGINT) AS key FROM t2) a ORDER BY key PREHOOK: type: QUERY POSTHOOK: query: -- Test simple union with double EXPLAIN SELECT * FROM (SELECT CAST(key AS DOUBLE) AS key FROM t1 UNION DISTINCT SELECT CAST(key AS BIGINT) AS key FROM t2) a ORDER BY key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 5 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: t1 Select Operator expressions: UDFToDouble(key) (type: double) outputColumnNames: _col0 Group By Operator keys: _col0 (type: double) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: double) sort order: + Map-reduce partition columns: _col0 (type: double) Map 5 Map Operator Tree: TableScan alias: t2 Select Operator expressions: UDFToDouble(UDFToLong(key)) (type: double) outputColumnNames: _col0 Group By Operator keys: _col0 (type: double) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: double) sort order: + Map-reduce partition columns: _col0 (type: double) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: double) sort order: + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reducer 4 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: double) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: SELECT * FROM (SELECT CAST(key AS DOUBLE) AS key FROM t1 UNION DISTINCT SELECT CAST(key AS BIGINT) AS key FROM t2) a PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### POSTHOOK: query: SELECT * FROM (SELECT CAST(key AS DOUBLE) AS key FROM t1 UNION DISTINCT SELECT CAST(key AS BIGINT) AS key FROM t2) a POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 #### A masked pattern was here #### 0.0 2.0 4.0 5.0 8.0 9.0 PREHOOK: query: -- Test union with join on the left EXPLAIN SELECT * FROM (SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key UNION DISTINCT SELECT CAST(key AS DOUBLE) AS key FROM t2) a PREHOOK: type: QUERY POSTHOOK: query: -- Test union with join on the left EXPLAIN SELECT * FROM (SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key UNION DISTINCT SELECT CAST(key AS DOUBLE) AS key FROM t2) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) Map 5 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: a Filter Operator predicate: key is not null (type: boolean) Map Join Operator condition map: Inner Join 0 to 1 keys: 0 key (type: string) 1 key (type: string) outputColumnNames: _col0 input vertices: 1 Map 4 Select Operator expressions: UDFToDouble(UDFToLong(_col0)) (type: double) outputColumnNames: _col0 Group By Operator keys: _col0 (type: double) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: double) sort order: + Map-reduce partition columns: _col0 (type: double) Map 4 Map Operator Tree: TableScan alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan alias: t2 Select Operator expressions: UDFToDouble(key) (type: double) outputColumnNames: _col0 Group By Operator keys: _col0 (type: double) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: double) sort order: + Map-reduce partition columns: _col0 (type: double) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: SELECT * FROM (SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key UNION DISTINCT SELECT CAST(key AS DOUBLE) AS key FROM t2) a PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### POSTHOOK: query: SELECT * FROM (SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key UNION DISTINCT SELECT CAST(key AS DOUBLE) AS key FROM t2) a POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 #### A masked pattern was here #### 0.0 2.0 4.0 5.0 8.0 9.0 PREHOOK: query: -- Test union with join on the right EXPLAIN SELECT * FROM (SELECT CAST(key AS DOUBLE) AS key FROM t2 UNION DISTINCT SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a PREHOOK: type: QUERY POSTHOOK: query: -- Test union with join on the right EXPLAIN SELECT * FROM (SELECT CAST(key AS DOUBLE) AS key FROM t2 UNION DISTINCT SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: t2 Select Operator expressions: UDFToDouble(key) (type: double) outputColumnNames: _col0 Group By Operator keys: _col0 (type: double) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: double) sort order: + Map-reduce partition columns: _col0 (type: double) Map 4 Map Operator Tree: TableScan alias: a Filter Operator predicate: key is not null (type: boolean) Map Join Operator condition map: Inner Join 0 to 1 keys: 0 key (type: string) 1 key (type: string) outputColumnNames: _col0 input vertices: 1 Map 5 Select Operator expressions: UDFToDouble(UDFToLong(_col0)) (type: double) outputColumnNames: _col0 Group By Operator keys: _col0 (type: double) mode: hash outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: double) sort order: + Map-reduce partition columns: _col0 (type: double) Map 5 Map Operator Tree: TableScan alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: SELECT * FROM (SELECT CAST(key AS DOUBLE) AS key FROM t2 UNION DISTINCT SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### POSTHOOK: query: SELECT * FROM (SELECT CAST(key AS DOUBLE) AS key FROM t2 UNION DISTINCT SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 #### A masked pattern was here #### 0.0 2.0 4.0 5.0 8.0 9.0 PREHOOK: query: -- Test union with join on the left selecting multiple columns EXPLAIN SELECT * FROM (SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key UNION DISTINCT SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a PREHOOK: type: QUERY POSTHOOK: query: -- Test union with join on the left selecting multiple columns EXPLAIN SELECT * FROM (SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key UNION DISTINCT SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) Map 5 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: a Filter Operator predicate: key is not null (type: boolean) Map Join Operator condition map: Inner Join 0 to 1 keys: 0 key (type: string) 1 key (type: string) outputColumnNames: _col0, _col5 input vertices: 1 Map 4 Select Operator expressions: UDFToDouble(UDFToLong(_col0)) (type: double), UDFToString(UDFToDouble(_col5)) (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: double), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: double), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: double), _col1 (type: string) Map 4 Map Operator Tree: TableScan alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan alias: t2 Select Operator expressions: UDFToDouble(key) (type: double), key (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: double), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: double), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: double), _col1 (type: string) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: double), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: SELECT * FROM (SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key UNION DISTINCT SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### POSTHOOK: query: SELECT * FROM (SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key UNION DISTINCT SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 #### A masked pattern was here #### 0.0 0 0.0 0.0 2.0 2 2.0 2.0 4.0 4 4.0 4.0 5.0 5 5.0 5.0 8.0 8 8.0 8.0 9.0 9 9.0 9.0 PREHOOK: query: -- Test union with join on the right selecting multiple columns EXPLAIN SELECT * FROM (SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 UNION DISTINCT SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a PREHOOK: type: QUERY POSTHOOK: query: -- Test union with join on the right selecting multiple columns EXPLAIN SELECT * FROM (SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 UNION DISTINCT SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: t2 Select Operator expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: double), _col1 (type: double) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: double), _col1 (type: double) sort order: ++ Map-reduce partition columns: _col0 (type: double), _col1 (type: double) Map 4 Map Operator Tree: TableScan alias: a Filter Operator predicate: key is not null (type: boolean) Map Join Operator condition map: Inner Join 0 to 1 keys: 0 key (type: string) 1 key (type: string) outputColumnNames: _col0, _col5 input vertices: 1 Map 5 Select Operator expressions: UDFToDouble(UDFToLong(_col0)) (type: double), UDFToDouble(_col5) (type: double) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: double), _col1 (type: double) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: double), _col1 (type: double) sort order: ++ Map-reduce partition columns: _col0 (type: double), _col1 (type: double) Map 5 Map Operator Tree: TableScan alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: double), KEY._col1 (type: double) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: SELECT * FROM (SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 UNION DISTINCT SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### POSTHOOK: query: SELECT * FROM (SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 UNION DISTINCT SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 #### A masked pattern was here #### 0.0 0.0 2.0 2.0 4.0 4.0 5.0 5.0 8.0 8.0 9.0 9.0 PREHOOK: query: -- union33.q -- SORT_BEFORE_DIFF -- This tests that a UNION DISTINCT with a map only subquery on one side and a -- subquery involving two map reduce jobs on the other runs correctly. drop table if exists test_src PREHOOK: type: DROPTABLE POSTHOOK: query: -- union33.q -- SORT_BEFORE_DIFF -- This tests that a UNION DISTINCT with a map only subquery on one side and a -- subquery involving two map reduce jobs on the other runs correctly. drop table if exists test_src POSTHOOK: type: DROPTABLE PREHOOK: query: CREATE TABLE test_src (key STRING, value STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@test_src POSTHOOK: query: CREATE TABLE test_src (key STRING, value STRING) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@test_src PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( SELECT key, value FROM src WHERE key = 0 UNION DISTINCT SELECT key, COUNT(*) AS value FROM src GROUP BY key )a PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( SELECT key, value FROM src WHERE key = 0 UNION DISTINCT SELECT key, COUNT(*) AS value FROM src GROUP BY key )a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src Filter Operator predicate: (UDFToDouble(key) = 0.0) (type: boolean) Select Operator expressions: '0' (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 4 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test_src Reducer 5 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Select Operator expressions: _col0 (type: string), UDFToString(_col1) (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Union 2 Vertex: Union 2 Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test_src Stage: Stage-3 Stats-Aggr Operator PREHOOK: query: INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( SELECT key, value FROM src WHERE key = 0 UNION DISTINCT SELECT key, COUNT(*) AS value FROM src GROUP BY key )a PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@test_src POSTHOOK: query: INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( SELECT key, value FROM src WHERE key = 0 UNION DISTINCT SELECT key, COUNT(*) AS value FROM src GROUP BY key )a POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@test_src POSTHOOK: Lineage: test_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: test_src.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.null, ] PREHOOK: query: SELECT COUNT(*) FROM test_src PREHOOK: type: QUERY PREHOOK: Input: default@test_src #### A masked pattern was here #### POSTHOOK: query: SELECT COUNT(*) FROM test_src POSTHOOK: type: QUERY POSTHOOK: Input: default@test_src #### A masked pattern was here #### 310 PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( SELECT key, COUNT(*) AS value FROM src GROUP BY key UNION DISTINCT SELECT key, value FROM src WHERE key = 0 )a PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( SELECT key, COUNT(*) AS value FROM src GROUP BY key UNION DISTINCT SELECT key, value FROM src WHERE key = 0 )a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 5 <- Union 3 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Map 5 Map Operator Tree: TableScan alias: src Filter Operator predicate: (UDFToDouble(key) = 0.0) (type: boolean) Select Operator expressions: '0' (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Select Operator expressions: _col0 (type: string), UDFToString(_col1) (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test_src Union 3 Vertex: Union 3 Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test_src Stage: Stage-3 Stats-Aggr Operator PREHOOK: query: INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( SELECT key, COUNT(*) AS value FROM src GROUP BY key UNION DISTINCT SELECT key, value FROM src WHERE key = 0 )a PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@test_src POSTHOOK: query: INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( SELECT key, COUNT(*) AS value FROM src GROUP BY key UNION DISTINCT SELECT key, value FROM src WHERE key = 0 )a POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@test_src POSTHOOK: Lineage: test_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: test_src.value EXPRESSION [(src)src.null, (src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: SELECT COUNT(*) FROM test_src PREHOOK: type: QUERY PREHOOK: Input: default@test_src #### A masked pattern was here #### POSTHOOK: query: SELECT COUNT(*) FROM test_src POSTHOOK: type: QUERY POSTHOOK: Input: default@test_src #### A masked pattern was here #### 310 PREHOOK: query: -- union34.q create table src10_1 (key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@src10_1 POSTHOOK: query: -- union34.q create table src10_1 (key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@src10_1 PREHOOK: query: create table src10_2 (key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@src10_2 POSTHOOK: query: create table src10_2 (key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@src10_2 PREHOOK: query: create table src10_3 (key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@src10_3 POSTHOOK: query: create table src10_3 (key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@src10_3 PREHOOK: query: create table src10_4 (key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@src10_4 POSTHOOK: query: create table src10_4 (key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@src10_4 PREHOOK: query: from (select * from src tablesample (10 rows)) a insert overwrite table src10_1 select * insert overwrite table src10_2 select * insert overwrite table src10_3 select * insert overwrite table src10_4 select * PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@src10_1 PREHOOK: Output: default@src10_2 PREHOOK: Output: default@src10_3 PREHOOK: Output: default@src10_4 POSTHOOK: query: from (select * from src tablesample (10 rows)) a insert overwrite table src10_1 select * insert overwrite table src10_2 select * insert overwrite table src10_3 select * insert overwrite table src10_4 select * POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@src10_1 POSTHOOK: Output: default@src10_2 POSTHOOK: Output: default@src10_3 POSTHOOK: Output: default@src10_4 POSTHOOK: Lineage: src10_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: src10_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: src10_2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: src10_2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: src10_3.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: src10_3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: src10_4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: src10_4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: -- When we convert the Join of sub1 and sub0 into a MapJoin, -- we can use a single MR job to evaluate this entire query. explain SELECT * FROM ( SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) UNION DISTINCT SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 ) alias1 order by key PREHOOK: type: QUERY POSTHOOK: query: -- When we convert the Join of sub1 and sub0 into a MapJoin, -- we can use a single MR job to evaluate this entire query. explain SELECT * FROM ( SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) UNION DISTINCT SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 ) alias1 order by key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) Map 6 <- Union 7 (CONTAINS) Map 9 <- Union 7 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 8 <- Union 2 (CONTAINS), Union 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src10_1 Filter Operator predicate: key is not null (type: boolean) Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: 1 Map 5 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 5 Map Operator Tree: TableScan alias: src10_2 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE Map 6 Map Operator Tree: TableScan alias: src10_3 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 9 Map Operator Tree: TableScan alias: src10_4 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reducer 4 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 8 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Union 2 Vertex: Union 2 Union 7 Vertex: Union 7 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: SELECT * FROM ( SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) UNION DISTINCT SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 ) alias1 order by key PREHOOK: type: QUERY PREHOOK: Input: default@src10_1 PREHOOK: Input: default@src10_2 PREHOOK: Input: default@src10_3 PREHOOK: Input: default@src10_4 #### A masked pattern was here #### POSTHOOK: query: SELECT * FROM ( SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) UNION DISTINCT SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 ) alias1 order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@src10_1 POSTHOOK: Input: default@src10_2 POSTHOOK: Input: default@src10_3 POSTHOOK: Input: default@src10_4 #### A masked pattern was here #### 165 val_165 238 val_238 255 val_255 27 val_27 278 val_278 311 val_311 409 val_409 484 val_484 86 val_86 98 val_98 PREHOOK: query: -- When we do not convert the Join of sub1 and sub0 into a MapJoin, -- we need to use two MR jobs to evaluate this query. -- The first job is for the Join of sub1 and sub2. The second job -- is for the UNION DISTINCT and ORDER BY. explain SELECT * FROM ( SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) UNION DISTINCT SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 ) alias1 order by key PREHOOK: type: QUERY POSTHOOK: query: -- When we do not convert the Join of sub1 and sub0 into a MapJoin, -- we need to use two MR jobs to evaluate this query. -- The first job is for the Join of sub1 and sub2. The second job -- is for the UNION DISTINCT and ORDER BY. explain SELECT * FROM ( SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) UNION DISTINCT SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 ) alias1 order by key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 10 <- Union 8 (CONTAINS) Map 7 <- Union 8 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 9 <- Union 3 (CONTAINS), Union 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: src10_1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 10 Map Operator Tree: TableScan alias: src10_4 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 6 Map Operator Tree: TableScan alias: src10_2 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE Map 7 Map Operator Tree: TableScan alias: src10_3 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 2 Reduce Operator Tree: Merge Join Operator condition map: Inner Join 0 to 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reducer 5 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 9 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Union 3 Vertex: Union 3 Union 8 Vertex: Union 8 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: SELECT * FROM ( SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) UNION DISTINCT SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 ) alias1 order by key PREHOOK: type: QUERY PREHOOK: Input: default@src10_1 PREHOOK: Input: default@src10_2 PREHOOK: Input: default@src10_3 PREHOOK: Input: default@src10_4 #### A masked pattern was here #### POSTHOOK: query: SELECT * FROM ( SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) UNION DISTINCT SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 ) alias1 order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@src10_1 POSTHOOK: Input: default@src10_2 POSTHOOK: Input: default@src10_3 POSTHOOK: Input: default@src10_4 #### A masked pattern was here #### 165 val_165 238 val_238 255 val_255 27 val_27 278 val_278 311 val_311 409 val_409 484 val_484 86 val_86 98 val_98 PREHOOK: query: -- union4.q -- union case: both subqueries are map-reduce jobs on same input, followed by filesink drop table if exists tmptable PREHOOK: type: DROPTABLE PREHOOK: Input: default@tmptable PREHOOK: Output: default@tmptable POSTHOOK: query: -- union4.q -- union case: both subqueries are map-reduce jobs on same input, followed by filesink drop table if exists tmptable POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@tmptable POSTHOOK: Output: default@tmptable PREHOOK: query: create table tmptable(key string, value int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tmptable POSTHOOK: query: create table tmptable(key string, value int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@tmptable PREHOOK: query: explain insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2) unionsrc PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2) unionsrc POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 5 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable Reducer 6 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst2' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Union 3 Vertex: Union 3 Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable Stage: Stage-3 Stats-Aggr Operator PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2) unionsrc PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@tmptable POSTHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2) unionsrc POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@tmptable POSTHOOK: Lineage: tmptable.key EXPRESSION [] POSTHOOK: Lineage: tmptable.value EXPRESSION [] PREHOOK: query: select * from tmptable x sort by x.key PREHOOK: type: QUERY PREHOOK: Input: default@tmptable #### A masked pattern was here #### POSTHOOK: query: select * from tmptable x sort by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@tmptable #### A masked pattern was here #### tst1 500 tst2 500 PREHOOK: query: -- union5.q -- SORT_BEFORE_DIFF -- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink explain select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key PREHOOK: type: QUERY POSTHOOK: query: -- union5.q -- SORT_BEFORE_DIFF -- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink explain select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 5 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 6 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst2' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string) Union 3 Vertex: Union 3 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 UNION DISTINCT select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### tst1 1 tst2 1 PREHOOK: query: -- union6.q -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink drop table if exists tmptable PREHOOK: type: DROPTABLE PREHOOK: Input: default@tmptable PREHOOK: Output: default@tmptable POSTHOOK: query: -- union6.q -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink drop table if exists tmptable POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@tmptable POSTHOOK: Output: default@tmptable PREHOOK: query: create table tmptable(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tmptable POSTHOOK: query: create table tmptable(key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@tmptable PREHOOK: query: explain insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src1 s2) unionsrc PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src1 s2) unionsrc POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 5 <- Union 3 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 5 Map Operator Tree: TableScan alias: s2 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable Union 3 Vertex: Union 3 Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable Stage: Stage-3 Stats-Aggr Operator PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src1 s2) unionsrc PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@src1 PREHOOK: Output: default@tmptable POSTHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src1 s2) unionsrc POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 POSTHOOK: Output: default@tmptable POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tmptable.value EXPRESSION [(src1)s2.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: select * from tmptable x sort by x.key, x.value PREHOOK: type: QUERY PREHOOK: Input: default@tmptable #### A masked pattern was here #### POSTHOOK: query: select * from tmptable x sort by x.key, x.value POSTHOOK: type: QUERY POSTHOOK: Input: default@tmptable #### A masked pattern was here #### val_165 val_193 val_265 val_27 val_409 val_484 128 146 val_146 150 val_150 213 val_213 224 238 val_238 255 val_255 273 val_273 278 val_278 311 val_311 369 401 val_401 406 val_406 66 val_66 98 val_98 tst1 500 PREHOOK: query: -- union7.q -- SORT_BEFORE_DIFF -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink explain select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key PREHOOK: type: QUERY POSTHOOK: query: -- union7.q -- SORT_BEFORE_DIFF -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink explain select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 5 <- Union 3 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Map 5 Map Operator Tree: TableScan alias: s2 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Select Operator expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL File Output Operator compressed: false Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 3 Vertex: Union 3 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@src1 #### A masked pattern was here #### POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### 7 128 1 146 1 150 1 213 1 224 1 238 1 255 1 273 1 278 1 311 1 369 1 401 1 406 1 66 1 98 1 tst1 1 PREHOOK: query: -- union8.q -- SORT_BEFORE_DIFF -- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by filesink explain select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2 UNION DISTINCT select s3.key as key, s3.value as value from src s3) unionsrc PREHOOK: type: QUERY POSTHOOK: query: -- union8.q -- SORT_BEFORE_DIFF -- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by filesink explain select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2 UNION DISTINCT select s3.key as key, s3.value as value from src s3) unionsrc POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 6 <- Union 2 (CONTAINS) Map 7 <- Union 4 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) Reducer 5 <- Union 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 6 Map Operator Tree: TableScan alias: s1 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 7 Map Operator Tree: TableScan alias: s1 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 5 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 Union 4 Vertex: Union 4 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2 UNION DISTINCT select s3.key as key, s3.value as value from src s3) unionsrc PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2 UNION DISTINCT select s3.key as key, s3.value as value from src s3) unionsrc POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 0 val_0 10 val_10 100 val_100 103 val_103 104 val_104 105 val_105 11 val_11 111 val_111 113 val_113 114 val_114 116 val_116 118 val_118 119 val_119 12 val_12 120 val_120 125 val_125 126 val_126 128 val_128 129 val_129 131 val_131 133 val_133 134 val_134 136 val_136 137 val_137 138 val_138 143 val_143 145 val_145 146 val_146 149 val_149 15 val_15 150 val_150 152 val_152 153 val_153 155 val_155 156 val_156 157 val_157 158 val_158 160 val_160 162 val_162 163 val_163 164 val_164 165 val_165 166 val_166 167 val_167 168 val_168 169 val_169 17 val_17 170 val_170 172 val_172 174 val_174 175 val_175 176 val_176 177 val_177 178 val_178 179 val_179 18 val_18 180 val_180 181 val_181 183 val_183 186 val_186 187 val_187 189 val_189 19 val_19 190 val_190 191 val_191 192 val_192 193 val_193 194 val_194 195 val_195 196 val_196 197 val_197 199 val_199 2 val_2 20 val_20 200 val_200 201 val_201 202 val_202 203 val_203 205 val_205 207 val_207 208 val_208 209 val_209 213 val_213 214 val_214 216 val_216 217 val_217 218 val_218 219 val_219 221 val_221 222 val_222 223 val_223 224 val_224 226 val_226 228 val_228 229 val_229 230 val_230 233 val_233 235 val_235 237 val_237 238 val_238 239 val_239 24 val_24 241 val_241 242 val_242 244 val_244 247 val_247 248 val_248 249 val_249 252 val_252 255 val_255 256 val_256 257 val_257 258 val_258 26 val_26 260 val_260 262 val_262 263 val_263 265 val_265 266 val_266 27 val_27 272 val_272 273 val_273 274 val_274 275 val_275 277 val_277 278 val_278 28 val_28 280 val_280 281 val_281 282 val_282 283 val_283 284 val_284 285 val_285 286 val_286 287 val_287 288 val_288 289 val_289 291 val_291 292 val_292 296 val_296 298 val_298 30 val_30 302 val_302 305 val_305 306 val_306 307 val_307 308 val_308 309 val_309 310 val_310 311 val_311 315 val_315 316 val_316 317 val_317 318 val_318 321 val_321 322 val_322 323 val_323 325 val_325 327 val_327 33 val_33 331 val_331 332 val_332 333 val_333 335 val_335 336 val_336 338 val_338 339 val_339 34 val_34 341 val_341 342 val_342 344 val_344 345 val_345 348 val_348 35 val_35 351 val_351 353 val_353 356 val_356 360 val_360 362 val_362 364 val_364 365 val_365 366 val_366 367 val_367 368 val_368 369 val_369 37 val_37 373 val_373 374 val_374 375 val_375 377 val_377 378 val_378 379 val_379 382 val_382 384 val_384 386 val_386 389 val_389 392 val_392 393 val_393 394 val_394 395 val_395 396 val_396 397 val_397 399 val_399 4 val_4 400 val_400 401 val_401 402 val_402 403 val_403 404 val_404 406 val_406 407 val_407 409 val_409 41 val_41 411 val_411 413 val_413 414 val_414 417 val_417 418 val_418 419 val_419 42 val_42 421 val_421 424 val_424 427 val_427 429 val_429 43 val_43 430 val_430 431 val_431 432 val_432 435 val_435 436 val_436 437 val_437 438 val_438 439 val_439 44 val_44 443 val_443 444 val_444 446 val_446 448 val_448 449 val_449 452 val_452 453 val_453 454 val_454 455 val_455 457 val_457 458 val_458 459 val_459 460 val_460 462 val_462 463 val_463 466 val_466 467 val_467 468 val_468 469 val_469 47 val_47 470 val_470 472 val_472 475 val_475 477 val_477 478 val_478 479 val_479 480 val_480 481 val_481 482 val_482 483 val_483 484 val_484 485 val_485 487 val_487 489 val_489 490 val_490 491 val_491 492 val_492 493 val_493 494 val_494 495 val_495 496 val_496 497 val_497 498 val_498 5 val_5 51 val_51 53 val_53 54 val_54 57 val_57 58 val_58 64 val_64 65 val_65 66 val_66 67 val_67 69 val_69 70 val_70 72 val_72 74 val_74 76 val_76 77 val_77 78 val_78 8 val_8 80 val_80 82 val_82 83 val_83 84 val_84 85 val_85 86 val_86 87 val_87 9 val_9 90 val_90 92 val_92 95 val_95 96 val_96 97 val_97 98 val_98 PREHOOK: query: -- union9.q -- SORT_BEFORE_DIFF -- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by reducesink explain select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2 UNION DISTINCT select s3.key as key, s3.value as value from src s3) unionsrc PREHOOK: type: QUERY POSTHOOK: query: -- union9.q -- SORT_BEFORE_DIFF -- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by reducesink explain select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2 UNION DISTINCT select s3.key as key, s3.value as value from src s3) unionsrc POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 Tez Edges: Map 1 <- Union 2 (CONTAINS) Map 7 <- Union 2 (CONTAINS) Map 8 <- Union 4 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) Reducer 5 <- Union 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s1 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 7 Map Operator Tree: TableScan alias: s1 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Map 8 Map Operator Tree: TableScan alias: s1 Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Reducer 5 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 6 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 Union 4 Vertex: Union 4 Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink PREHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2 UNION DISTINCT select s3.key as key, s3.value as value from src s3) unionsrc PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2 UNION DISTINCT select s3.key as key, s3.value as value from src s3) unionsrc POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 309