PREHOOK: query: -- SORT_QUERY_RESULTS create table src_10 as select * from src limit 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@src_10 POSTHOOK: query: -- SORT_QUERY_RESULTS create table src_10 as select * from src limit 10 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@src_10 PREHOOK: query: create table src_lv1 (key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@src_lv1 POSTHOOK: query: create table src_lv1 (key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@src_lv1 PREHOOK: query: create table src_lv2 (key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@src_lv2 POSTHOOK: query: create table src_lv2 (key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@src_lv2 PREHOOK: query: create table src_lv3 (key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@src_lv3 POSTHOOK: query: create table src_lv3 (key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@src_lv3 PREHOOK: query: -- 2LV -- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-FS[12] -- -SEL[3]-UDTF[4]-LVJ[5] -- -LVF[6]-SEL[7]-LVJ[10]-SEL[13]-FS[14] -- -SEL[8]-UDTF[9]-LVJ[10] explain from src_10 insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C PREHOOK: type: QUERY POSTHOOK: query: -- 2LV -- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-FS[12] -- -SEL[3]-UDTF[4]-LVJ[5] -- -LVF[6]-SEL[7]-LVJ[10]-SEL[13]-FS[14] -- -SEL[8]-UDTF[9]-LVJ[10] explain from src_10 insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 Stage-6 Stage-7 depends on stages: Stage-6 Stage-14 depends on stages: Stage-2 , consists of Stage-11, Stage-10, Stage-12 Stage-11 Stage-1 depends on stages: Stage-11, Stage-10, Stage-13 Stage-9 depends on stages: Stage-1 Stage-10 Stage-12 Stage-13 depends on stages: Stage-12 STAGE PLANS: Stage: Stage-2 Map Reduce Map Operator Tree: TableScan alias: src_10 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Forward Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col5 (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 Select Operator expressions: array((key + 1),(key + 2)) (type: array) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE UDTF Operator Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col5 (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 Lateral View Forward Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col5 (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 Select Operator expressions: array((key + 3),(key + 4)) (type: array) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE UDTF Operator Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col5 (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 Stage: Stage-8 Conditional Operator Stage: Stage-5 Move Operator files: hdfs directory: true #### A masked pattern was here #### Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 Stage: Stage-3 Stats-Aggr Operator Stage: Stage-4 Map Reduce Map Operator Tree: TableScan File Output Operator compressed: false table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 Stage: Stage-6 Map Reduce Map Operator Tree: TableScan File Output Operator compressed: false table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 Stage: Stage-7 Move Operator files: hdfs directory: true #### A masked pattern was here #### Stage: Stage-14 Conditional Operator Stage: Stage-11 Move Operator files: hdfs directory: true #### A masked pattern was here #### Stage: Stage-1 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 Stage: Stage-9 Stats-Aggr Operator Stage: Stage-10 Map Reduce Map Operator Tree: TableScan File Output Operator compressed: false table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 Stage: Stage-12 Map Reduce Map Operator Tree: TableScan File Output Operator compressed: false table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 Stage: Stage-13 Move Operator files: hdfs directory: true #### A masked pattern was here #### PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C PREHOOK: type: QUERY PREHOOK: Input: default@src_10 PREHOOK: Output: default@src_lv1 PREHOOK: Output: default@src_lv2 POSTHOOK: query: from src_10 insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C POSTHOOK: type: QUERY POSTHOOK: Input: default@src_10 POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv1.value SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct, comment:), ] POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv2.value SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct, comment:), ] PREHOOK: query: select * from src_lv1 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv1 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv1 #### A masked pattern was here #### 165 166.0 165 167.0 238 239.0 238 240.0 255 256.0 255 257.0 27 28.0 27 29.0 278 279.0 278 280.0 311 312.0 311 313.0 409 410.0 409 411.0 484 485.0 484 486.0 86 87.0 86 88.0 98 100.0 98 99.0 PREHOOK: query: select * from src_lv2 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv2 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv2 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv2 #### A masked pattern was here #### 165 168.0 165 169.0 238 241.0 238 242.0 255 258.0 255 259.0 27 30.0 27 31.0 278 281.0 278 282.0 311 314.0 311 315.0 409 412.0 409 413.0 484 487.0 484 488.0 86 89.0 86 90.0 98 101.0 98 102.0 PREHOOK: query: -- 2(LV+GBY) -- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16] -- -SEL[3]-UDTF[4]-LVJ[5] -- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22] -- -SEL[8]-UDTF[9]-LVJ[10] explain from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key PREHOOK: type: QUERY POSTHOOK: query: -- 2(LV+GBY) -- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16] -- -SEL[3]-UDTF[4]-LVJ[5] -- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22] -- -SEL[8]-UDTF[9]-LVJ[10] explain from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 Stage-4 depends on stages: Stage-2 Stage-1 depends on stages: Stage-4 Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 Map Reduce Map Operator Tree: TableScan alias: src_10 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Forward Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col5) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Select Operator expressions: array((key + 1),(key + 2)) (type: array) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE UDTF Operator Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col5) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Lateral View Forward Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col5) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Select Operator expressions: array((key + 3),(key + 4)) (type: array) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE UDTF Operator Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col5) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 Stage: Stage-3 Stats-Aggr Operator Stage: Stage-4 Map Reduce Map Operator Tree: TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 Stage: Stage-1 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 Stage: Stage-5 Stats-Aggr Operator PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key PREHOOK: type: QUERY PREHOOK: Input: default@src_10 PREHOOK: Output: default@src_lv1 PREHOOK: Output: default@src_lv2 POSTHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_10 POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct, comment:), ] POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct, comment:), ] PREHOOK: query: select * from src_lv1 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv1 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv1 #### A masked pattern was here #### 165 333.0 238 479.0 255 513.0 27 57.0 278 559.0 311 625.0 409 821.0 484 971.0 86 175.0 98 199.0 PREHOOK: query: select * from src_lv2 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv2 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv2 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv2 #### A masked pattern was here #### 165 337.0 238 483.0 255 517.0 27 61.0 278 563.0 311 629.0 409 825.0 484 975.0 86 179.0 98 203.0 PREHOOK: query: -- (LV+GBY) + RS:2GBY -- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[6]-GBY[7]-RS[8]-GBY[9]-SEL[10]-FS[11] -- -SEL[3]-UDTF[4]-LVJ[5] -- -FIL[12]-SEL[13]-RS[14]-FOR[15]-FIL[16]-GBY[17]-SEL[18]-FS[19] -- -FIL[20]-GBY[21]-SEL[22]-FS[23] explain from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key insert overwrite table src_lv2 select key, count(value) where key > 200 group by key insert overwrite table src_lv3 select key, count(value) where key < 200 group by key PREHOOK: type: QUERY POSTHOOK: query: -- (LV+GBY) + RS:2GBY -- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[6]-GBY[7]-RS[8]-GBY[9]-SEL[10]-FS[11] -- -SEL[3]-UDTF[4]-LVJ[5] -- -FIL[12]-SEL[13]-RS[14]-FOR[15]-FIL[16]-GBY[17]-SEL[18]-FS[19] -- -FIL[20]-GBY[21]-SEL[22]-FS[23] explain from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key insert overwrite table src_lv2 select key, count(value) where key > 200 group by key insert overwrite table src_lv3 select key, count(value) where key < 200 group by key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-3 is a root stage Stage-0 depends on stages: Stage-3 Stage-4 depends on stages: Stage-0 Stage-5 depends on stages: Stage-3 Stage-1 depends on stages: Stage-5 Stage-6 depends on stages: Stage-1 Stage-2 depends on stages: Stage-5 Stage-7 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-3 Map Reduce Map Operator Tree: TableScan alias: src_10 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Forward Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col5) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Select Operator expressions: array((key + 1),(key + 2)) (type: array) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE UDTF Operator Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col5) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Filter Operator predicate: ((key > 200) or (key < 200)) (type: boolean) Statistics: Num rows: 6 Data size: 62 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 Stage: Stage-4 Stats-Aggr Operator Stage: Stage-5 Map Reduce Map Operator Tree: TableScan Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 6 Data size: 62 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Forward Statistics: Num rows: 6 Data size: 62 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (KEY._col0 > 200) (type: boolean) Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 Filter Operator predicate: (KEY._col0 < 200) (type: boolean) Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv3 Stage: Stage-1 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 Stage: Stage-6 Stats-Aggr Operator Stage: Stage-2 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv3 Stage: Stage-7 Stats-Aggr Operator PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key insert overwrite table src_lv2 select key, count(value) where key > 200 group by key insert overwrite table src_lv3 select key, count(value) where key < 200 group by key PREHOOK: type: QUERY PREHOOK: Input: default@src_10 PREHOOK: Output: default@src_lv1 PREHOOK: Output: default@src_lv2 PREHOOK: Output: default@src_lv3 POSTHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key insert overwrite table src_lv2 select key, count(value) where key > 200 group by key insert overwrite table src_lv3 select key, count(value) where key < 200 group by key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_10 POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 POSTHOOK: Output: default@src_lv3 POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct, comment:), ] POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.value EXPRESSION [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: select * from src_lv1 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv1 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv1 #### A masked pattern was here #### 165 333.0 238 479.0 255 513.0 27 57.0 278 559.0 311 625.0 409 821.0 484 971.0 86 175.0 98 199.0 PREHOOK: query: select * from src_lv2 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv2 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv2 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv2 #### A masked pattern was here #### 238 1 255 1 278 1 311 1 409 1 484 1 PREHOOK: query: select * from src_lv3 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv3 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv3 #### A masked pattern was here #### 165 1 27 1 86 1 98 1 PREHOOK: query: -- todo: shared distinct columns (should work with hive.optimize.multigroupby.common.distincts) -- 2(LV+GBY) + RS:2GBY -- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16] -- -SEL[3]-UDTF[4]-LVJ[5] -- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22] -- -SEL[8]-UDTF[9]-LVJ[10] -- -SEL[23]-GBY[24]-RS[25]-GBY[26]-SEL[27]-FS[28] explain from src_10 insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C insert overwrite table src_lv3 select value, sum(distinct key) group by value PREHOOK: type: QUERY POSTHOOK: query: -- todo: shared distinct columns (should work with hive.optimize.multigroupby.common.distincts) -- 2(LV+GBY) + RS:2GBY -- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16] -- -SEL[3]-UDTF[4]-LVJ[5] -- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22] -- -SEL[8]-UDTF[9]-LVJ[10] -- -SEL[23]-GBY[24]-RS[25]-GBY[26]-SEL[27]-FS[28] explain from src_10 insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C insert overwrite table src_lv3 select value, sum(distinct key) group by value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-3 is a root stage Stage-0 depends on stages: Stage-3 Stage-4 depends on stages: Stage-0 Stage-5 depends on stages: Stage-3 Stage-1 depends on stages: Stage-5 Stage-6 depends on stages: Stage-1 Stage-7 depends on stages: Stage-3 Stage-2 depends on stages: Stage-7 Stage-8 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-3 Map Reduce Map Operator Tree: TableScan alias: src_10 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Forward Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col5 (type: double), _col0 (type: string) outputColumnNames: _col5, _col0 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(DISTINCT _col0) keys: _col5 (type: double), _col0 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: double), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: array((key + 1),(key + 2)) (type: array) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE UDTF Operator Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col5 (type: double), _col0 (type: string) outputColumnNames: _col5, _col0 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(DISTINCT _col0) keys: _col5 (type: double), _col0 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: double), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Lateral View Forward Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col5 (type: double), _col0 (type: string) outputColumnNames: _col5, _col0 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(DISTINCT _col0) keys: _col5 (type: double), _col0 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Select Operator expressions: array((key + 3),(key + 4)) (type: array) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE UDTF Operator Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col5 (type: double), _col0 (type: string) outputColumnNames: _col5, _col0 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(DISTINCT _col0) keys: _col5 (type: double), _col0 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Select Operator expressions: value (type: string), key (type: string) outputColumnNames: value, key Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(DISTINCT key) keys: value (type: string), key (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Reduce Operator Tree: Group By Operator aggregations: sum(DISTINCT KEY._col1:0._col0) keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 Stage: Stage-4 Stats-Aggr Operator Stage: Stage-5 Map Reduce Map Operator Tree: TableScan Reduce Output Operator key expressions: _col0 (type: double), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: sum(DISTINCT KEY._col1:0._col0) keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 Stage: Stage-1 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 Stage: Stage-6 Stats-Aggr Operator Stage: Stage-7 Map Reduce Map Operator Tree: TableScan Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: sum(DISTINCT KEY._col1:0._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv3 Stage: Stage-2 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv3 Stage: Stage-8 Stats-Aggr Operator PREHOOK: query: from src_10 insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C insert overwrite table src_lv3 select value, sum(distinct key) group by value PREHOOK: type: QUERY PREHOOK: Input: default@src_10 PREHOOK: Output: default@src_lv1 PREHOOK: Output: default@src_lv2 PREHOOK: Output: default@src_lv3 POSTHOOK: query: from src_10 insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C insert overwrite table src_lv3 select value, sum(distinct key) group by value POSTHOOK: type: QUERY POSTHOOK: Input: default@src_10 POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 POSTHOOK: Output: default@src_lv3 POSTHOOK: Lineage: src_lv1.key SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct, comment:), ] POSTHOOK: Lineage: src_lv1.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct, comment:), ] POSTHOOK: Lineage: src_lv2.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.value EXPRESSION [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] PREHOOK: query: select * from src_lv1 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv1 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv1 #### A masked pattern was here #### 100.0 98.0 166.0 165.0 167.0 165.0 239.0 238.0 240.0 238.0 256.0 255.0 257.0 255.0 279.0 278.0 28.0 27.0 280.0 278.0 29.0 27.0 312.0 311.0 313.0 311.0 410.0 409.0 411.0 409.0 485.0 484.0 486.0 484.0 87.0 86.0 88.0 86.0 99.0 98.0 PREHOOK: query: select * from src_lv2 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv2 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv2 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv2 #### A masked pattern was here #### 101.0 98.0 102.0 98.0 168.0 165.0 169.0 165.0 241.0 238.0 242.0 238.0 258.0 255.0 259.0 255.0 281.0 278.0 282.0 278.0 30.0 27.0 31.0 27.0 314.0 311.0 315.0 311.0 412.0 409.0 413.0 409.0 487.0 484.0 488.0 484.0 89.0 86.0 90.0 86.0 PREHOOK: query: select * from src_lv3 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv3 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv3 #### A masked pattern was here #### val_165 165.0 val_238 238.0 val_255 255.0 val_27 27.0 val_278 278.0 val_311 311.0 val_409 409.0 val_484 484.0 val_86 86.0 val_98 98.0 PREHOOK: query: create table src_lv4 (key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@src_lv4 POSTHOOK: query: create table src_lv4 (key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@src_lv4 PREHOOK: query: -- Common distincts optimization works across non-lateral view queries, but not across lateral view multi inserts explain from src_10 insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value PREHOOK: type: QUERY POSTHOOK: query: -- Common distincts optimization works across non-lateral view queries, but not across lateral view multi inserts explain from src_10 insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-4 is a root stage Stage-0 depends on stages: Stage-4 Stage-5 depends on stages: Stage-0 Stage-6 depends on stages: Stage-4 Stage-1 depends on stages: Stage-6 Stage-7 depends on stages: Stage-1 Stage-8 depends on stages: Stage-4 Stage-2 depends on stages: Stage-8 Stage-9 depends on stages: Stage-2 Stage-3 depends on stages: Stage-8 Stage-10 depends on stages: Stage-3 STAGE PLANS: Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: src_10 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Forward Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(DISTINCT _col5) keys: _col0 (type: string), _col5 (type: double) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: double) sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: array((key + 1),(key + 2)) (type: array) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE UDTF Operator Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(DISTINCT _col5) keys: _col0 (type: string), _col5 (type: double) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: double) sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Lateral View Forward Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(DISTINCT _col5) keys: _col0 (type: string), _col5 (type: double) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Select Operator expressions: array((key + 3),(key + 4)) (type: array) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE UDTF Operator Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator outputColumnNames: _col0, _col5 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(DISTINCT _col5) keys: _col0 (type: string), _col5 (type: double) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Filter Operator predicate: ((key > 200) or (key < 200)) (type: boolean) Statistics: Num rows: 6 Data size: 62 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Reduce Operator Tree: Group By Operator aggregations: sum(DISTINCT KEY._col1:0._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 Stage: Stage-5 Stats-Aggr Operator Stage: Stage-6 Map Reduce Map Operator Tree: TableScan Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: double) sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: sum(DISTINCT KEY._col1:0._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 Stage: Stage-1 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 Stage: Stage-7 Stats-Aggr Operator Stage: Stage-8 Map Reduce Map Operator Tree: TableScan Reduce Output Operator key expressions: value (type: string), key (type: string) sort order: ++ Map-reduce partition columns: value (type: string) Statistics: Num rows: 6 Data size: 62 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Forward Statistics: Num rows: 6 Data size: 62 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (KEY._col1:0._col0 > 200) (type: boolean) Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(DISTINCT KEY._col1:0._col0) keys: KEY._col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv3 Filter Operator predicate: (KEY._col1:0._col0 < 200) (type: boolean) Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(DISTINCT KEY._col1:0._col0) keys: KEY._col0 (type: string) mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv4 Stage: Stage-2 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv3 Stage: Stage-9 Stats-Aggr Operator Stage: Stage-3 Move Operator tables: replace: true table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv4 Stage: Stage-10 Stats-Aggr Operator PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value PREHOOK: type: QUERY PREHOOK: Input: default@src_10 PREHOOK: Output: default@src_lv1 PREHOOK: Output: default@src_lv2 PREHOOK: Output: default@src_lv3 PREHOOK: Output: default@src_lv4 POSTHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value POSTHOOK: type: QUERY POSTHOOK: Input: default@src_10 POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 POSTHOOK: Output: default@src_lv3 POSTHOOK: Output: default@src_lv4 POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct, comment:), ] POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct, comment:), ] POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.value EXPRESSION [(src_10)src_10.null, ] POSTHOOK: Lineage: src_lv4.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: src_lv4.value EXPRESSION [(src_10)src_10.null, ] PREHOOK: query: select * from src_lv1 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv1 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv1 #### A masked pattern was here #### 165 333.0 238 479.0 255 513.0 27 57.0 278 559.0 311 625.0 409 821.0 484 971.0 86 175.0 98 199.0 PREHOOK: query: select * from src_lv2 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv2 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv2 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv2 #### A masked pattern was here #### 165 337.0 238 483.0 255 517.0 27 61.0 278 563.0 311 629.0 409 825.0 484 975.0 86 179.0 98 203.0 PREHOOK: query: select * from src_lv3 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv3 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv3 #### A masked pattern was here #### val_238 238.0 val_255 255.0 val_278 278.0 val_311 311.0 val_409 409.0 val_484 484.0 PREHOOK: query: select * from src_lv4 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv4 #### A masked pattern was here #### POSTHOOK: query: select * from src_lv4 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_lv4 #### A masked pattern was here #### val_165 165.0 val_27 27.0 val_86 86.0 val_98 98.0